Merge pull request #217 from NHSDigital/release/2024-06-04

Release/2024-06-04
NHSDigital · Jun 14, 2024 · 5c4692c · 5c4692c
2 parents bbcc01a + 213f400
commit 5c4692c
Show file tree

Hide file tree

Showing 232 changed files with 7,815 additions and 376 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -4,7 +4,7 @@ repos:
     rev: v1.4.0
     hooks:
       - id: detect-secrets
-        exclude: ".pre-commit-config.yaml|infrastructure/localstack/provider.tf|src/api/searchDevice/src/data/response.py"
+        exclude: ".pre-commit-config.yaml|infrastructure/localstack/provider.tf|src/api/searchDevice/src/data/response.py|src/etl/sds/tests/changelog"
 
   - repo: https://github.com/prettier/pre-commit
     rev: 57f39166b5a5a504d6808b87ab98d41ebf095b46

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Changelog
 
+## 2024-06-04
+- [PI-322] State machine lambda error messages truncated
+- [PI-346] Update now includes modify changes, plus testing suite
+- [PI-347] Updates implemented sequentially
+- [PI-358] Disable SDS ETL update timer on persistent environments
+
 ## 2024-05-31
 - [PI-342] Add redacted-fields to context loggers
 - [PI-376] Check releases fully rebased on main

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-2024.05.31
+2024.06.04
diff --git a/changelog/2024-06-04.md b/changelog/2024-06-04.md
@@ -0,0 +1,4 @@
+- [PI-322] State machine lambda error messages truncated
+- [PI-346] Update now includes modify changes, plus testing suite
+- [PI-347] Updates implemented sequentially
+- [PI-358] Disable SDS ETL update timer on persistent environments
diff --git a/...e/terraform/per_workspace/modules/etl/sds/etl-diagram--update-transform-and-load.asl.json b/...e/terraform/per_workspace/modules/etl/sds/etl-diagram--update-transform-and-load.asl.json
@@ -0,0 +1,91 @@
+{
+  "Comment": "SDS ETL - Sequential Transform and Load for updates",
+  "TimeoutSeconds": 60,
+  "StartAt": "load",
+  "States": {
+    "load": {
+      "Type": "Task",
+      "Resource": "arn:aws:states:::lambda:invoke",
+      "OutputPath": "$.Payload",
+      "Parameters": {
+        "FunctionName": "${load_worker_arn}:$LATEST",
+        "Payload": {
+          "max_records": "1"
+        }
+      },
+      "Retry": [
+        {
+          "ErrorEquals": [
+            "Lambda.ServiceException",
+            "Lambda.AWSLambdaException",
+            "Lambda.SdkClientException",
+            "Lambda.TooManyRequestsException"
+          ],
+          "IntervalSeconds": 1,
+          "MaxAttempts": 3,
+          "BackoffRate": 2
+        }
+      ],
+      "Next": "load-result"
+    },
+    "load-result": {
+      "Type": "Choice",
+      "Choices": [
+        {
+          "Not": {
+            "Variable": "$.error_message",
+            "IsNull": true
+          },
+          "Next": "etl-stopped"
+        },
+        {
+          "And": [
+            {
+              "Variable": "$.error_message",
+              "IsNull": true
+            },
+            {
+              "Not": {
+                "Variable": "$.unprocessed_records",
+                "NumericEquals": 0
+              }
+            }
+          ],
+          "Next": "load"
+        }
+      ],
+      "Default": "transform"
+    },
+    "transform": {
+      "Type": "Task",
+      "Resource": "arn:aws:states:::lambda:invoke",
+      "OutputPath": "$.Payload",
+      "Parameters": {
+        "FunctionName": "${transform_worker_arn}:$LATEST",
+        "Payload": {
+          "max_records": "1",
+          "trust": false
+        }
+      },
+      "Retry": [
+        {
+          "ErrorEquals": [
+            "Lambda.ServiceException",
+            "Lambda.AWSLambdaException",
+            "Lambda.SdkClientException",
+            "Lambda.TooManyRequestsException"
+          ],
+          "IntervalSeconds": 1,
+          "MaxAttempts": 3,
+          "BackoffRate": 2
+        }
+      ],
+      "Next": "etl-stopped"
+    },
+
+    "etl-stopped": {
+      "Type": "Pass",
+      "End": true
+    }
+  }
+}
diff --git a/...ce/modules/etl/sds/step-function.asl.json → ...pace/modules/etl/sds/etl-diagram.asl.json b/...ce/modules/etl/sds/step-function.asl.json → ...pace/modules/etl/sds/etl-diagram.asl.json
@@ -13,7 +13,7 @@
               "Type": "Choice",
               "Choices": [
                 {
-                  "Variable": "$.changelog-number",
+                  "Variable": "$.changelog_number_end",
                   "IsPresent": true,
                   "Next": "write-changelog-number"
                 }
@@ -24,7 +24,7 @@
               "Type": "Task",
               "End": true,
               "Parameters": {
-                "Body.$": "$.changelog-number",
+                "Body.$": "$.changelog_number_end",
                 "Bucket": "${etl_bucket}",
                 "Key": "${changelog_key}"
               },
@@ -37,25 +37,29 @@
           }
         },
         {
-          "StartAt": "parse-init",
+          "StartAt": "parse-etl-type",
           "States": {
-            "parse-init": {
+            "parse-etl-type": {
               "Type": "Choice",
               "Choices": [
                 {
-                  "Variable": "$.init",
-                  "StringEquals": "extract",
-                  "Next": "extract"
+                  "Variable": "$.etl_type",
+                  "StringEquals": "bulk",
+                  "Next": "extract-bulk"
+                },
+                {
+                  "Variable": "$.etl_type",
+                  "StringEquals": "update",
+                  "Next": "extract-update"
                 }
-              ],
-              "Default": "invalid-init-value"
+              ]
             },
-            "extract": {
+            "extract-bulk": {
               "Type": "Task",
               "Resource": "arn:aws:states:::lambda:invoke",
               "OutputPath": "$.Payload",
               "Parameters": {
-                "Payload.$": "$",
+                "Payload": {},
                 "FunctionName": "${extract_worker_arn}:$LATEST"
               },
               "Retry": [
@@ -71,25 +75,27 @@
                   "BackoffRate": 2
                 }
               ],
-              "Next": "extract-result"
+              "Next": "extract-bulk-result"
             },
-            "extract-result": {
+            "extract-bulk-result": {
               "Type": "Choice",
               "Choices": [
                 {
                   "Variable": "$.error_message",
                   "IsNull": true,
-                  "Next": "transform"
+                  "Next": "transform-bulk"
                 }
               ],
-              "Default": "worker-failed"
+              "Default": "etl-stopped"
             },
-            "transform": {
+            "transform-bulk": {
               "Type": "Task",
               "Resource": "arn:aws:states:::lambda:invoke",
               "OutputPath": "$.Payload",
               "Parameters": {
-                "Payload.$": "$",
+                "Payload": {
+                  "trust": true
+                },
                 "FunctionName": "${transform_worker_arn}:$LATEST"
               },
               "Retry": [
@@ -105,25 +111,27 @@
                   "BackoffRate": 2
                 }
               ],
-              "Next": "transform-result"
+              "Next": "transform-bulk-result"
             },
-            "transform-result": {
+            "transform-bulk-result": {
               "Type": "Choice",
               "Choices": [
                 {
                   "Variable": "$.error_message",
                   "IsNull": true,
-                  "Next": "load"
+                  "Next": "load-bulk"
                 }
               ],
-              "Default": "worker-failed"
+              "Default": "etl-stopped"
             },
-            "load": {
+            "load-bulk": {
               "Type": "Task",
               "Resource": "arn:aws:states:::lambda:invoke",
               "OutputPath": "$.Payload",
               "Parameters": {
-                "Payload.$": "$",
+                "Payload": {
+                  "max_records": "${bulk_load_chunksize}"
+                },
                 "FunctionName": "${load_worker_arn}:$LATEST"
               },
               "Retry": [
@@ -139,18 +147,11 @@
                   "BackoffRate": 2
                 }
               ],
-              "Next": "load-result"
+              "Next": "load-bulk-result"
             },
-            "load-result": {
+            "load-bulk-result": {
               "Type": "Choice",
               "Choices": [
-                {
-                  "Not": {
-                    "Variable": "$.error_message",
-                    "IsNull": true
-                  },
-                  "Next": "worker-failed"
-                },
                 {
                   "And": [
                     {
@@ -164,23 +165,90 @@
                       }
                     }
                   ],
-                  "Next": "load"
+                  "Next": "load-bulk"
                 }
               ],
-              "Default": "load-successful"
+              "Default": "etl-stopped"
             },
-            "load-successful": {
-              "Type": "Pass",
-              "End": true
+            "extract-update": {
+              "Type": "Task",
+              "Resource": "arn:aws:states:::lambda:invoke",
+              "OutputPath": "$.Payload",
+              "Parameters": {
+                "Payload": {},
+                "FunctionName": "${extract_worker_arn}:$LATEST"
+              },
+              "Retry": [
+                {
+                  "ErrorEquals": [
+                    "Lambda.ServiceException",
+                    "Lambda.AWSLambdaException",
+                    "Lambda.SdkClientException",
+                    "Lambda.TooManyRequestsException"
+                  ],
+                  "IntervalSeconds": 1,
+                  "MaxAttempts": 3,
+                  "BackoffRate": 2
+                }
+              ],
+              "Next": "extract-update-result"
             },
-            "worker-failed": {
+            "extract-update-result": {
+              "Type": "Choice",
+              "Choices": [
+                {
+                  "Variable": "$.error_message",
+                  "IsNull": true,
+                  "Next": "etl-update"
+                }
+              ],
+              "Default": "etl-stopped"
+            },
+            "etl-update": {
+              "Type": "Task",
+              "Resource": "arn:aws:states:::states:startExecution.sync:2",
+              "Parameters": {
+                "StateMachineArn": "${etl_update_state_machine_arn}",
+                "Input": {
+                  "StatePayload": {},
+                  "AWS_STEP_FUNCTIONS_STARTED_BY_EXECUTION_ID.$": "$$.Execution.Id"
+                }
+              },
+              "OutputPath": "$.Output",
+              "Next": "etl-update-result"
+            },
+            "etl-update-result": {
+              "Type": "Choice",
+              "Choices": [
+                {
+                  "Or": [
+                    {
+                      "Not": {
+                        "Variable": "$.error_message",
+                        "IsNull": true
+                      }
+                    },
+                    {
+                      "And": [
+                        {
+                          "Variable": "$.processed_records",
+                          "NumericEquals": 0
+                        },
+                        {
+                          "Variable": "$.unprocessed_records",
+                          "NumericEquals": 0
+                        }
+                      ]
+                    }
+                  ],
+                  "Next": "etl-stopped"
+                }
+              ],
+              "Default": "etl-update"
+            },
+            "etl-stopped": {
               "Type": "Pass",
               "End": true
-            },
-            "invalid-init-value": {
-              "Type": "Fail",
-              "Error": "InvalidInitValue",
-              "Cause": "Invalid Value of Init"
             }
           }
         }