Merge pull request #164 from wri/run-dist

Schedule DIST-alerts sync for 1pm PST every day on production

src/datapump/sync/sync.py

@@ -21,6 +21,7 @@
 from ..util.gcs import get_gs_file_as_text, get_gs_files, get_gs_subfolders
 from ..util.models import ContentDateRange
 from ..util.util import log_and_notify_error
+from ..util.slack import slack_webhook
 
 
 class Sync(ABC):
@@ -769,6 +770,8 @@ def build_jobs (self, config: DatapumpConfig) -> List[Job]:
 
         jobs: List[Job] = []
 
+        slack_webhook("INFO", f"Starting dist-alerts jobs for {self.dataset_name}/{latest_release}")
+
         job = RasterVersionUpdateJob(
             # Current week alerts tile set
             id=str(uuid1()),

terraform/modules/datapump/cloudwatch.tf

@@ -1,3 +1,6 @@
+# Note: when it says est/EST here, it really means PST (Pacific Standard time)
+# The hours in the cron() expression are in UTC.
+
 resource "aws_cloudwatch_event_rule" "everyday-11-pm-est" {
   name                = substr("everyday-11-pm-est${local.name_suffix}", 0, 64)
   description         = "Run everyday at 11 pm EST"
@@ -26,6 +29,15 @@ resource "aws_cloudwatch_event_rule" "everyday-3-am-est" {
   tags                = local.tags
 }
 
+resource "aws_cloudwatch_event_rule" "everyday-1-pm-est" {
+  name                = substr("everyday-1-pm-est${local.name_suffix}", 0, 64)
+  description         = "Run everyday at 1 pm EST"
+  schedule_expression = "cron(0 21 ? * * *)"
+  tags                = local.tags
+}
+
+# The count condition in each of the resources below ensures that the CloudWatch
+# events only happen in production.
 resource "aws_cloudwatch_event_target" "sync-areas" {
   rule      = aws_cloudwatch_event_rule.everyday-7-pm-est.name
   target_id = substr("${local.project}-sync-areas${local.name_suffix}", 0, 64)
@@ -78,4 +90,15 @@ resource "aws_cloudwatch_event_target" "sync-integrated-alerts" {
   input    = "{\"command\": \"sync\", \"parameters\": {\"types\": [\"integrated_alerts\"]}}"
   role_arn  = aws_iam_role.datapump_states.arn
   count     = var.environment == "production" ? 1 : 0
-}
+}
+
+# Run every day at 1pm PST, but new data from UMD should only be available on Saturday morning,
+# so should only do a full run generating a new version once a week on Saturday/Sunday.
+resource "aws_cloudwatch_event_target" "sync-dist-alerts" {
+  rule      = aws_cloudwatch_event_rule.everyday-1-pm-est.name
+  target_id = substr("${local.project}-sync-umd-glad-dist-alerts${local.name_suffix}", 0, 64)
+  arn       = aws_sfn_state_machine.datapump.id
+  input    = "{\"command\": \"sync\", \"parameters\": {\"types\": [\"umd_glad_dist_alerts\"]}}"
+  role_arn  = aws_iam_role.datapump_states.arn
+  count     = var.environment == "production" ? 1 : 0
+}