skip polling providers still processing (#5181)

* skip polling providers that are still processing
project-koku · Jun 28, 2024 · 4baa511 · 4baa511
1 parent 1810fa1
commit 4baa511
Show file tree

Hide file tree

Showing 29 changed files with 332 additions and 7 deletions.
diff --git a/.env.example b/.env.example
@@ -23,6 +23,8 @@ PROMETHEUS_MULTIPROC_DIR='/tmp'
 CURRENCY_URL=https://open.er-api.com/v6/latest/USD
 UNLEASH_TOKEN='*:*.dbffffc83b1f92eeaf133a7eb878d4c58231acc159b5e1478ce53cfc'
 POLLING_TIMER=60 # Set how often you can trigger downloads per provider
+PROCESSING_WAIT_TIMER=3 # Set how many days before a long processing provider can be polled again
+LARGE_PROCESSING_WAIT_TIMER=7 # Set how many days before a long LARGE customer processing provider can be polled again
 MAX_GROUP_BY_OVERRIDE=3 # Set maximum aloud group bys
 TAG_ENABLED_LIMIT=200 # Set the max amount of tags per account
 DELAYED_TASK_TIME=30 # Set the seconds before a delayed summary task should expire

diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml
@@ -492,6 +492,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: KAFKA_CONNECT
@@ -671,6 +675,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -1150,6 +1158,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -1326,6 +1338,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -1508,6 +1524,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -1690,6 +1710,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -1874,6 +1898,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST
@@ -2060,6 +2088,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST
@@ -2246,6 +2278,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST
@@ -2430,6 +2466,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST
@@ -2614,6 +2654,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST
@@ -2798,6 +2842,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST
@@ -2984,6 +3032,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: DELAYED_TASK_TIME
           value: ${DELAYED_TASK_TIME}
         - name: DELAYED_TASK_POLLING_MINUTES
@@ -3174,6 +3226,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: DELAYED_TASK_TIME
           value: ${DELAYED_TASK_TIME}
         - name: DELAYED_TASK_POLLING_MINUTES
@@ -3364,6 +3420,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: DELAYED_TASK_TIME
           value: ${DELAYED_TASK_TIME}
         - name: DELAYED_TASK_POLLING_MINUTES
@@ -3552,6 +3612,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -3734,6 +3798,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -3916,6 +3984,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -4098,6 +4170,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: DELAYED_TASK_TIME
           value: ${DELAYED_TASK_TIME}
         - name: DELAYED_TASK_POLLING_MINUTES
@@ -4284,6 +4360,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: DELAYED_TASK_TIME
           value: ${DELAYED_TASK_TIME}
         - name: DELAYED_TASK_POLLING_MINUTES
@@ -4470,6 +4550,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: DELAYED_TASK_TIME
           value: ${DELAYED_TASK_TIME}
         - name: DELAYED_TASK_POLLING_MINUTES
@@ -4658,6 +4742,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -5554,6 +5642,12 @@ parameters:
 - displayName: Provider count for batch polling
   name: POLLING_BATCH_SIZE
   value: "100"
+- displayName: Timer to prevent triggering tasks while still processing in days
+  name: PROCESSING_WAIT_TIMER
+  value: "3"
+- displayName: Timer to prevent triggering tasks while still processing in days
+  name: LARGE_PROCESSING_WAIT_TIMER
+  value: "7"
 - displayName: Enable Tags Limit
   name: TAG_ENABLED_LIMIT
   value: "200"

diff --git a/deploy/kustomize/base/base.yaml b/deploy/kustomize/base/base.yaml
@@ -515,6 +515,12 @@ parameters:
 - displayName: Provider count for batch polling
   name: POLLING_BATCH_SIZE
   value: "100"
+- displayName: Timer to prevent triggering tasks while still processing in days
+  name: PROCESSING_WAIT_TIMER
+  value: "3"
+- displayName: Timer to prevent triggering tasks while still processing in days
+  name: LARGE_PROCESSING_WAIT_TIMER
+  value: "7"
 - displayName: Enable Tags Limit
   name: TAG_ENABLED_LIMIT
   value: "200"

diff --git a/deploy/kustomize/patches/masu.yaml b/deploy/kustomize/patches/masu.yaml
@@ -106,6 +106,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: KAFKA_CONNECT

diff --git a/deploy/kustomize/patches/scheduler.yaml b/deploy/kustomize/patches/scheduler.yaml
@@ -101,6 +101,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT

diff --git a/deploy/kustomize/patches/worker-celery.yaml b/deploy/kustomize/patches/worker-celery.yaml
@@ -91,6 +91,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT

diff --git a/deploy/kustomize/patches/worker-cost-model-penalty.yaml b/deploy/kustomize/patches/worker-cost-model-penalty.yaml
@@ -91,6 +91,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT

diff --git a/deploy/kustomize/patches/worker-cost-model-xl.yaml b/deploy/kustomize/patches/worker-cost-model-xl.yaml
@@ -91,6 +91,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT

diff --git a/deploy/kustomize/patches/worker-cost-model.yaml b/deploy/kustomize/patches/worker-cost-model.yaml
@@ -91,6 +91,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT

diff --git a/deploy/kustomize/patches/worker-download-penalty.yaml b/deploy/kustomize/patches/worker-download-penalty.yaml
@@ -93,6 +93,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST

diff --git a/deploy/kustomize/patches/worker-download-xl.yaml b/deploy/kustomize/patches/worker-download-xl.yaml
@@ -93,6 +93,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST

diff --git a/deploy/kustomize/patches/worker-download.yaml b/deploy/kustomize/patches/worker-download.yaml
@@ -93,6 +93,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST

diff --git a/deploy/kustomize/patches/worker-hcs.yaml b/deploy/kustomize/patches/worker-hcs.yaml
@@ -93,6 +93,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT

diff --git a/deploy/kustomize/patches/worker-ocp-penalty.yaml b/deploy/kustomize/patches/worker-ocp-penalty.yaml
@@ -91,6 +91,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST