Merge branch 'main' into unpause_csi_volume_handle

project-koku · Jul 1, 2024 · 14ca6e4 · 14ca6e4
2 parents 6abac54 + a42cf32
commit 14ca6e4
Show file tree

Hide file tree

Showing 42 changed files with 417 additions and 81 deletions.
diff --git a/.env.example b/.env.example
@@ -23,6 +23,8 @@ PROMETHEUS_MULTIPROC_DIR='/tmp'
 CURRENCY_URL=https://open.er-api.com/v6/latest/USD
 UNLEASH_TOKEN='*:*.dbffffc83b1f92eeaf133a7eb878d4c58231acc159b5e1478ce53cfc'
 POLLING_TIMER=60 # Set how often you can trigger downloads per provider
+PROCESSING_WAIT_TIMER=3 # Set how many days before a long processing provider can be polled again
+LARGE_PROCESSING_WAIT_TIMER=7 # Set how many days before a long LARGE customer processing provider can be polled again
 MAX_GROUP_BY_OVERRIDE=3 # Set maximum aloud group bys
 TAG_ENABLED_LIMIT=200 # Set the max amount of tags per account
 DELAYED_TASK_TIME=30 # Set the seconds before a delayed summary task should expire

diff --git a/Dockerfile b/Dockerfile
@@ -1,3 +1,5 @@
+ARG TARGETARCH
+
 FROM registry.access.redhat.com/ubi8/ubi-minimal:latest AS base
 
 USER root
@@ -61,8 +63,6 @@ RUN ldconfig
 # No intermetiate steps for x86_64, but declare it so it can be used for the final image
 FROM --platform=amd64 base AS stage-amd64
 
-ARG TARGETARCH
-
 FROM stage-${TARGETARCH} AS final
 # PIPENV_DEV is set to true in the docker-compose allowing
 # local builds to install the dev dependencies

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -22,7 +22,7 @@ def secrets = [
 def configuration = [vaultUrl: params.VAULT_ADDRESS, vaultCredentialId: params.VAULT_CREDS_ID, engineVersion: 1]
 
 pipeline {
-    agent { label 'insights' }
+    agent { label 'rhel8' }
     options {
         timestamps()
     }

diff --git a/build_deploy.sh b/build_deploy.sh
@@ -30,13 +30,9 @@ function job_cleanup() {
 
 trap job_cleanup EXIT ERR SIGINT SIGTERM
 
-DOCKER_CONF="$TMP_JOB_DIR/.docker"
-mkdir -p "$DOCKER_CONF"
-docker --config="$DOCKER_CONF" login -u="$QUAY_USER" -p="$QUAY_TOKEN" quay.io
-docker --config="$DOCKER_CONF" build --build-arg GIT_COMMIT="$GIT_COMMIT" -t "${IMAGE}:${IMAGE_TAG}" .
-docker --config="$DOCKER_CONF" push "${IMAGE}:${IMAGE_TAG}"
+podman login -u="$QUAY_USER" -p="$QUAY_TOKEN" quay.io
+podman build --build-arg GIT_COMMIT="$GIT_COMMIT" -t "${IMAGE}:${IMAGE_TAG}" .
+podman push "${IMAGE}:${IMAGE_TAG}"
 
-docker --config="$DOCKER_CONF" tag "${IMAGE}:${IMAGE_TAG}" "${IMAGE}:latest"
-docker --config="$DOCKER_CONF" push "${IMAGE}:latest"
-
-docker --config="$DOCKER_CONF" logout
+podman tag "${IMAGE}:${IMAGE_TAG}" "${IMAGE}:latest"
+podman push "${IMAGE}:latest"
diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml
@@ -492,6 +492,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: KAFKA_CONNECT
@@ -671,6 +675,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -1150,6 +1158,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -1326,6 +1338,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -1508,6 +1524,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -1690,6 +1710,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -1874,6 +1898,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST
@@ -2060,6 +2088,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST
@@ -2246,6 +2278,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST
@@ -2430,6 +2466,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST
@@ -2614,6 +2654,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST
@@ -2798,6 +2842,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST
@@ -2984,6 +3032,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: DELAYED_TASK_TIME
           value: ${DELAYED_TASK_TIME}
         - name: DELAYED_TASK_POLLING_MINUTES
@@ -3174,6 +3226,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: DELAYED_TASK_TIME
           value: ${DELAYED_TASK_TIME}
         - name: DELAYED_TASK_POLLING_MINUTES
@@ -3364,6 +3420,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: DELAYED_TASK_TIME
           value: ${DELAYED_TASK_TIME}
         - name: DELAYED_TASK_POLLING_MINUTES
@@ -3552,6 +3612,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -3734,6 +3798,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -3916,6 +3984,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -4098,6 +4170,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: DELAYED_TASK_TIME
           value: ${DELAYED_TASK_TIME}
         - name: DELAYED_TASK_POLLING_MINUTES
@@ -4284,6 +4360,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: DELAYED_TASK_TIME
           value: ${DELAYED_TASK_TIME}
         - name: DELAYED_TASK_POLLING_MINUTES
@@ -4470,6 +4550,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: DELAYED_TASK_TIME
           value: ${DELAYED_TASK_TIME}
         - name: DELAYED_TASK_POLLING_MINUTES
@@ -4658,6 +4742,10 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT
@@ -5554,6 +5642,12 @@ parameters:
 - displayName: Provider count for batch polling
   name: POLLING_BATCH_SIZE
   value: "100"
+- displayName: Timer to prevent triggering tasks while still processing in days
+  name: PROCESSING_WAIT_TIMER
+  value: "3"
+- displayName: Timer to prevent triggering tasks while still processing in days
+  name: LARGE_PROCESSING_WAIT_TIMER
+  value: "7"
 - displayName: Enable Tags Limit
   name: TAG_ENABLED_LIMIT
   value: "200"

diff --git a/deploy/kustomize/base/base.yaml b/deploy/kustomize/base/base.yaml
@@ -515,6 +515,12 @@ parameters:
 - displayName: Provider count for batch polling
   name: POLLING_BATCH_SIZE
   value: "100"
+- displayName: Timer to prevent triggering tasks while still processing in days
+  name: PROCESSING_WAIT_TIMER
+  value: "3"
+- displayName: Timer to prevent triggering tasks while still processing in days
+  name: LARGE_PROCESSING_WAIT_TIMER
+  value: "7"
 - displayName: Enable Tags Limit
   name: TAG_ENABLED_LIMIT
   value: "200"

diff --git a/deploy/kustomize/patches/masu.yaml b/deploy/kustomize/patches/masu.yaml
@@ -106,6 +106,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: KAFKA_CONNECT

diff --git a/deploy/kustomize/patches/scheduler.yaml b/deploy/kustomize/patches/scheduler.yaml
@@ -101,6 +101,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT

diff --git a/deploy/kustomize/patches/worker-celery.yaml b/deploy/kustomize/patches/worker-celery.yaml
@@ -91,6 +91,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT

diff --git a/deploy/kustomize/patches/worker-cost-model-penalty.yaml b/deploy/kustomize/patches/worker-cost-model-penalty.yaml
@@ -91,6 +91,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT

diff --git a/deploy/kustomize/patches/worker-cost-model-xl.yaml b/deploy/kustomize/patches/worker-cost-model-xl.yaml
@@ -91,6 +91,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT

diff --git a/deploy/kustomize/patches/worker-cost-model.yaml b/deploy/kustomize/patches/worker-cost-model.yaml
@@ -91,6 +91,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TRINO_HOST
           value: ${TRINO_HOST}
         - name: TRINO_PORT

diff --git a/deploy/kustomize/patches/worker-download-penalty.yaml b/deploy/kustomize/patches/worker-download-penalty.yaml
@@ -93,6 +93,10 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: PROCESSING_WAIT_TIMER
+          value: ${PROCESSING_WAIT_TIMER}
+        - name: LARGE_PROCESSING_WAIT_TIMER
+          value: ${LARGE_PROCESSING_WAIT_TIMER}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
         - name: TRINO_HOST