Enable copy-on-write in pandas

Testing out the new behavior that will be the default in Pandas 3.0 https://pandas.pydata.org/docs/dev/whatsnew/v2.2.0.html#copy-on-write
project-koku · Apr 4, 2024 · efcfec3 · efcfec3
1 parent 67bb9f3
commit efcfec3
Show file tree

Hide file tree

Showing 24 changed files with 98 additions and 0 deletions.
diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml
@@ -142,6 +142,8 @@ objects:
           value: ${CACHE_TIMEOUT}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         image: ${IMAGE}:${IMAGE_TAG}
         initContainers:
         - command:
@@ -315,6 +317,8 @@ objects:
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -528,6 +532,8 @@ objects:
           value: ${TAG_ENABLED_LIMIT}
         - name: KAFKA_CONNECT
           value: ${KAFKA_CONNECT}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: PROMETHEUS_PUSHGATEWAY
           value: ${PROMETHEUS_PUSHGATEWAY}
         - name: UNLEASH_CACHE_DIR
@@ -673,6 +679,8 @@ objects:
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -877,6 +885,8 @@ objects:
           value: ${RBAC_SERVICE_PATH}
         - name: RBAC_CACHE_TTL
           value: ${RBAC_CACHE_TTL}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: PROMETHEUS_MULTIPROC_DIR
           value: ${PROMETHEUS_DIR}
         - name: KOKU_ENABLE_SENTRY
@@ -1054,6 +1064,8 @@ objects:
           value: ${RBAC_SERVICE_PATH}
         - name: RBAC_CACHE_TTL
           value: ${RBAC_CACHE_TTL}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: PROMETHEUS_MULTIPROC_DIR
           value: ${PROMETHEUS_DIR}
         - name: KOKU_ENABLE_SENTRY
@@ -1212,6 +1224,8 @@ objects:
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -1408,6 +1422,8 @@ objects:
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -1610,6 +1626,8 @@ objects:
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -1814,6 +1832,8 @@ objects:
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
         - name: PANDAS_COLUMN_BATCH_SIZE
           value: ${PANDAS_COLUMN_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -2020,6 +2040,8 @@ objects:
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
         - name: PANDAS_COLUMN_BATCH_SIZE
           value: ${PANDAS_COLUMN_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -2224,6 +2246,8 @@ objects:
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -2428,6 +2452,8 @@ objects:
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -2634,6 +2660,8 @@ objects:
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
         - name: PANDAS_COLUMN_BATCH_SIZE
           value: ${PANDAS_COLUMN_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -2844,6 +2872,8 @@ objects:
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
         - name: PANDAS_COLUMN_BATCH_SIZE
           value: ${PANDAS_COLUMN_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -3052,6 +3082,8 @@ objects:
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -3254,6 +3286,8 @@ objects:
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -3456,6 +3490,8 @@ objects:
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -3662,6 +3698,8 @@ objects:
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -3870,6 +3908,8 @@ objects:
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -4072,6 +4112,8 @@ objects:
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -4238,6 +4280,8 @@ objects:
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY
@@ -4715,6 +4759,11 @@ parameters:
   name: PANDAS_COLUMN_BATCH_SIZE
   required: true
   value: "250"
+- description: Enable copy-on-write in Pandas
+  displayName: Pandas copy-on-write
+  name: PANDAS_COPY_ON_WRITE
+  required: true
+  value: "1"
 - description: Processing batch size
   displayName: Processing batch size
   name: REPORT_PROCESSING_BATCH_SIZE

diff --git a/deploy/kustomize/base/base.yaml b/deploy/kustomize/base/base.yaml
@@ -430,6 +430,11 @@ parameters:
   name: PANDAS_COLUMN_BATCH_SIZE
   required: true
   value: "250"
+- description: Enable copy-on-write in Pandas
+  displayName: Pandas copy-on-write
+  name: PANDAS_COPY_ON_WRITE
+  required: true
+  value: "1"
 - description: Processing batch size
   displayName: Processing batch size
   name: REPORT_PROCESSING_BATCH_SIZE

diff --git a/deploy/kustomize/patches/koku.yaml b/deploy/kustomize/patches/koku.yaml
@@ -143,6 +143,8 @@
           value: ${CACHE_TIMEOUT}
         - name: TAG_ENABLED_LIMIT
           value: ${TAG_ENABLED_LIMIT}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
       livenessProbe:
         httpGet:
           path: ${API_PATH_PREFIX}/v1/status/

diff --git a/deploy/kustomize/patches/listener.yaml b/deploy/kustomize/patches/listener.yaml
@@ -59,6 +59,8 @@
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY

diff --git a/deploy/kustomize/patches/masu.yaml b/deploy/kustomize/patches/masu.yaml
@@ -114,6 +114,8 @@
           value: ${TAG_ENABLED_LIMIT}
         - name: KAFKA_CONNECT
           value: ${KAFKA_CONNECT}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: PROMETHEUS_PUSHGATEWAY
           value: ${PROMETHEUS_PUSHGATEWAY}
         - name: UNLEASH_CACHE_DIR

diff --git a/deploy/kustomize/patches/scheduler.yaml b/deploy/kustomize/patches/scheduler.yaml
@@ -63,6 +63,8 @@
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY

diff --git a/deploy/kustomize/patches/sources-client.yaml b/deploy/kustomize/patches/sources-client.yaml
@@ -83,6 +83,8 @@
           value: ${RBAC_SERVICE_PATH}
         - name: RBAC_CACHE_TTL
           value: ${RBAC_CACHE_TTL}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: PROMETHEUS_MULTIPROC_DIR
           value: ${PROMETHEUS_DIR}
         - name: KOKU_ENABLE_SENTRY

diff --git a/deploy/kustomize/patches/sources-listener.yaml b/deploy/kustomize/patches/sources-listener.yaml
@@ -83,6 +83,8 @@
           value: ${RBAC_SERVICE_PATH}
         - name: RBAC_CACHE_TTL
           value: ${RBAC_CACHE_TTL}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: PROMETHEUS_MULTIPROC_DIR
           value: ${PROMETHEUS_DIR}
         - name: KOKU_ENABLE_SENTRY

diff --git a/deploy/kustomize/patches/worker-celery.yaml b/deploy/kustomize/patches/worker-celery.yaml
@@ -65,6 +65,8 @@
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY

diff --git a/deploy/kustomize/patches/worker-cost-model-xl.yaml b/deploy/kustomize/patches/worker-cost-model-xl.yaml
@@ -65,6 +65,8 @@
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY

diff --git a/deploy/kustomize/patches/worker-cost-model.yaml b/deploy/kustomize/patches/worker-cost-model.yaml
@@ -65,6 +65,8 @@
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY

diff --git a/deploy/kustomize/patches/worker-download-xl.yaml b/deploy/kustomize/patches/worker-download-xl.yaml
@@ -67,6 +67,8 @@
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
         - name: PANDAS_COLUMN_BATCH_SIZE
           value: ${PANDAS_COLUMN_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY

diff --git a/deploy/kustomize/patches/worker-download.yaml b/deploy/kustomize/patches/worker-download.yaml
@@ -67,6 +67,8 @@
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
         - name: PANDAS_COLUMN_BATCH_SIZE
           value: ${PANDAS_COLUMN_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY

diff --git a/deploy/kustomize/patches/worker-hcs.yaml b/deploy/kustomize/patches/worker-hcs.yaml
@@ -67,6 +67,8 @@
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY

diff --git a/deploy/kustomize/patches/worker-ocp-xl.yaml b/deploy/kustomize/patches/worker-ocp-xl.yaml
@@ -65,6 +65,8 @@
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY

diff --git a/deploy/kustomize/patches/worker-ocp.yaml b/deploy/kustomize/patches/worker-ocp.yaml
@@ -65,6 +65,8 @@
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY

diff --git a/deploy/kustomize/patches/worker-priority-xl.yaml b/deploy/kustomize/patches/worker-priority-xl.yaml
@@ -67,6 +67,8 @@
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
         - name: PANDAS_COLUMN_BATCH_SIZE
           value: ${PANDAS_COLUMN_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY

diff --git a/deploy/kustomize/patches/worker-priority.yaml b/deploy/kustomize/patches/worker-priority.yaml
@@ -67,6 +67,8 @@
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
         - name: PANDAS_COLUMN_BATCH_SIZE
           value: ${PANDAS_COLUMN_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY

diff --git a/deploy/kustomize/patches/worker-refresh-xl.yaml b/deploy/kustomize/patches/worker-refresh-xl.yaml
@@ -65,6 +65,8 @@
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY

diff --git a/deploy/kustomize/patches/worker-refresh.yaml b/deploy/kustomize/patches/worker-refresh.yaml
@@ -65,6 +65,8 @@
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY

diff --git a/deploy/kustomize/patches/worker-subs-extraction.yaml b/deploy/kustomize/patches/worker-subs-extraction.yaml
@@ -63,6 +63,8 @@
           value: ${ENABLE_S3_ARCHIVING}
         - name: PARQUET_PROCESSING_BATCH_SIZE
           value: ${PARQUET_PROCESSING_BATCH_SIZE}
+        - name: PANDAS_COPY_ON_WRITE
+          value: ${PANDAS_COPY_ON_WRITE}
         - name: TRINO_DATE_STEP
           value: ${TRINO_DATE_STEP}
         - name: KOKU_ENABLE_SENTRY