[celery] No over parallelism

Making part of the process synchronous (parallelize ony by schedd) will create less and small messages, and with a small number of workers will have a better performance.
dmwm · Jun 4, 2020 · 83516ca · 83516ca
1 parent 6ca7af9
commit 83516ca
Show file tree

Hide file tree

Showing 6 changed files with 15 additions and 13 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -49,7 +49,7 @@ services:
     restart: 'no'
     environment: *env
     # entrypoint: ["python","/cms-htcondor-es/tests/celery_test.py"]
-    entrypoint: ["python","/cms-htcondor-es/celery_spider_cms.py","--feed_es","--query_queue_batch_size","5000"]
+    entrypoint: ["python","/cms-htcondor-es/celery_spider_cms.py","--feed_es","--query_queue_batch_size","1000", "--amq_bunch_size", "500"]
     depends_on:
       - spider-worker
 

diff --git a/k8s/cronjobs/spider-cron-queues.yaml b/k8s/cronjobs/spider-cron-queues.yaml
@@ -19,7 +19,9 @@ spec:
             - /cms-htcondor-es/celery_spider_cms.py
             - --feed_es
             - --query_queue_batch_size
-            - "5000"
+            - "500"
+            - --amq_bunch_size
+            - "500"
             env: &spider_env
               - name: AFFILIATION_DIR_LOCATION
                 value: /cms_shared/affiliation_dir.json

diff --git a/k8s/deployments/spider-flower.yaml b/k8s/deployments/spider-flower.yaml
@@ -30,7 +30,7 @@ spec:
         resources:
           requests:
              cpu: 100m
-             memory: 500Mi
+             memory: 100Mi
           limits:
             cpu: 300m
             memory: 1Gi

diff --git a/k8s/deployments/spider-redis.yaml b/k8s/deployments/spider-redis.yaml
@@ -26,11 +26,11 @@ spec:
         - containerPort: 6379
         resources: 
           requests:
-            cpu: 200m
-            memory: 4Gi
+            cpu: 150m
+            memory: 2Gi
           limits:
             cpu: 500m
-            memory: 8Gi
+            memory: 4Gi
         volumeMounts:
         - mountPath: /data
           name: redis-claim0

diff --git a/k8s/deployments/spider-worker.yaml b/k8s/deployments/spider-worker.yaml
@@ -30,7 +30,7 @@ spec:
         - "celery,es_post,convert"
         resources:
           requests:
-             cpu: 600m
+             cpu: 300m
              memory: 500Mi
           limits:
             cpu: 1000m

diff --git a/src/htcondor_es/celery/tasks.py b/src/htcondor_es/celery/tasks.py
@@ -224,7 +224,7 @@ def grouper(iterable, n, fillvalue=None):
     """
     # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx"
     args = [iter(iterable)] * n
-    return filter(None, zip_longest(*args, fillvalue=fillvalue))
+    return zip_longest(*args, fillvalue=fillvalue)
 
 
 def consume(iterator, n=None):
@@ -266,8 +266,8 @@ def send_data(
     responses = []
     total_tasks = 0
     for docs_bunch in grouper(query_iter, bunch):
-        process_and_send = group(
-            process_docs.si(
+        process_and_send = [
+            process_docs(
                 list(filter(None, X)),
                 reduce_data=not keep_full_queue_data,
                 pool_name=pool_name,
@@ -276,9 +276,9 @@ def send_data(
                 metadata=metadata,
             )
             for X in grouper(docs_bunch, chunk_size)
-        )
-        total_tasks += len(process_and_send.tasks)
-        responses.append(process_and_send.apply_async(serializer="pickle"))
+        ]
+        total_tasks += len(filter(None,docs_bunch))
+        #responses.append(process_and_send.apply_async(serializer="pickle"))
     return total_tasks