Merge pull request #274 from CybercentreCanada/kubernetes-watches

Set http timeout on kubernetes api as well
CybercentreCanada · Aug 9, 2021 · bbe3c62 · bbe3c62
2 parents b240ce7 + bce127c
commit bbe3c62
Showing 1 changed file with 7 additions and 4 deletions.
diff --git a/assemblyline_core/scaler/controllers/kubernetes_ctl.py b/assemblyline_core/scaler/controllers/kubernetes_ctl.py
@@ -29,6 +29,7 @@
 
 API_TIMEOUT = 90
 WATCH_TIMEOUT = 10 * 60
+WATCH_API_TIMEOUT = WATCH_TIMEOUT + 10
 
 _exponents = {
     'Ki': 2**10,
@@ -242,7 +243,8 @@ def _monitor_node_pool(self):
         self._node_pool_max_ram = 0
         watch = TypelessWatch()
 
-        for event in watch.stream(func=self.api.list_node, timeout_seconds=WATCH_TIMEOUT):
+        for event in watch.stream(func=self.api.list_node, timeout_seconds=WATCH_TIMEOUT,
+                                  _request_timeout=WATCH_API_TIMEOUT):
             if not self.running:
                 break
 
@@ -259,7 +261,8 @@ def _monitor_pods(self):
         self._pod_used_cpu = 0
         self._pod_used_ram = 0
 
-        for event in watch.stream(func=self.api.list_pod_for_all_namespaces, timeout_seconds=WATCH_TIMEOUT):
+        for event in watch.stream(func=self.api.list_pod_for_all_namespaces, timeout_seconds=WATCH_TIMEOUT,
+                                  _request_timeout=WATCH_API_TIMEOUT):
             if not self.running:
                 break
 
@@ -296,7 +299,7 @@ def _monitor_quotas(self):
         self._quota_mem_used = None
 
         for event in watch.stream(func=self.api.list_namespaced_resource_quota, namespace=self.namespace,
-                                  timeout_seconds=WATCH_TIMEOUT):
+                                  timeout_seconds=WATCH_TIMEOUT, _request_timeout=WATCH_API_TIMEOUT):
             if not self.running:
                 break
 
@@ -355,7 +358,7 @@ def _monitor_deployments(self):
 
         for event in watch.stream(func=self.apps_api.list_namespaced_deployment,
                                   namespace=self.namespace, label_selector=label_selector,
-                                  timeout_seconds=WATCH_TIMEOUT):
+                                  timeout_seconds=WATCH_TIMEOUT, _request_timeout=WATCH_API_TIMEOUT):
             if event['type'] in ['ADDED', 'MODIFIED']:
                 name = event['raw_object']['metadata']['labels'].get('component', None)
                 if name is not None: