support stopping paused crawls

Author: ikreymer

backend/btrixcloud/models.py

@@ -218,11 +218,13 @@ class UserOrgInfoOut(BaseModel):
 
 # ============================================================================
 TYPE_RUNNING_STATES = Literal[
-    "running", "pending-wait", "generate-wacz", "uploading-wacz", "paused"
+    "running", "pending-wait", "generate-wacz", "uploading-wacz"
 ]
 RUNNING_STATES = get_args(TYPE_RUNNING_STATES)
 
-TYPE_WAITING_STATES = Literal["starting", "waiting_capacity", "waiting_org_limit"]
+TYPE_WAITING_STATES = Literal[
+    "starting", "waiting_capacity", "waiting_org_limit", "paused"
+]
 WAITING_STATES = get_args(TYPE_WAITING_STATES)
 
 TYPE_FAILED_STATES = Literal[

backend/btrixcloud/operator/crawls.py

@@ -262,13 +262,15 @@ async def sync_crawls(self, data: MCSyncData):
             )
 
         else:
-            if crawl.paused:
-                await self.set_state(
-                    "paused", status, crawl, allowed_from=RUNNING_AND_WAITING_STATES
-                )
-
             status.scale = 1
 
+        # stopping paused crawls
+        if crawl.paused and crawl.stopping:
+            status.stopReason = "stopped_by_user"
+            status.stopping = True
+            print(f"Paused crawl stopped by user, id: {crawl.id}")
+            await self.mark_finished(crawl, status, "stopped_by_user")
+
         children = self._load_redis(params, status, data.children)
 
         storage_path = crawl.storage.get_storage_extra_path(oid)
@@ -881,7 +883,7 @@ async def sync_crawl_state(
                         status.initRedis = False
 
                     # crawler pods already shut down, remove redis pause key to allow unpausing later
-                    if crawl.paused or not status.initRedis:
+                    if crawl.paused:
                         await redis.delete(f"{crawl.id}:paused")
 
                 elif crawler_running and not redis:
@@ -892,6 +894,10 @@ async def sync_crawl_state(
                 status.resync_after = self.fast_retry_secs
                 return status
 
+            # only get here if at least one crawler pod is running
+            if crawl.paused:
+                await redis.set(f"{crawl.id}:paused", "1")
+
             # update lastActiveTime if crawler is running
             if crawler_running:
                 status.lastActiveTime = date_to_str(dt_now())
@@ -1299,6 +1305,8 @@ async def add_file_to_crawl(self, cc_data, crawl: CrawlSpec, redis):
             crawl.db_crawl_id, crawl.is_qa, crawl_file, filecomplete.size
         )
 
+        print("FILE ADDED", filecomplete.size)
+
         # no replicas for QA for now
         if crawl.is_qa:
             return True
@@ -1454,9 +1462,6 @@ async def update_crawl_state(
             # backwards compatibility with older crawler
             await redis.set("crawl-stop", "1")
 
-        if crawl.paused:
-            await redis.set(f"{crawl.id}:paused", "1")
-
         # resolve scale
         if crawl.scale != status.scale:
             status.scale = await self._resolve_scale(