Consider default job timeout for VM image build babysits (#2421)

Consider default job timeout for VM image build babysits Reviewed-by: Maja Massarini
packit · May 7, 2024 · 9c7b13d · 9c7b13d
2 parents 0221340 + 2012157
commit 9c7b13d
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 2 deletions.
diff --git a/packit_service/worker/helpers/build/babysit.py b/packit_service/worker/helpers/build/babysit.py
@@ -575,6 +575,19 @@ def check_pending_vm_image_builds() -> None:
     pending_vm_image_builds = VMImageBuildTargetModel.get_all_by_status(
         VMImageBuildStatus.pending
     )
-
+    current_time = datetime.now(timezone.utc)
     for build in pending_vm_image_builds:
+        logger.debug(f"Checking status of VM image build {build.build_id}")
+        if build.build_submitted_time:
+            elapsed = elapsed_seconds(
+                begin=build.build_submitted_time, end=current_time
+            )
+            if elapsed > DEFAULT_JOB_TIMEOUT:
+                logger.info(
+                    f"VM image build {build.build_id} has been running for "
+                    f"{elapsed}s, probably an internal error occurred. "
+                    "Not checking it anymore."
+                )
+                build.set_status(VMImageBuildStatus.error)
+                continue
         update_vm_image_build(build.build_id, build)
diff --git a/tests/unit/test_babysit_vm_image.py b/tests/unit/test_babysit_vm_image.py
@@ -1,5 +1,7 @@
 # Copyright Contributors to the Packit project.
 # SPDX-License-Identifier: MIT
+import datetime
+
 import pytest
 from flexmock import Mock
 from flexmock import flexmock
@@ -26,13 +28,41 @@
 def test_check_pending_vm_image_builds():
     flexmock(VMImageBuildTargetModel).should_receive("get_all_by_status").with_args(
         VMImageBuildStatus.pending
-    ).and_return([flexmock(build_id=1)])
+    ).and_return(
+        [
+            flexmock(
+                build_id=1,
+                build_submitted_time=datetime.datetime.utcnow()
+                - datetime.timedelta(days=1),
+            )
+        ]
+    )
     flexmock(packit_service.worker.helpers.build.babysit).should_receive(
         "update_vm_image_build"
     ).with_args(1, Mock)
     check_pending_vm_image_builds()
 
 
+def test_check_pending_vm_image_builds_timeout():
+    flexmock(VMImageBuildTargetModel).should_receive("get_all_by_status").with_args(
+        VMImageBuildStatus.pending
+    ).and_return(
+        [
+            flexmock(
+                build_id=1,
+                build_submitted_time=datetime.datetime.utcnow()
+                - datetime.timedelta(weeks=2),
+            )
+            .should_receive("set_status")
+            .mock()
+        ]
+    )
+    flexmock(packit_service.worker.helpers.build.babysit).should_receive(
+        "update_vm_image_build"
+    ).never()
+    check_pending_vm_image_builds()
+
+
 def test_check_no_pending_vm_image_builds():
     flexmock(VMImageBuildTargetModel).should_receive("get_all_by_status").with_args(
         VMImageBuildStatus.pending