Merge pull request #985 from skalenetwork/fix-update-config-monitor

Fix update config monitor

core/schains/monitor/action.py

@@ -297,7 +297,8 @@ def firewall_rules(self) -> bool:
     def skaled_container(
         self,
         download_snapshot: bool = False,
-        start_ts: Optional[int] = None
+        start_ts: Optional[int] = None,
+        ignore_reached_exit: bool = True,
     ) -> bool:
         logger.info(
             'Starting skaled container watchman snapshot: %s, start_ts: %s',
@@ -310,6 +311,7 @@ def skaled_container(
             skaled_status=self.skaled_status,
             download_snapshot=download_snapshot,
             start_ts=start_ts,
+            ignore_reached_exit=ignore_reached_exit,
             dutils=self.dutils
         )
         time.sleep(CONTAINER_POST_RUN_DELAY)
@@ -346,7 +348,7 @@ def reset_restart_counter(self) -> bool:
         return True
 
     @BaseActionManager.monitor_block
-    def reloaded_skaled_container(self) -> bool:
+    def reloaded_skaled_container(self, ignore_reached_exit: bool = True) -> bool:
         logger.info('Starting skaled from scratch')
         initial_status = True
         if is_container_exists(self.name, dutils=self.dutils):
@@ -357,7 +359,8 @@ def reloaded_skaled_container(self) -> bool:
         self.schain_record.set_restart_count(0)
         self.schain_record.set_failed_rpc_count(0)
         self.schain_record.set_needs_reload(False)
-        initial_status = self.skaled_container()
+        initial_status = self.skaled_container(
+            ignore_reached_exit=ignore_reached_exit)
         return initial_status
 
     @BaseActionManager.monitor_block

core/schains/monitor/containers.py

@@ -52,6 +52,7 @@ def monitor_schain_container(
     skaled_status,
     download_snapshot=False,
     start_ts=None,
+    ignore_reached_exit: bool = True,
     dutils=None
 ) -> None:
     dutils = dutils or DockerUtils()
@@ -62,7 +63,7 @@ def monitor_schain_container(
         logger.error(f'Data volume for sChain {schain_name} does not exist')
         return
 
-    if skaled_status.exit_time_reached:
+    if skaled_status.exit_time_reached and ignore_reached_exit:
         logger.info(
             f'{schain_name} - Skipping container monitor: exit time reached')
         skaled_status.log()

core/schains/monitor/skaled_monitor.py

@@ -145,7 +145,7 @@ def execute(self) -> None:
             self.am.firewall_rules()
         if not self.checks.volume:
             self.am.volume()
-        self.am.reloaded_skaled_container()
+        self.am.reloaded_skaled_container(ignore_reached_exit=False)
         if not self.checks.ima_container:
             self.am.restart_ima_container()
 

tests/conftest.py

@@ -619,7 +619,7 @@ def skaled_mock_image(scope='module'):
 
 @pytest.fixture
 def cleanup_schain_dirs_before():
-    shutil.rmtree(SCHAINS_DIR_PATH)
+    shutil.rmtree(SCHAINS_DIR_PATH, ignore_errors=True)
     pathlib.Path(SCHAINS_DIR_PATH).mkdir(parents=True, exist_ok=True)
     return
 
@@ -743,7 +743,7 @@ def new_upstream(schain_db):
         Path(upath).touch()
         yield upath
     finally:
-        shutil.rmtree(config_dir)
+        shutil.rmtree(config_dir, ignore_errors=True)
 
 
 @pytest.fixture
@@ -781,4 +781,4 @@ def upstreams(schain_db, schain_config):
                 json.dump(schain_config, f)
         yield files
     finally:
-        shutil.rmtree(config_folder)
+        shutil.rmtree(config_folder, ignore_errors=True)

tests/routes/schains_test.py

@@ -58,7 +58,7 @@ def test_schain_config(skale_bp, skale, schain_config, schain_on_contracts):
                         'status': 'ok'}
     finally:
         os.remove(filepath)
-        shutil.rmtree(os.path.dirname(filepath))
+        shutil.rmtree(os.path.dirname(filepath), ignore_errors=True)
 
 
 def test_schains_list(skale_bp, skale):

tests/schains/monitor/action/skaled_action_test.py

@@ -39,16 +39,18 @@ def monitor_schain_container_mock(
     skaled_status,
     download_snapshot=False,
     start_ts=None,
+    ignore_reached_exit=True,
     dutils=None
 ):
     image_name, container_name, _, _ = get_container_info(
         SCHAIN_CONTAINER, schain['name'])
     dutils.safe_rm(container_name)
-    dutils.run_container(
-        image_name=image_name,
-        name=container_name,
-        entrypoint='bash -c "while true; do foo; sleep 2; done"'
-    )
+    if not skaled_status.exit_time_reached or not ignore_reached_exit:
+        dutils.run_container(
+            image_name=image_name,
+            name=container_name,
+            entrypoint='bash -c "while true; do foo; sleep 2; done"'
+        )
 
 
 @pytest.fixture
@@ -132,6 +134,7 @@ def test_skaled_container_with_snapshot_action(skaled_am):
             skaled_status=skaled_am.skaled_status,
             download_snapshot=True,
             start_ts=None,
+            ignore_reached_exit=True,
             dutils=skaled_am.dutils
         )
         assert monitor_schain_mock.call_count == 1
@@ -155,6 +158,7 @@ def test_skaled_container_snapshot_delay_start_action(skaled_am):
             skaled_status=skaled_am.skaled_status,
             download_snapshot=True,
             start_ts=ts,
+            ignore_reached_exit=True,
             dutils=skaled_am.dutils
         )
         assert monitor_schain_mock.call_count == 1
@@ -163,7 +167,6 @@ def test_skaled_container_snapshot_delay_start_action(skaled_am):
 
 
 def test_restart_skaled_container_action(skaled_am, skaled_checks):
-    skaled_am.reloaded_skaled_container()
     try:
         skaled_am.volume()
         with mock.patch(
@@ -177,6 +180,28 @@ def test_restart_skaled_container_action(skaled_am, skaled_checks):
             assert skaled_checks.skaled_container
             skaled_am.reloaded_skaled_container()
             assert skaled_checks.skaled_container
+            skaled_am.reloaded_skaled_container()
+            assert skaled_checks.skaled_container
+    finally:
+        skaled_am.cleanup_schain_docker_entity()
+
+
+def test_restart_skaled_container_action_exit_reached(
+    skaled_am,
+    skaled_checks,
+    skaled_status_exit_time_reached
+):
+    try:
+        skaled_am.volume()
+        with mock.patch(
+            'core.schains.monitor.action.monitor_schain_container',
+            monitor_schain_container_mock
+        ):
+            assert not skaled_checks.skaled_container
+            skaled_am.reloaded_skaled_container()
+            assert not skaled_checks.skaled_container
+            skaled_am.reloaded_skaled_container(ignore_reached_exit=False)
+            assert skaled_checks.skaled_container
     finally:
         skaled_am.cleanup_schain_docker_entity()
 

tests/schains/monitor/containers.py → tests/schains/monitor/containers_test.py

@@ -4,7 +4,7 @@
 from core.schains.runner import is_container_exists
 from web.models.schain import upsert_schain_record
 
-from tests.schains.monitor.main_test import run_exited_schain_container
+from tests.utils import run_custom_schain_container
 
 
 def test_monitor_schain_container(
@@ -37,7 +37,6 @@ def test_monitor_schain_container_exit_time_reached(
     schain_record = upsert_schain_record(schain_db)
     schain = {'name': schain_db, 'partOfNode': 0, 'generation': 0}
 
-    run_exited_schain_container(dutils, schain_db, 0)
     with mock.patch('core.schains.monitor.containers.is_volume_exists', return_value=True):
         schain_record.set_failed_rpc_count(100)
         schain_record.set_restart_count(100)
@@ -47,31 +46,18 @@ def test_monitor_schain_container_exit_time_reached(
             skaled_status_exit_time_reached,
             dutils=dutils
         )
+        assert len(dutils.get_all_schain_containers()) == 0
         assert schain_record.restart_count == 0
         assert schain_record.failed_rpc_count == 0
 
-
-def test_monitor_schain_container_cleanup(
-    schain_db,
-    skaled_status_repair,
-    dutils,
-    ssl_folder,
-    schain_config,
-    cleanup_schain_containers
-):
-    schain_record = upsert_schain_record(schain_db)
-    schain = {'name': schain_db, 'partOfNode': 0, 'generation': 0}
-
-    run_exited_schain_container(dutils, schain_db, 0)
-    with mock.patch('core.schains.monitor.containers.is_volume_exists', return_value=True):
-        schain_record.set_failed_rpc_count(100)
-        schain_record.set_restart_count(100)
         monitor_schain_container(
             schain,
             schain_record,
-            skaled_status_repair,
+            skaled_status_exit_time_reached,
+            ignore_reached_exit=False,
             dutils=dutils
         )
+        assert len(dutils.get_all_schain_containers()) == 1
         assert schain_record.restart_count == 0
         assert schain_record.failed_rpc_count == 0
 
@@ -86,8 +72,9 @@ def test_monitor_schain_container_ec(
 ):
     schain_record = upsert_schain_record(schain_db)
     schain = {'name': schain_db, 'partOfNode': 0, 'generation': 0}
+    schain_name = schain_db
 
-    run_exited_schain_container(dutils, schain_db, 123)
+    run_custom_schain_container(dutils, schain_name, entrypoint=['sh', 'exit', '1'])
     with mock.patch('core.schains.monitor.containers.is_volume_exists', return_value=True):
         schain_record.set_failed_rpc_count(100)
         schain_record.set_restart_count(0)
@@ -99,28 +86,3 @@ def test_monitor_schain_container_ec(
         )
         assert schain_record.restart_count == 1
         assert schain_record.failed_rpc_count == 0
-
-
-def test_monitor_schain_container_ec_0(
-    schain_db,
-    skaled_status,
-    dutils,
-    ssl_folder,
-    schain_config,
-    cleanup_schain_containers
-):
-    schain_record = upsert_schain_record(schain_db)
-    schain = {'name': schain_db, 'partOfNode': 0, 'generation': 0}
-
-    run_exited_schain_container(dutils, schain_db, 0)
-    with mock.patch('core.schains.monitor.containers.is_volume_exists', return_value=True):
-        schain_record.set_failed_rpc_count(100)
-        schain_record.set_restart_count(0)
-        monitor_schain_container(
-            schain,
-            schain_record,
-            skaled_status,
-            dutils=dutils
-        )
-        assert schain_record.restart_count == 0
-        assert schain_record.failed_rpc_count == 100

tests/schains/monitor/skaled_monitor_test.py

@@ -497,7 +497,14 @@ def test_recreate_skaled_monitor(skaled_am, skaled_checks, clean_docker, dutils)
     assert dutils.get_container_created_ts(schain_container.id) > ts_before
 
 
-def test_update_config_skaled_monitor(skaled_am, skaled_checks, dutils, clean_docker, upstreams):
+def test_update_config_skaled_monitor(
+    skaled_am,
+    skaled_checks,
+    dutils,
+    clean_docker,
+    upstreams,
+    skaled_status_exit_time_reached
+):
     name = skaled_checks.name
     ts_before = time.time()
     time.sleep(1)

tests/test_nginx.py

@@ -67,7 +67,7 @@ def tmp_dir():
     try:
         yield path
     finally:
-        shutil.rmtree(path)
+        shutil.rmtree(path, ignore_errors=True)
 
 
 @pytest.fixture
@@ -91,7 +91,7 @@ def ssl_dir():
     try:
         yield path
     finally:
-        shutil.rmtree(path)
+        shutil.rmtree(path, ignore_errors=True)
 
 
 @pytest.fixture