Merge branch 'beta-sync-node' into no-volume-limits

Author: badrogger

.github/workflows/publish.yml

@@ -8,6 +8,7 @@ on:
       - beta
       - stable
       - sync-node
+      - beta-sync-node
 
 jobs:
   create_release:

node_cli/cli/__init__.py

@@ -1,4 +1,5 @@
 __version__ = '2.2.0'
 
+
 if __name__ == "__main__":
     print(__version__)

node_cli/configs/env.py

@@ -44,7 +44,8 @@
     'DEFAULT_GAS_PRICE_WEI': '',
     'DISABLE_IMA': '',
     'SKIP_DOCKER_CONFIG': '',
-    'ENFORCE_BTRFS': ''
+    'ENFORCE_BTRFS': '',
+    'SKIP_DOCKER_CLEANUP': ''
 }
 
 

node_cli/core/node.py

@@ -127,6 +127,7 @@ def init(env_filepath):
     env = get_node_env(env_filepath)
     if env is None:
         return
+    configure_firewall_rules()
     inited_ok = init_op(env_filepath, env)
     if not inited_ok:
         error_exit(

node_cli/operations/base.py

@@ -43,9 +43,13 @@
 from node_cli.core.checks import CheckType, run_checks as run_host_checks
 from node_cli.core.iptables import configure_iptables
 from node_cli.utils.docker_utils import (
-    compose_rm, compose_up, remove_dynamic_containers, compose_up_sync
+    compose_rm,
+    compose_up,
+    compose_up_sync,
+    docker_cleanup,
+    remove_dynamic_containers
 )
-from node_cli.utils.meta import update_meta
+from node_cli.utils.meta import get_meta_info, update_meta
 from node_cli.utils.print_formatters import print_failed_requirements_checks
 
 
@@ -110,6 +114,16 @@ def update(env_filepath: str, env: Dict) -> None:
     )
     init_shared_space_volume(env['ENV_TYPE'])
 
+    current_stream = get_meta_info().config_stream
+    skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True'
+    if not skip_cleanup and current_stream != env['CONTAINER_CONFIGS_STREAM']:
+        logger.info(
+            'Stream version was changed from %s to %s',
+            current_stream,
+            env['CONTAINER_CONFIGS_STREAM']
+        )
+        docker_cleanup()
+
     update_meta(
         VERSION,
         env['CONTAINER_CONFIGS_STREAM'],

node_cli/utils/docker_utils.py

@@ -256,3 +256,42 @@ def restart_nginx_container(dutils=None):
     dutils = dutils or docker_client()
     nginx_container = dutils.containers.get(NGINX_CONTAINER_NAME)
     nginx_container.restart()
+
+
+def remove_images(images, dclient=None):
+    dc = dclient or docker_client()
+    for image in images:
+        dc.images.remove(image.id)
+
+
+def get_used_images(dclient=None):
+    dc = dclient or docker_client()
+    return [c.image for c in dc.containers.list()]
+
+
+def cleanup_unused_images(dclient=None, ignore=None):
+    logger.info('Removing unused docker images')
+    ignore = ignore or []
+    dc = dclient or docker_client()
+    used = get_used_images(dclient=dc)
+    remove_images(
+        filter(lambda i: i not in used and i not in ignore, dc.images.list()),
+        dclient=dc
+    )
+
+
+def system_prune():
+    logger.info('Removing dangling docker artifacts')
+    cmd = ['docker', 'system', 'prune', '-f']
+    run_cmd(cmd=cmd)
+
+
+def docker_cleanup(dclient=None, ignore=None):
+    ignore = ignore or []
+    try:
+        dc = dclient or docker_client()
+        cleanup_unused_images(dclient=dc, ignore=ignore)
+        system_prune()
+    except Exception as e:
+        logger.warning('Image cleanuping errored with %s', e)
+        logger.debug('Image cleanuping errored', exc_info=True)

tests/conftest.py

@@ -148,25 +148,33 @@ def removed_containers_folder():
 
 
 @pytest.fixture()
-def simple_image():
-    client = docker.from_env()
+def dclient():
+    return docker.from_env()
+
+
+@pytest.fixture()
+def simple_image(dclient):
     name = 'simple-image'
     try:
-        client.images.build(
+        dclient.images.build(
             tag=name,
             rm=True,
             nocache=True,
             path='tests/simple_container'
         )
         yield name
     finally:
-        client.images.remove(name, force=True)
+        try:
+            dclient.images.get(name)
+        except docker.errors.ImageNotFound:
+            return
+        dclient.images.remove(name, force=True)
 
 
 @pytest.fixture()
-def docker_hc():
-    client = docker.from_env()
-    return client.api.create_host_config(
+def docker_hc(dclient):
+    dclient = docker.from_env()
+    return dclient.api.create_host_config(
         log_config=docker.types.LogConfig(
             type=docker.types.LogConfig.types.JSON
         )

tests/core/host/docker_config_test.py

@@ -5,7 +5,6 @@
 from contextlib import contextmanager
 from timeit import default_timer as timer
 
-import docker
 import pytest
 
 from node_cli.core.docker_config import (
@@ -127,11 +126,12 @@ def test_ensure_run_dir(tmp_dir):
 
 
 @pytest.fixture
-def container():
-    client = docker.from_env()
-    c = client.containers.run('hello-world', detach=True)
-    yield c
-    c.remove(force=True)
+def container(dclient):
+    c = dclient.containers.run('hello-world', detach=True)
+    try:
+        yield c
+    finally:
+        c.remove(force=True)
 
 
 def test_assert_no_contaners():

tests/core_node_test.py

@@ -158,6 +158,7 @@ def test_init_node(no_resource_file):  # todo: write new init node test
                        return_value=BIG_DISK_SIZE), \
             mock.patch('node_cli.core.host.prepare_host'), \
             mock.patch('node_cli.core.host.init_data_dir'), \
+            mock.patch('node_cli.core.node.configure_firewall_rules'), \
             mock.patch('node_cli.core.node.init_op'), \
             mock.patch('node_cli.core.node.is_base_containers_alive',
                        return_value=True), \

tests/docker_utils_test.py

@@ -2,28 +2,29 @@
 import time
 from time import sleep
 
-import docker
+import mock
 import pytest
 
-from node_cli.utils.docker_utils import save_container_logs, safe_rm
+from node_cli.utils.docker_utils import (
+    docker_cleanup,
+    save_container_logs,
+    safe_rm
+)
 from node_cli.configs import REMOVED_CONTAINERS_FOLDER_PATH
 
 
-client = docker.from_env()
-
-
 @pytest.fixture
-def simple_container(simple_image, docker_hc):
+def simple_container(dclient, simple_image, docker_hc):
     name = 'simple-container'
     c = None
     try:
-        info = client.api.create_container(
+        info = dclient.api.create_container(
             simple_image,
             detach=True,
             name=name,
             host_config=docker_hc
         )
-        c = client.containers.get(info['Id'])
+        c = dclient.containers.get(info['Id'])
         c.restart()
         yield c
     finally:
@@ -83,3 +84,21 @@ def test_safe_rm(simple_container, removed_containers_folder):
     with open(log_path) as log_file:
         log_lines = log_file.readlines()
     assert log_lines[-1] == 'signal_handler completed, exiting...\n'
+
+
+def test_docker_cleanup(dclient, simple_container):
+    c = simple_container
+    image = c.image
+    docker_cleanup(dclient=dclient)
+    assert image in dclient.images.list()
+
+    c.stop()
+    docker_cleanup(dclient=dclient)
+    assert image in dclient.images.list()
+
+    c.remove()
+    docker_cleanup(dclient=dclient)
+    assert image not in dclient.images.list()
+
+    with mock.patch('node_cli.utils.docker_utils.run_cmd', side_effect=ValueError):
+        docker_cleanup(dclient=dclient)