Merge pull request #697 from skalenetwork/no-major-snapshot-option

No major snapshot option

node_cli/cli/node.py

@@ -18,6 +18,7 @@
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import ipaddress
+from typing import Optional
 from urllib.parse import urlparse
 
 import click
@@ -131,19 +132,33 @@ def register_node(name, ip, port, domain):
 
 @node.command('init', help="Initialize SKALE node")
 @click.argument('env_file')
+@click.option(
+    '--snapshot-from',
+    type=IP_TYPE,
+    default=None,
+    hidden=True,
+    help='Ip of the node from to download snapshot from'
+)
 @streamed_cmd
-def init_node(env_file):
-    init(env_file)
+def init_node(env_file, snapshot_from: Optional[str] = None):
+    init(env_file, snapshot_from)
 
 
 @node.command('update', help='Update node from .env file')
 @click.option('--yes', is_flag=True, callback=abort_if_false,
               expose_value=False,
               prompt='Are you sure you want to update SKALE node software?')
 @click.argument('env_file')
+@click.option(
+    '--snapshot-from',
+    type=IP_TYPE,
+    default=None,
+    hidden=True,
+    help='Ip of the node from to download snapshot from'
+)
 @streamed_cmd
-def update_node(env_file):
-    update(env_file)
+def update_node(env_file, snapshot_from: Optional[str] = None):
+    update(env_file, snapshot_from)
 
 
 @node.command('signature', help='Get node signature for given validator id')

node_cli/configs/__init__.py

@@ -148,3 +148,6 @@ def _get_env():
 DOCKER_SOCKET_PATH = '/var/run/skale/docker.sock'
 
 CHECK_REPORT_PATH = os.path.join(REPORTS_PATH, 'checks.json')
+
+AUTOLOAD_KERNEL_MODULES_PATH = '/etc/modules'
+BTRFS_KERNEL_MODULE = 'btrfs'

node_cli/configs/node_options.py

@@ -0,0 +1,23 @@
+#   -*- coding: utf-8 -*-
+#
+#   This file is part of node-cli
+#
+#   Copyright (C) 2022 SKALE Labs
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import os
+from node_cli.configs import NODE_DATA_PATH
+
+NODE_OPTIONS_FILEPATH = os.path.join(NODE_DATA_PATH, 'node_options.json')

node_cli/core/host.py

@@ -26,7 +26,8 @@
 from node_cli.core.resources import update_resource_allocation
 
 from node_cli.configs import (
-    ADMIN_PORT, DEFAULT_URL_SCHEME, NODE_DATA_PATH,
+    ADMIN_PORT, AUTOLOAD_KERNEL_MODULES_PATH,
+    BTRFS_KERNEL_MODULE, DEFAULT_URL_SCHEME, NODE_DATA_PATH,
     SKALE_DIR, CONTAINER_CONFIG_PATH, CONTRACTS_PATH,
     ETH_STATE_PATH, NODE_CERTS_PATH, SGX_CERTS_PATH,
     REPORTS_PATH, REDIS_DATA_PATH,
@@ -124,6 +125,38 @@ def init_data_dir():
     safe_mkdir(NODE_DATA_PATH)
 
 
+def is_btrfs_module_autoloaded(modules_filepath=AUTOLOAD_KERNEL_MODULES_PATH):
+    if not os.path.isfile(modules_filepath):
+        return False
+    with open(modules_filepath) as modules_file:
+        modules = set(
+            map(
+                lambda line: line.strip(),
+                filter(
+                    lambda line: not line.startswith('#'),
+                    modules_file.readlines()
+                )
+            )
+        )
+        return BTRFS_KERNEL_MODULE in modules
+
+
+def add_btrfs_module_to_autoload(modules_filepath=AUTOLOAD_KERNEL_MODULES_PATH):
+    with open(modules_filepath, 'a') as modules_file:
+        modules_file.write(f'{BTRFS_KERNEL_MODULE}\n')
+
+
+def ensure_btrfs_kernel_module_autoloaded(
+    modules_filepath=AUTOLOAD_KERNEL_MODULES_PATH
+):
+    logger.debug('Checking if btrfs is in %s', modules_filepath)
+    if not is_btrfs_module_autoloaded(modules_filepath):
+        logger.info('Adding btrfs module to %s', modules_filepath)
+        add_btrfs_module_to_autoload(modules_filepath)
+    else:
+        logger.debug('btrfs is already in %s', modules_filepath)
+
+
 def validate_abi_files(json_result=False):
     results = [
         validate_abi(abi_filepath)

node_cli/core/node.py

@@ -117,12 +117,12 @@ def register_node(name, p2p_ip,
 
 
 @check_not_inited
-def init(env_filepath):
+def init(env_filepath, snapshot_from: Optional[str] = None):
     env = get_node_env(env_filepath)
     if env is None:
         return
     configure_firewall_rules()
-    inited_ok = init_op(env_filepath, env)
+    inited_ok = init_op(env_filepath, env, snapshot_from=snapshot_from)
     if not inited_ok:
         error_exit(
             'Init operation failed',
@@ -187,11 +187,11 @@ def get_node_env(env_filepath, inited_node=False, sync_schains=None):
 
 @check_inited
 @check_user
-def update(env_filepath):
+def update(env_filepath, snapshot_from: Optional[str] = None):
     logger.info('Node update started')
     configure_firewall_rules()
     env = get_node_env(env_filepath, inited_node=True, sync_schains=False)
-    update_ok = update_op(env_filepath, env)
+    update_ok = update_op(env_filepath, env, snapshot_from=snapshot_from)
     if update_ok:
         logger.info('Waiting for containers initialization')
         time.sleep(TM_INIT_TIMEOUT)

node_cli/core/node_options.py

@@ -0,0 +1,55 @@
+#   -*- coding: utf-8 -*-
+#
+#   This file is part of node-cli
+#
+#   Copyright (C) 2022 SKALE Labs
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import logging
+from typing import Optional
+
+from node_cli.utils.helper import read_json, write_json, init_file
+from node_cli.configs.node_options import NODE_OPTIONS_FILEPATH
+
+logger = logging.getLogger(__name__)
+
+
+class NodeOptions:
+    def __init__(
+        self,
+        filepath: str = NODE_OPTIONS_FILEPATH
+    ):
+        self.filepath = filepath
+        init_file(filepath, {})
+
+    def _get(self, field_name: str):
+        config = read_json(self.filepath)
+        return config.get(field_name)
+
+    def _set(self, field_name: str, field_value) -> None:
+        config = read_json(self.filepath)
+        config[field_name] = field_value
+        write_json(self.filepath, config)
+
+    @property
+    def snapshot_from(self) -> Optional[str]:
+        return self._get('snapshot_from')
+
+    @snapshot_from.setter
+    def snapshot_from(self, ip: Optional[str]) -> None:
+        return self._set('snapshot_from', ip)
+
+    def all(self) -> dict:
+        return read_json(self.filepath)

node_cli/operations/base.py

@@ -19,14 +19,15 @@
 
 import functools
 import logging
-from typing import Dict
+from typing import Dict, Optional
 
 from node_cli.cli.info import VERSION
 from node_cli.configs import CONTAINER_CONFIG_PATH, CONTAINER_CONFIG_TMP_PATH
-from node_cli.core.host import link_env_file, prepare_host
+from node_cli.core.host import ensure_btrfs_kernel_module_autoloaded, link_env_file, prepare_host
 
 from node_cli.core.docker_config import configure_docker
 from node_cli.core.nginx import generate_nginx_config
+from node_cli.core.node_options import NodeOptions
 from node_cli.core.resources import update_resource_allocation, init_shared_space_volume
 
 from node_cli.operations.common import (
@@ -88,12 +89,14 @@ def wrapper(env_filepath: str, env: Dict, *args, **kwargs):
 
 
 @checked_host
-def update(env_filepath: str, env: Dict) -> None:
+def update(env_filepath: str, env: Dict, snapshot_from: Optional[str] = None) -> None:
     compose_rm(env)
     remove_dynamic_containers()
 
     sync_skale_node()
 
+    ensure_btrfs_kernel_module_autoloaded()
+
     if env.get('SKIP_DOCKER_CONFIG') != 'True':
         configure_docker()
 
@@ -103,6 +106,9 @@ def update(env_filepath: str, env: Dict) -> None:
     docker_lvmpy_update(env)
     generate_nginx_config()
 
+    node_options = NodeOptions()
+    node_options.snapshot_from = snapshot_from
+
     prepare_host(
         env_filepath,
         env['DISK_MOUNTPOINT'],
@@ -133,9 +139,10 @@ def update(env_filepath: str, env: Dict) -> None:
 
 
 @checked_host
-def init(env_filepath: str, env: str) -> bool:
+def init(env_filepath: str, env: Dict, snapshot_from: Optional[str] = None) -> bool:
     sync_skale_node()
 
+    ensure_btrfs_kernel_module_autoloaded()
     if env.get('SKIP_DOCKER_CONFIG') != 'True':
         configure_docker()
 
@@ -156,6 +163,9 @@ def init(env_filepath: str, env: str) -> bool:
     docker_lvmpy_install(env)
     init_shared_space_volume(env['ENV_TYPE'])
 
+    node_options = NodeOptions()
+    node_options.snapshot_from = snapshot_from
+
     update_meta(
         VERSION,
         env['CONTAINER_CONFIGS_STREAM'],
@@ -194,6 +204,7 @@ def restore(env, backup_path):
         print_failed_requirements_checks(failed_checks)
         return False
 
+    ensure_btrfs_kernel_module_autoloaded()
     if env.get('SKIP_DOCKER_CONFIG') != 'True':
         configure_docker()
 

node_cli/utils/helper.py

@@ -81,6 +81,11 @@ def write_json(path, content):
         json.dump(content, outfile, indent=4)
 
 
+def init_file(path, content=None):
+    if not os.path.exists(path):
+        write_json(path, content)
+
+
 def run_cmd(
     cmd,
     env={},

tests/core/host/kernel_config_test.py

@@ -0,0 +1,34 @@
+import os
+
+import pytest
+
+from node_cli.core.host import (
+    is_btrfs_module_autoloaded,
+    ensure_btrfs_kernel_module_autoloaded
+)
+
+
+@pytest.fixture
+def tmp_path(tmp_dir_path):
+    path = os.path.join(tmp_dir_path, 'modules')
+    return path
+
+
+def test_btrfs_module_autoload_config(tmp_path):
+    ensure_btrfs_kernel_module_autoloaded(tmp_path)
+    assert is_btrfs_module_autoloaded(tmp_path)
+    with open(tmp_path) as tmp_file:
+        tmp_file.read() == 'btrfs\n'
+
+
+def test_is_btrfs_module_autoloaded_negative(tmp_path):
+    assert not is_btrfs_module_autoloaded(tmp_path)
+    with open(tmp_path, 'w') as tmp_file:
+        tmp_file.write('')
+
+    assert not is_btrfs_module_autoloaded(tmp_path)
+
+    with open(tmp_path, 'w') as tmp_file:
+        tmp_file.write('# btrfs')
+
+    assert not is_btrfs_module_autoloaded(tmp_path)