diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 85721274..2dabef84 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -18,6 +18,7 @@ # along with this program. If not, see . import ipaddress +from typing import Optional from urllib.parse import urlparse import click @@ -131,9 +132,16 @@ def register_node(name, ip, port, domain): @node.command('init', help="Initialize SKALE node") @click.argument('env_file') +@click.option( + '--snapshot-from', + type=IP_TYPE, + default=None, + hidden=True, + help='Ip of the node from to download snapshot from' +) @streamed_cmd -def init_node(env_file): - init(env_file) +def init_node(env_file, snapshot_from: Optional[str] = None): + init(env_file, snapshot_from) @node.command('update', help='Update node from .env file') @@ -141,9 +149,16 @@ def init_node(env_file): expose_value=False, prompt='Are you sure you want to update SKALE node software?') @click.argument('env_file') +@click.option( + '--snapshot-from', + type=IP_TYPE, + default=None, + hidden=True, + help='Ip of the node from to download snapshot from' +) @streamed_cmd -def update_node(env_file): - update(env_file) +def update_node(env_file, snapshot_from: Optional[str] = None): + update(env_file, snapshot_from) @node.command('signature', help='Get node signature for given validator id') diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 44b9485b..cb7a1b36 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -148,3 +148,6 @@ def _get_env(): DOCKER_SOCKET_PATH = '/var/run/skale/docker.sock' CHECK_REPORT_PATH = os.path.join(REPORTS_PATH, 'checks.json') + +AUTOLOAD_KERNEL_MODULES_PATH = '/etc/modules' +BTRFS_KERNEL_MODULE = 'btrfs' diff --git a/node_cli/configs/node_options.py b/node_cli/configs/node_options.py new file mode 100644 index 00000000..6565b3d7 --- /dev/null +++ b/node_cli/configs/node_options.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2022 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +from node_cli.configs import NODE_DATA_PATH + +NODE_OPTIONS_FILEPATH = os.path.join(NODE_DATA_PATH, 'node_options.json') diff --git a/node_cli/core/host.py b/node_cli/core/host.py index f6736c4f..2dbdcc14 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -26,7 +26,8 @@ from node_cli.core.resources import update_resource_allocation from node_cli.configs import ( - ADMIN_PORT, DEFAULT_URL_SCHEME, NODE_DATA_PATH, + ADMIN_PORT, AUTOLOAD_KERNEL_MODULES_PATH, + BTRFS_KERNEL_MODULE, DEFAULT_URL_SCHEME, NODE_DATA_PATH, SKALE_DIR, CONTAINER_CONFIG_PATH, CONTRACTS_PATH, ETH_STATE_PATH, NODE_CERTS_PATH, SGX_CERTS_PATH, REPORTS_PATH, REDIS_DATA_PATH, @@ -124,6 +125,38 @@ def init_data_dir(): safe_mkdir(NODE_DATA_PATH) +def is_btrfs_module_autoloaded(modules_filepath=AUTOLOAD_KERNEL_MODULES_PATH): + if not os.path.isfile(modules_filepath): + return False + with open(modules_filepath) as modules_file: + modules = set( + map( + lambda line: line.strip(), + filter( + lambda line: not line.startswith('#'), + modules_file.readlines() + ) + ) + ) + return BTRFS_KERNEL_MODULE in modules + + +def add_btrfs_module_to_autoload(modules_filepath=AUTOLOAD_KERNEL_MODULES_PATH): + with open(modules_filepath, 'a') as modules_file: + modules_file.write(f'{BTRFS_KERNEL_MODULE}\n') + + +def ensure_btrfs_kernel_module_autoloaded( + modules_filepath=AUTOLOAD_KERNEL_MODULES_PATH +): + logger.debug('Checking if btrfs is in %s', modules_filepath) + if not is_btrfs_module_autoloaded(modules_filepath): + logger.info('Adding btrfs module to %s', modules_filepath) + add_btrfs_module_to_autoload(modules_filepath) + else: + logger.debug('btrfs is already in %s', modules_filepath) + + def validate_abi_files(json_result=False): results = [ validate_abi(abi_filepath) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 3a4601a7..bd731238 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -117,12 +117,12 @@ def register_node(name, p2p_ip, @check_not_inited -def init(env_filepath): +def init(env_filepath, snapshot_from: Optional[str] = None): env = get_node_env(env_filepath) if env is None: return configure_firewall_rules() - inited_ok = init_op(env_filepath, env) + inited_ok = init_op(env_filepath, env, snapshot_from=snapshot_from) if not inited_ok: error_exit( 'Init operation failed', @@ -187,11 +187,11 @@ def get_node_env(env_filepath, inited_node=False, sync_schains=None): @check_inited @check_user -def update(env_filepath): +def update(env_filepath, snapshot_from: Optional[str] = None): logger.info('Node update started') configure_firewall_rules() env = get_node_env(env_filepath, inited_node=True, sync_schains=False) - update_ok = update_op(env_filepath, env) + update_ok = update_op(env_filepath, env, snapshot_from=snapshot_from) if update_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) diff --git a/node_cli/core/node_options.py b/node_cli/core/node_options.py new file mode 100644 index 00000000..91cbdcd2 --- /dev/null +++ b/node_cli/core/node_options.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2022 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +from typing import Optional + +from node_cli.utils.helper import read_json, write_json, init_file +from node_cli.configs.node_options import NODE_OPTIONS_FILEPATH + +logger = logging.getLogger(__name__) + + +class NodeOptions: + def __init__( + self, + filepath: str = NODE_OPTIONS_FILEPATH + ): + self.filepath = filepath + init_file(filepath, {}) + + def _get(self, field_name: str): + config = read_json(self.filepath) + return config.get(field_name) + + def _set(self, field_name: str, field_value) -> None: + config = read_json(self.filepath) + config[field_name] = field_value + write_json(self.filepath, config) + + @property + def snapshot_from(self) -> Optional[str]: + return self._get('snapshot_from') + + @snapshot_from.setter + def snapshot_from(self, ip: Optional[str]) -> None: + return self._set('snapshot_from', ip) + + def all(self) -> dict: + return read_json(self.filepath) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 78d5e41c..3b0d923a 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -19,14 +19,15 @@ import functools import logging -from typing import Dict +from typing import Dict, Optional from node_cli.cli.info import VERSION from node_cli.configs import CONTAINER_CONFIG_PATH, CONTAINER_CONFIG_TMP_PATH -from node_cli.core.host import link_env_file, prepare_host +from node_cli.core.host import ensure_btrfs_kernel_module_autoloaded, link_env_file, prepare_host from node_cli.core.docker_config import configure_docker from node_cli.core.nginx import generate_nginx_config +from node_cli.core.node_options import NodeOptions from node_cli.core.resources import update_resource_allocation, init_shared_space_volume from node_cli.operations.common import ( @@ -88,12 +89,14 @@ def wrapper(env_filepath: str, env: Dict, *args, **kwargs): @checked_host -def update(env_filepath: str, env: Dict) -> None: +def update(env_filepath: str, env: Dict, snapshot_from: Optional[str] = None) -> None: compose_rm(env) remove_dynamic_containers() sync_skale_node() + ensure_btrfs_kernel_module_autoloaded() + if env.get('SKIP_DOCKER_CONFIG') != 'True': configure_docker() @@ -103,6 +106,9 @@ def update(env_filepath: str, env: Dict) -> None: docker_lvmpy_update(env) generate_nginx_config() + node_options = NodeOptions() + node_options.snapshot_from = snapshot_from + prepare_host( env_filepath, env['DISK_MOUNTPOINT'], @@ -133,9 +139,10 @@ def update(env_filepath: str, env: Dict) -> None: @checked_host -def init(env_filepath: str, env: str) -> bool: +def init(env_filepath: str, env: Dict, snapshot_from: Optional[str] = None) -> bool: sync_skale_node() + ensure_btrfs_kernel_module_autoloaded() if env.get('SKIP_DOCKER_CONFIG') != 'True': configure_docker() @@ -156,6 +163,9 @@ def init(env_filepath: str, env: str) -> bool: docker_lvmpy_install(env) init_shared_space_volume(env['ENV_TYPE']) + node_options = NodeOptions() + node_options.snapshot_from = snapshot_from + update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], @@ -194,6 +204,7 @@ def restore(env, backup_path): print_failed_requirements_checks(failed_checks) return False + ensure_btrfs_kernel_module_autoloaded() if env.get('SKIP_DOCKER_CONFIG') != 'True': configure_docker() diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index f8c3235b..8379e2c6 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -81,6 +81,11 @@ def write_json(path, content): json.dump(content, outfile, indent=4) +def init_file(path, content=None): + if not os.path.exists(path): + write_json(path, content) + + def run_cmd( cmd, env={}, diff --git a/tests/core/host/kernel_config_test.py b/tests/core/host/kernel_config_test.py new file mode 100644 index 00000000..92e4e8e1 --- /dev/null +++ b/tests/core/host/kernel_config_test.py @@ -0,0 +1,34 @@ +import os + +import pytest + +from node_cli.core.host import ( + is_btrfs_module_autoloaded, + ensure_btrfs_kernel_module_autoloaded +) + + +@pytest.fixture +def tmp_path(tmp_dir_path): + path = os.path.join(tmp_dir_path, 'modules') + return path + + +def test_btrfs_module_autoload_config(tmp_path): + ensure_btrfs_kernel_module_autoloaded(tmp_path) + assert is_btrfs_module_autoloaded(tmp_path) + with open(tmp_path) as tmp_file: + tmp_file.read() == 'btrfs\n' + + +def test_is_btrfs_module_autoloaded_negative(tmp_path): + assert not is_btrfs_module_autoloaded(tmp_path) + with open(tmp_path, 'w') as tmp_file: + tmp_file.write('') + + assert not is_btrfs_module_autoloaded(tmp_path) + + with open(tmp_path, 'w') as tmp_file: + tmp_file.write('# btrfs') + + assert not is_btrfs_module_autoloaded(tmp_path)