From 6908f68cc7026b53910a9e458fde016468ab31eb Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 13 Jun 2024 15:43:49 +0000 Subject: [PATCH 01/30] Toggle repair through node_cli schain status file --- node_cli/core/schains.py | 31 +++++---- node_cli/utils/helper.py | 17 ++++- tests/conftest.py | 24 +++++-- tests/{ => core}/core_checks_test.py | 0 tests/{ => core}/core_node_test.py | 0 tests/core/core_schains_test.py | 34 ++++++++++ tests/core_ssl_test.py | 95 ---------------------------- 7 files changed, 83 insertions(+), 118 deletions(-) rename tests/{ => core}/core_checks_test.py (100%) rename tests/{ => core}/core_node_test.py (100%) create mode 100644 tests/core/core_schains_test.py delete mode 100644 tests/core_ssl_test.py diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index f9fa64aa..652210d1 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -2,6 +2,7 @@ import os import pprint import shutil +import time from pathlib import Path from typing import Dict, Optional @@ -16,8 +17,7 @@ from node_cli.utils.helper import ( get_request, error_exit, - safe_load_yml, - post_request + safe_load_yml ) from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.print_formatters import ( @@ -27,7 +27,7 @@ print_schains ) from node_cli.utils.docker_utils import ensure_volume, is_volume_exists -from node_cli.utils.helper import read_json, run_cmd +from node_cli.utils.helper import read_json, run_cmd, save_json from lvmpy.src.core import mount, volume_mountpoint @@ -89,22 +89,25 @@ def show_config(name: str) -> None: error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) +def get_node_cli_schain_status_filepath(schain_name: str) -> str: + return os.path.join(SCHAIN_NODE_DATA_PATH, schain_name, 'node-cli.status') + + +def update_node_cli_schain_status(schain_name: str, status: dict) -> None: + path = get_node_cli_schain_status_filepath(schain_name) + save_json(path, status) + + def toggle_schain_repair_mode( schain: str, snapshot_from: Optional[str] = None ) -> None: - json_params = {'schain_name': schain} + ts = int(time.time()) + status = {'schain_name': schain, 'repair_ts': ts} if snapshot_from: - json_params.update({'snapshot_from': snapshot_from}) - status, payload = post_request( - blueprint=BLUEPRINT_NAME, - method='repair', - json=json_params - ) - if status == 'ok': - print('Schain has been set for repair') - else: - error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) + status.update({'snapshot_from': snapshot_from}) + update_node_cli_schain_status(schain, status) + print('Schain has been set for repair') def describe(schain: str, raw=False) -> None: diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 4d76164b..cba65ddd 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -22,6 +22,7 @@ import os import re import sys +import uuid from urllib.parse import urlparse import yaml @@ -77,16 +78,22 @@ class InvalidEnvFileError(Exception): pass -def read_json(path): +def read_json(path: str) -> dict: with open(path, encoding='utf-8') as data_file: return json.loads(data_file.read()) -def write_json(path, content): +def write_json(path: str, content: dict) -> None: with open(path, 'w') as outfile: json.dump(content, outfile, indent=4) +def save_json(path: str, content: dict) -> None: + tmp_path = get_tmp_path(path) + write_json(tmp_path, content) + shutil.move(tmp_path, path) + + def init_file(path, content=None): if not os.path.exists(path): write_json(path, content) @@ -400,3 +407,9 @@ def convert(self, value, param, ctx): URL_TYPE = UrlType() IP_TYPE = IpType() + + +def get_tmp_path(path: str) -> str: + base, ext = os.path.splitext(path) + salt = uuid.uuid4().hex[:5] + return base + salt + '.tmp' + ext diff --git a/tests/conftest.py b/tests/conftest.py index 9504523f..93d82521 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,13 +29,14 @@ import yaml from node_cli.configs import ( - CONTAINER_CONFIG_TMP_PATH, - GLOBAL_SKALE_CONF_FILEPATH, - GLOBAL_SKALE_DIR, - META_FILEPATH, - NGINX_CONTAINER_NAME, - REMOVED_CONTAINERS_FOLDER_PATH, - STATIC_PARAMS_FILEPATH + CONTAINER_CONFIG_TMP_PATH, + GLOBAL_SKALE_CONF_FILEPATH, + GLOBAL_SKALE_DIR, + META_FILEPATH, + NGINX_CONTAINER_NAME, + REMOVED_CONTAINERS_FOLDER_PATH, + STATIC_PARAMS_FILEPATH, + SCHAIN_NODE_DATA_PATH ) from node_cli.configs.node_options import NODE_OPTIONS_FILEPATH from node_cli.configs.ssl import SSL_FOLDER_PATH @@ -302,3 +303,12 @@ def tmp_config_dir(): yield CONTAINER_CONFIG_TMP_PATH finally: shutil.rmtree(CONTAINER_CONFIG_TMP_PATH) + + +@pytest.fixture +def tmp_schains_dir(): + os.makedirs(SCHAIN_NODE_DATA_PATH) + try: + yield SCHAIN_NODE_DATA_PATH + finally: + shutil.rmtree(SCHAIN_NODE_DATA_PATH) diff --git a/tests/core_checks_test.py b/tests/core/core_checks_test.py similarity index 100% rename from tests/core_checks_test.py rename to tests/core/core_checks_test.py diff --git a/tests/core_node_test.py b/tests/core/core_node_test.py similarity index 100% rename from tests/core_node_test.py rename to tests/core/core_node_test.py diff --git a/tests/core/core_schains_test.py b/tests/core/core_schains_test.py new file mode 100644 index 00000000..29ef337a --- /dev/null +++ b/tests/core/core_schains_test.py @@ -0,0 +1,34 @@ +import os +import datetime + +import freezegun + +from node_cli.core.schains import toggle_schain_repair_mode +from node_cli.utils.helper import read_json + + +CURRENT_TIMESTAMP = 1594903080 +CURRENT_DATETIME = datetime.datetime.utcfromtimestamp(CURRENT_TIMESTAMP) + + +@freezegun.freeze_time(CURRENT_DATETIME) +def test_toggle_repair_mode(tmp_schains_dir): + schain_name = "test_schain" + schain_folder = os.path.join(tmp_schains_dir, schain_name) + os.mkdir(schain_folder) + toggle_schain_repair_mode(schain_name) + schain_status_path = os.path.join(schain_folder, "node-cli.status") + assert os.path.isfile(schain_status_path) + + assert read_json(schain_status_path) == { + 'repair_ts': CURRENT_TIMESTAMP, + 'schain_name': 'test_schain', + } + + toggle_schain_repair_mode(schain_name, snapshot_from='127.0.0.1') + + assert read_json(schain_status_path) == { + 'repair_ts': CURRENT_TIMESTAMP, + 'schain_name': 'test_schain', + 'snapshot_from': '127.0.0.1', + } diff --git a/tests/core_ssl_test.py b/tests/core_ssl_test.py deleted file mode 100644 index a7b3eaff..00000000 --- a/tests/core_ssl_test.py +++ /dev/null @@ -1,95 +0,0 @@ -import os -import pathlib -from docker import APIClient - -import pytest - -from node_cli.core.ssl import upload_cert -from node_cli.core.ssl.check import check_cert_openssl, SSLHealthcheckError -from node_cli.utils.helper import run_cmd -from node_cli.configs.ssl import SSL_CERT_FILEPATH, SSL_KEY_FILEPATH -from node_cli.configs import NGINX_CONTAINER_NAME - - -HOST = '127.0.0.1' - - -@pytest.fixture -def cert_key_pair(): - cert_path = os.path.abspath('ssl-test-cert') - key_path = os.path.abspath('ssl-test-key') - run_cmd([ - 'openssl', 'req', - '-newkey', 'rsa:4096', - '-x509', - '-sha256', - '-days', '365', - '-nodes', - '-subj', '/', - '-out', cert_path, - '-keyout', key_path - ]) - yield cert_path, key_path - if os.path.isfile(cert_path): - pathlib.Path(cert_path).unlink() - if os.path.isfile(key_path): - pathlib.Path(key_path).unlink() - - -@pytest.fixture -def bad_cert(cert_key_pair): - cert, key = cert_key_pair - with open(cert, 'w') as cert_file: - cert_file.write('WRONG CERT') - yield cert, key - - -@pytest.fixture -def bad_key(cert_key_pair): - cert, key = cert_key_pair - with open(key, 'w') as key_file: - key_file.write('WRONG KEY') - yield cert, key - - -def test_verify_cert(cert_key_pair): - cert, key = cert_key_pair - check_cert_openssl(cert, key, host=HOST, no_client=True) - - -def test_verify_cert_self_signed_alert(cert_key_pair): - cert, key = cert_key_pair - with pytest.raises(SSLHealthcheckError): - check_cert_openssl(cert, key, host=HOST, no_client=False) - - -def test_verify_cert_bad_cert(bad_cert): - cert, key = bad_cert - with pytest.raises(SSLHealthcheckError): - check_cert_openssl(cert, key, host=HOST, no_client=True) - - -def test_verify_cert_bad_key(bad_key): - cert, key = bad_key - with pytest.raises(SSLHealthcheckError): - check_cert_openssl(cert, key, host=HOST, no_client=True) - - -def test_upload_cert(cert_key_pair, nginx_container, dutils): - cert, key = cert_key_pair - - docker_api = APIClient() - nginx_container = dutils.containers.get(NGINX_CONTAINER_NAME) - stats = docker_api.inspect_container(nginx_container.id) - started_at = stats['State']['StartedAt'] - - assert not os.path.isfile(SSL_KEY_FILEPATH) - assert not os.path.isfile(SSL_CERT_FILEPATH) - - upload_cert(cert, key, force=False, no_client=True) - - assert os.path.isfile(SSL_KEY_FILEPATH) - assert os.path.isfile(SSL_CERT_FILEPATH) - - stats = docker_api.inspect_container(nginx_container.id) - assert started_at != stats['State']['StartedAt'] From f6d413aa120f0e059d0257f0318892e97c3bb0aa Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 20 Jun 2024 11:36:46 +0000 Subject: [PATCH 02/30] Rename ncli status file --- node_cli/configs/__init__.py | 1 + node_cli/core/schains.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 36f89004..05b88d7f 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -43,6 +43,7 @@ NODE_DATA_PATH = os.path.join(SKALE_DIR, 'node_data') SCHAIN_NODE_DATA_PATH = os.path.join(NODE_DATA_PATH, 'schains') +NODE_CLI_STATUS_FILENAME = 'node_cli.status' NODE_CONFIG_PATH = os.path.join(NODE_DATA_PATH, 'node_config.json') CONTAINER_CONFIG_PATH = os.path.join(SKALE_DIR, 'config') CONTAINER_CONFIG_TMP_PATH = os.path.join(SKALE_TMP_DIR, 'config') diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index 652210d1..59798bda 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -10,6 +10,7 @@ from node_cli.configs import ( ALLOCATION_FILEPATH, NODE_CONFIG_PATH, + NODE_CLI_STATUS_FILENAME, SCHAIN_NODE_DATA_PATH ) from node_cli.configs.env import get_env_config @@ -90,7 +91,7 @@ def show_config(name: str) -> None: def get_node_cli_schain_status_filepath(schain_name: str) -> str: - return os.path.join(SCHAIN_NODE_DATA_PATH, schain_name, 'node-cli.status') + return os.path.join(SCHAIN_NODE_DATA_PATH, schain_name, NODE_CLI_STATUS_FILENAME) def update_node_cli_schain_status(schain_name: str, status: dict) -> None: @@ -104,8 +105,7 @@ def toggle_schain_repair_mode( ) -> None: ts = int(time.time()) status = {'schain_name': schain, 'repair_ts': ts} - if snapshot_from: - status.update({'snapshot_from': snapshot_from}) + status.update({'snapshot_from': snapshot_from}) update_node_cli_schain_status(schain, status) print('Schain has been set for repair') From 38d41e23ec6994fd58ed5bdcee5dce2b1eeef26d Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 18 Jul 2024 16:36:05 +0000 Subject: [PATCH 03/30] Fix tests --- tests/cli/schains_test.py | 19 ++++--------------- tests/core/core_schains_test.py | 3 ++- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/tests/cli/schains_test.py b/tests/cli/schains_test.py index e2bd3669..6db4f21c 100644 --- a/tests/cli/schains_test.py +++ b/tests/cli/schains_test.py @@ -23,7 +23,7 @@ import requests from node_cli.configs import G_CONF_HOME -from tests.helper import response_mock, run_command_mock +from tests.helper import response_mock, run_command, run_command_mock from node_cli.cli.schains import (get_schain_config, ls, dkg, show_rules, repair, info_) @@ -153,7 +153,8 @@ def test_schain_rules(): assert result.output == ' IP range Port \n-----------------------------\n127.0.0.2 - 127.0.0.2 10000\n127.0.0.2 - 127.0.0.2 10001\nAll IPs 10002\nAll IPs 10003\n127.0.0.2 - 127.0.0.2 10004\n127.0.0.2 - 127.0.0.2 10005\nAll IPs 10007\nAll IPs 10008\nAll IPs 10009\n' # noqa -def test_repair(): +def test_repair(tmp_schains_dir): + os.mkdir(os.path.join(tmp_schains_dir, 'test-schain')) os.environ['TZ'] = 'Europe/London' time.tzset() payload = [] @@ -161,22 +162,10 @@ def test_repair(): requests.codes.ok, json_data={'payload': payload, 'status': 'ok'} ) - result = run_command_mock('node_cli.utils.helper.requests.post', resp_mock, repair, - ['test-schain', '--yes']) + result = run_command(repair, ['test-schain', '--yes']) assert result.output == 'Schain has been set for repair\n' assert result.exit_code == 0 - payload = ['error'] - resp_mock = response_mock( - requests.codes.ok, - json_data={'payload': payload, 'status': 'error'} - ) - result = run_command_mock('node_cli.utils.helper.requests.post', resp_mock, repair, - ['test-schain', '--yes']) - print(repr(result.output)) - assert result.exit_code == 3 - assert result.output == f'Command failed with following errors:\n--------------------------------------------------\nerror\n--------------------------------------------------\nYou can find more info in {G_CONF_HOME}.skale/.skale-cli-log/debug-node-cli.log\n' # noqa - def test_info(): payload = { diff --git a/tests/core/core_schains_test.py b/tests/core/core_schains_test.py index 29ef337a..0c27a7ad 100644 --- a/tests/core/core_schains_test.py +++ b/tests/core/core_schains_test.py @@ -17,12 +17,13 @@ def test_toggle_repair_mode(tmp_schains_dir): schain_folder = os.path.join(tmp_schains_dir, schain_name) os.mkdir(schain_folder) toggle_schain_repair_mode(schain_name) - schain_status_path = os.path.join(schain_folder, "node-cli.status") + schain_status_path = os.path.join(schain_folder, "node_cli.status") assert os.path.isfile(schain_status_path) assert read_json(schain_status_path) == { 'repair_ts': CURRENT_TIMESTAMP, 'schain_name': 'test_schain', + 'snapshot_from': None, } toggle_schain_repair_mode(schain_name, snapshot_from='127.0.0.1') From 5840e3a19820ea1418503292b1a252fa0f3058ab Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 18 Jul 2024 16:39:21 +0000 Subject: [PATCH 04/30] Fix linter --- tests/cli/schains_test.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/cli/schains_test.py b/tests/cli/schains_test.py index 6db4f21c..3126976a 100644 --- a/tests/cli/schains_test.py +++ b/tests/cli/schains_test.py @@ -157,11 +157,6 @@ def test_repair(tmp_schains_dir): os.mkdir(os.path.join(tmp_schains_dir, 'test-schain')) os.environ['TZ'] = 'Europe/London' time.tzset() - payload = [] - resp_mock = response_mock( - requests.codes.ok, - json_data={'payload': payload, 'status': 'ok'} - ) result = run_command(repair, ['test-schain', '--yes']) assert result.output == 'Schain has been set for repair\n' assert result.exit_code == 0 From 2ddd6df52d9cfae0d8f05fd1a872258c079f78b8 Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 24 Sep 2024 14:59:53 +0000 Subject: [PATCH 05/30] Bump version --- node_cli/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/cli/__init__.py b/node_cli/cli/__init__.py index 96368eff..4b85052a 100644 --- a/node_cli/cli/__init__.py +++ b/node_cli/cli/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.4.1' +__version__ = '2.5.0' if __name__ == "__main__": print(__version__) From e566b89292d49a2e6bbf29f46d7268e82c4b63ba Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 26 Sep 2024 19:12:36 +0100 Subject: [PATCH 06/30] Run `update_meta` in `turn-on` method --- node_cli/operations/base.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 9a5acda0..3df09aa0 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -281,6 +281,13 @@ def turn_off(): def turn_on(env): logger.info('Turning on the node...') + update_meta( + VERSION, + env['CONTAINER_CONFIGS_STREAM'], + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() + ) if env.get('SKIP_DOCKER_CONFIG') != 'True': configure_docker() logger.info('Launching containers on the node...') From 198d1e4f2a74f18ba0cd48557846b90bbd56ab03 Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 27 Sep 2024 14:58:58 +0000 Subject: [PATCH 07/30] Add quotes to publish branch pattern --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bd164016..15407c80 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,7 +7,7 @@ on: - develop - beta - stable - - v*.*.* + - 'v*.*.*' jobs: create_release: From ccb850eec2a57392a63d78bee68266f71bde374c Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 4 Oct 2024 19:16:20 +0000 Subject: [PATCH 08/30] Check that is safe to update --- node_cli/cli/node.py | 5 +++-- node_cli/core/node.py | 15 ++++++++++++++- node_cli/utils/exit_codes.py | 1 + 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 95437b6b..57d09a15 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -109,10 +109,11 @@ def init_node(env_file): expose_value=False, prompt='Are you sure you want to update SKALE node software?') @click.option('--pull-config', 'pull_config_for_schain', hidden=True, type=str) +@click.option('--unsafe', 'unsafe_ok', hidden=True, type=str) @click.argument('env_file') @streamed_cmd -def update_node(env_file, pull_config_for_schain): - update(env_file, pull_config_for_schain) +def update_node(env_file, pull_config_for_schain, unsafe_ok): + update(env_file, pull_config_for_schain, unsafe_ok) @node.command('signature', help='Get node signature for given validator id') diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 9d5c83f7..b3ff8245 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -92,6 +92,15 @@ class NodeStatuses(Enum): NOT_CREATED = 5 +def is_update_safe() -> bool: + response = get_request(BLUEPRINT_NAME, 'update-safe') + payload = response['payload'] + safe = payload['update_safe'] + if not safe: + logger.info('Locked schains: %s', payload['locked_schains']) + return safe + + @check_inited @check_user def register_node(name, p2p_ip, @@ -259,7 +268,11 @@ def get_node_env( @check_inited @check_user -def update(env_filepath, pull_config_for_schain): +def update(env_filepath, pull_config_for_schain, unsafe_ok: bool = False): + if not unsafe_ok and not is_update_safe(): + error_msg = 'Cannot update safetly' + error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) + logger.info('Node update started') configure_firewall_rules() env = get_node_env( diff --git a/node_cli/utils/exit_codes.py b/node_cli/utils/exit_codes.py index 1173aad0..85656fb1 100644 --- a/node_cli/utils/exit_codes.py +++ b/node_cli/utils/exit_codes.py @@ -30,3 +30,4 @@ class CLIExitCodes(IntEnum): REVERT_ERROR = 6 BAD_USER_ERROR = 7 NODE_STATE_ERROR = 8 + UNSAFE_UPDATE = 9 From 3a0c3ff99bc9a926f0772273239bc727724d189c Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 7 Oct 2024 17:43:49 +0000 Subject: [PATCH 09/30] Fix safe update check. Add tests --- node_cli/cli/node.py | 19 ++++++-- node_cli/cli/sync_node.py | 9 +++- node_cli/configs/routes.py | 30 +++++++++--- node_cli/core/node.py | 19 +++++--- node_cli/utils/helper.py | 3 +- tests/core/core_node_test.py | 91 ++++++++++++++++++------------------ 6 files changed, 108 insertions(+), 63 deletions(-) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 57d09a15..3c907f42 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -109,7 +109,13 @@ def init_node(env_file): expose_value=False, prompt='Are you sure you want to update SKALE node software?') @click.option('--pull-config', 'pull_config_for_schain', hidden=True, type=str) -@click.option('--unsafe', 'unsafe_ok', hidden=True, type=str) +@click.option( + '--unsafe', + 'unsafe_ok', + help='Allow unsafe turn-off', + hidden=True, + is_flag=True +) @click.argument('env_file') @streamed_cmd def update_node(env_file, pull_config_for_schain, unsafe_ok): @@ -174,9 +180,16 @@ def remove_node_from_maintenance(): @click.option('--yes', is_flag=True, callback=abort_if_false, expose_value=False, prompt='Are you sure you want to turn off the node?') +@click.option( + '--unsafe', + 'unsafe_ok', + help='Allow unsafe turn-off', + hidden=True, + is_flag=True +) @streamed_cmd -def _turn_off(maintenance_on): - turn_off(maintenance_on) +def _turn_off(maintenance_on, unsafe_ok): + turn_off(maintenance_on, unsafe_ok) @node.command('turn-on', help='Turn on the node') diff --git a/node_cli/cli/sync_node.py b/node_cli/cli/sync_node.py index bca887b6..5a541e3f 100644 --- a/node_cli/cli/sync_node.py +++ b/node_cli/cli/sync_node.py @@ -74,7 +74,14 @@ def _init_sync(env_file, archive, catchup, historic_state): @click.option('--yes', is_flag=True, callback=abort_if_false, expose_value=False, prompt='Are you sure you want to update SKALE node software?') +@click.option( + '--unsafe', + 'unsafe_ok', + help='Allow unsafe turn-off', + hidden=True, + is_flag=True +) @click.argument('env_file') @streamed_cmd -def _update_sync(env_file): +def _update_sync(env_file, unsafe_ok): update_sync(env_file) diff --git a/node_cli/configs/routes.py b/node_cli/configs/routes.py index 87fac8f5..285e56bd 100644 --- a/node_cli/configs/routes.py +++ b/node_cli/configs/routes.py @@ -25,12 +25,22 @@ ROUTES = { 'v1': { - 'node': ['info', 'register', 'maintenance-on', 'maintenance-off', 'signature', - 'send-tg-notification', 'exit/start', 'exit/status', 'set-domain-name'], + 'node': [ + 'info', + 'register', + 'maintenance-on', + 'maintenance-off', + 'signature', + 'send-tg-notification', + 'exit/start', + 'exit/status', + 'set-domain-name', + 'update-safe', + ], 'health': ['containers', 'schains', 'sgx'], 'schains': ['config', 'list', 'dkg-statuses', 'firewall-rules', 'repair', 'get'], 'ssl': ['status', 'upload'], - 'wallet': ['info', 'send-eth'] + 'wallet': ['info', 'send-eth'], } } @@ -40,8 +50,11 @@ class RouteNotFoundException(Exception): def route_exists(blueprint, method, api_version): - return ROUTES.get(api_version) and ROUTES[api_version].get(blueprint) and \ - method in ROUTES[api_version][blueprint] + return ( + ROUTES.get(api_version) + and ROUTES[api_version].get(blueprint) + and method in ROUTES[api_version][blueprint] + ) def get_route(blueprint, method, api_version=CURRENT_API_VERSION, check=True): @@ -53,5 +66,8 @@ def get_route(blueprint, method, api_version=CURRENT_API_VERSION, check=True): def get_all_available_routes(api_version=CURRENT_API_VERSION): routes = ROUTES[api_version] - return [get_route(blueprint, method, api_version) for blueprint in routes - for method in routes[blueprint]] + return [ + get_route(blueprint, method, api_version) + for blueprint in routes + for method in routes[blueprint] + ] diff --git a/node_cli/core/node.py b/node_cli/core/node.py index b3ff8245..d47cb0b5 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -93,11 +93,12 @@ class NodeStatuses(Enum): def is_update_safe() -> bool: - response = get_request(BLUEPRINT_NAME, 'update-safe') - payload = response['payload'] + status, payload = get_request(BLUEPRINT_NAME, 'update-safe') + if status == 'error': + return False safe = payload['update_safe'] if not safe: - logger.info('Locked schains: %s', payload['locked_schains']) + logger.info('Locked schains: %s', payload['unsafe_chains']) return safe @@ -215,7 +216,10 @@ def init_sync( @check_inited @check_user -def update_sync(env_filepath): +def update_sync(env_filepath: str, unsafe_ok: bool = False) -> None: + if not unsafe_ok and not is_update_safe(): + error_msg = 'Cannot update safetly' + error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) logger.info('Node update started') configure_firewall_rules() env = get_node_env(env_filepath, sync_node=True) @@ -268,7 +272,7 @@ def get_node_env( @check_inited @check_user -def update(env_filepath, pull_config_for_schain, unsafe_ok: bool = False): +def update(env_filepath: str, pull_config_for_schain: str, unsafe_ok: bool = False) -> None: if not unsafe_ok and not is_update_safe(): error_msg = 'Cannot update safetly' error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) @@ -401,7 +405,10 @@ def set_maintenance_mode_off(): @check_inited @check_user -def turn_off(maintenance_on): +def turn_off(maintenance_on: bool = False, unsafe_ok: bool = False) -> None: + if not unsafe_ok and not is_update_safe(): + error_msg = 'Cannot turn off safetly' + error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) if maintenance_on: set_maintenance_mode_on() turn_off_op() diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index cba65ddd..71e559cf 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -24,6 +24,7 @@ import sys import uuid from urllib.parse import urlparse +from typing import Optional import yaml import shutil @@ -230,7 +231,7 @@ def post_request(blueprint, method, json=None, files=None): return status, payload -def get_request(blueprint, method, params=None): +def get_request(blueprint: str, method: str, params: Optional[dict] = None) -> tuple[str, str]: route = get_route(blueprint, method) url = construct_url(route) try: diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index 2ee12036..329e2669 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -12,12 +12,9 @@ from node_cli.configs import NODE_DATA_PATH from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.core.node import BASE_CONTAINERS_AMOUNT, is_base_containers_alive -from node_cli.core.node import init, pack_dir, update +from node_cli.core.node import init, pack_dir, update, is_update_safe -from tests.helper import ( - response_mock, - subprocess_run_mock -) +from tests.helper import response_mock, subprocess_run_mock from tests.resources_test import BIG_DISK_SIZE dclient = docker.from_env() @@ -30,8 +27,7 @@ @pytest.fixture def skale_base_containers(): containers = [ - dclient.containers.run(ALPINE_IMAGE_NAME, detach=True, - name=f'skale_test{i}', command=CMD) + dclient.containers.run(ALPINE_IMAGE_NAME, detach=True, name=f'skale_test{i}', command=CMD) for i in range(BASE_CONTAINERS_AMOUNT) ] yield containers @@ -42,8 +38,7 @@ def skale_base_containers(): @pytest.fixture def skale_base_containers_without_one(): containers = [ - dclient.containers.run(ALPINE_IMAGE_NAME, detach=True, - name=f'skale_test{i}', command=CMD) + dclient.containers.run(ALPINE_IMAGE_NAME, detach=True, name=f'skale_test{i}', command=CMD) for i in range(BASE_CONTAINERS_AMOUNT - 1) ] yield containers @@ -54,8 +49,7 @@ def skale_base_containers_without_one(): @pytest.fixture def skale_base_containers_exited(): containers = [ - dclient.containers.run(HELLO_WORLD_IMAGE_NAME, detach=True, - name=f'skale_test{i}') + dclient.containers.run(HELLO_WORLD_IMAGE_NAME, detach=True, name=f'skale_test{i}') for i in range(BASE_CONTAINERS_AMOUNT) ] time.sleep(10) @@ -92,18 +86,14 @@ def test_pack_dir(tmp_dir): print(tar.getnames()) assert Path(a_data).relative_to(tmp_dir).as_posix() in tar.getnames() assert Path(b_data).relative_to(tmp_dir).as_posix() in tar.getnames() - assert Path(trash_data).relative_to(tmp_dir).as_posix() in \ - tar.getnames() + assert Path(trash_data).relative_to(tmp_dir).as_posix() in tar.getnames() - cleaned_archive_path = os.path.abspath( - os.path.join(tmp_dir, 'cleaned-archive.tar.gz') - ) + cleaned_archive_path = os.path.abspath(os.path.join(tmp_dir, 'cleaned-archive.tar.gz')) pack_dir(backup_dir, cleaned_archive_path, exclude=(trash_dir,)) with tarfile.open(cleaned_archive_path) as tar: assert Path(a_data).relative_to(tmp_dir).as_posix() in tar.getnames() assert Path(b_data).relative_to(tmp_dir).as_posix() in tar.getnames() - assert Path(trash_data).relative_to(tmp_dir).as_posix() not in \ - tar.getnames() + assert Path(trash_data).relative_to(tmp_dir).as_posix() not in tar.getnames() # Not absolute or unrelated path in exclude raises ValueError with pytest.raises(ValueError): @@ -116,9 +106,7 @@ def test_is_base_containers_alive(skale_base_containers): assert is_base_containers_alive() -def test_is_base_containers_alive_one_failed( - skale_base_containers_without_one -): +def test_is_base_containers_alive_one_failed(skale_base_containers_without_one): assert not is_base_containers_alive() @@ -153,17 +141,15 @@ def test_init_node(no_resource_file): # todo: write new init node test resp_mock = response_mock(requests.codes.created) assert not os.path.isfile(RESOURCE_ALLOCATION_FILEPATH) env_filepath = './tests/test-env' - with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.resources.get_disk_size', - return_value=BIG_DISK_SIZE), \ - mock.patch('node_cli.core.host.prepare_host'), \ - mock.patch('node_cli.core.host.init_data_dir'), \ - mock.patch('node_cli.core.node.configure_firewall_rules'), \ - mock.patch('node_cli.core.node.init_op'), \ - mock.patch('node_cli.core.node.is_base_containers_alive', - return_value=True), \ - mock.patch('node_cli.utils.helper.post_request', - resp_mock): + with mock.patch('subprocess.run', new=subprocess_run_mock), mock.patch( + 'node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE + ), mock.patch('node_cli.core.host.prepare_host'), mock.patch( + 'node_cli.core.host.init_data_dir' + ), mock.patch('node_cli.core.node.configure_firewall_rules'), mock.patch( + 'node_cli.core.node.init_op' + ), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch( + 'node_cli.utils.helper.post_request', resp_mock + ): init(env_filepath) assert os.path.isfile(RESOURCE_ALLOCATION_FILEPATH) @@ -172,17 +158,32 @@ def test_update_node(mocked_g_config, resource_file): env_filepath = './tests/test-env' resp_mock = response_mock(requests.codes.created) os.makedirs(NODE_DATA_PATH, exist_ok=True) - with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.node.update_op'), \ - mock.patch('node_cli.core.node.get_flask_secret_key'), \ - mock.patch('node_cli.core.node.save_env_params'), \ - mock.patch('node_cli.core.node.configure_firewall_rules'), \ - mock.patch('node_cli.core.host.prepare_host'), \ - mock.patch('node_cli.core.node.is_base_containers_alive', - return_value=True), \ - mock.patch('node_cli.utils.helper.post_request', - resp_mock), \ - mock.patch('node_cli.core.resources.get_disk_size', - return_value=BIG_DISK_SIZE), \ - mock.patch('node_cli.core.host.init_data_dir'): + with mock.patch('subprocess.run', new=subprocess_run_mock), mock.patch( + 'node_cli.core.node.update_op' + ), mock.patch('node_cli.core.node.get_flask_secret_key'), mock.patch( + 'node_cli.core.node.save_env_params' + ), mock.patch('node_cli.core.node.configure_firewall_rules'), mock.patch( + 'node_cli.core.host.prepare_host' + ), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch( + 'node_cli.utils.helper.post_request', resp_mock + ), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch( + 'node_cli.core.host.init_data_dir' + ): update(env_filepath, pull_config_for_schain=None) + + +def test_is_update_safe(): + assert not is_update_safe() + safe_response = response_mock( + requests.codes.ok, + {'status': 'ok', 'payload': {'update_safe': True, 'unsafe_chains': []}}, + ) + with mock.patch('node_cli.utils.helper.requests.get', return_value=safe_response): + assert is_update_safe() + + unsafe_response = response_mock( + requests.codes.ok, + {'status': 'ok', 'payload': {'update_safe': False, 'unsafe_chains': ['test_chain']}}, + ) + with mock.patch('node_cli.utils.helper.requests.get', return_value=unsafe_response): + assert not is_update_safe() From fb32776f6ea23358de02495cfc7c2712de22e9a9 Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 8 Oct 2024 14:35:29 +0000 Subject: [PATCH 10/30] Fix tests --- tests/cli/node_test.py | 20 ++++-- tests/cli/sync_node_test.py | 120 +++++++++++++++++++---------------- tests/core/core_node_test.py | 21 +++--- tests/helper.py | 15 +++++ tests/routes_test.py | 1 + 5 files changed, 105 insertions(+), 72 deletions(-) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 319137ed..ec09d768 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -38,12 +38,14 @@ _turn_on, _set_domain_name ) +from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import init_default_logger from tests.helper import ( response_mock, run_command, run_command_mock, + safe_update_api_response, subprocess_run_mock ) from tests.resources_test import BIG_DISK_SIZE @@ -123,7 +125,6 @@ def test_register_with_no_alloc(mocked_g_config): register_node, ['--name', 'test-node', '-d', 'skale.test'], input='0.0.0.0\n') assert result.exit_code == 8 - print(repr(result.output)) assert result.output == f'Enter node public IP: 0.0.0.0\nCommand failed with following errors:\n--------------------------------------------------\nNode hasn\'t been inited before.\nYou should run < skale node init >\n--------------------------------------------------\nYou can find more info in {G_CONF_HOME}.skale/.skale-cli-log/debug-node-cli.log\n' # noqa @@ -375,7 +376,18 @@ def test_turn_off_maintenance_on(mocked_g_config): ) with mock.patch('subprocess.run', new=subprocess_run_mock), \ mock.patch('node_cli.core.node.turn_off_op'), \ - mock.patch('node_cli.core.node.is_node_inited', return_value=True): + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): + with mock.patch('node_cli.utils.helper.requests.get', return_value=safe_update_api_response()): + result = run_command_mock( + 'node_cli.utils.helper.requests.post', + resp_mock, + _turn_off, + [ + '--maintenance-on', + '--yes' + ]) + assert result.output == 'Setting maintenance mode on...\nNode is successfully set in maintenance mode\n' # noqa + assert result.exit_code == 0 result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, @@ -384,8 +396,8 @@ def test_turn_off_maintenance_on(mocked_g_config): '--maintenance-on', '--yes' ]) - assert result.exit_code == 0 - assert result.output == 'Setting maintenance mode on...\nNode is successfully set in maintenance mode\n' # noqa + assert 'Cannot turn off safetly' in result.output + assert result.exit_code == CLIExitCodes.UNSAFE_UPDATE def test_turn_on_maintenance_off(mocked_g_config): diff --git a/tests/cli/sync_node_test.py b/tests/cli/sync_node_test.py index 3966d3c8..36c7dd75 100644 --- a/tests/cli/sync_node_test.py +++ b/tests/cli/sync_node_test.py @@ -23,13 +23,12 @@ import logging from node_cli.configs import SKALE_DIR, NODE_DATA_PATH +from node_cli.core.node_options import NodeOptions from node_cli.cli.sync_node import _init_sync, _update_sync +from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import init_default_logger -from node_cli.core.node_options import NodeOptions -from tests.helper import ( - run_command, subprocess_run_mock -) +from tests.helper import run_command, safe_update_api_response, subprocess_run_mock from tests.resources_test import BIG_DISK_SIZE logger = logging.getLogger(__name__) @@ -38,43 +37,41 @@ def test_init_sync(mocked_g_config): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) - with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.node.init_sync_op'), \ - mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ - mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ - mock.patch('node_cli.core.node.configure_firewall_rules'), \ - mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): - result = run_command( - _init_sync, - ['./tests/test-env'] - ) + with mock.patch('subprocess.run', new=subprocess_run_mock), mock.patch( + 'node_cli.core.node.init_sync_op' + ), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch( + 'node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE + ), mock.patch('node_cli.core.node.configure_firewall_rules'), mock.patch( + 'node_cli.utils.decorators.is_node_inited', return_value=False + ): + result = run_command(_init_sync, ['./tests/test-env']) assert result.exit_code == 0 def test_init_sync_archive_catchup(mocked_g_config, clean_node_options): pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) -# with mock.patch('subprocess.run', new=subprocess_run_mock), \ - with mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ - mock.patch('node_cli.operations.base.cleanup_volume_artifacts'), \ - mock.patch('node_cli.operations.base.download_skale_node'), \ - mock.patch('node_cli.operations.base.sync_skale_node'), \ - mock.patch('node_cli.operations.base.configure_docker'), \ - mock.patch('node_cli.operations.base.prepare_host'), \ - mock.patch('node_cli.operations.base.ensure_filestorage_mapping'), \ - mock.patch('node_cli.operations.base.link_env_file'), \ - mock.patch('node_cli.operations.base.download_contracts'), \ - mock.patch('node_cli.operations.base.generate_nginx_config'), \ - mock.patch('node_cli.operations.base.prepare_block_device'), \ - mock.patch('node_cli.operations.base.update_meta'), \ - mock.patch('node_cli.operations.base.update_resource_allocation'), \ - mock.patch('node_cli.operations.base.update_images'), \ - mock.patch('node_cli.operations.base.compose_up'), \ - mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ - mock.patch('node_cli.core.node.configure_firewall_rules'), \ - mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): + # with mock.patch('subprocess.run', new=subprocess_run_mock), \ + with mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch( + 'node_cli.operations.base.cleanup_volume_artifacts' + ), mock.patch('node_cli.operations.base.download_skale_node'), mock.patch( + 'node_cli.operations.base.sync_skale_node' + ), mock.patch('node_cli.operations.base.configure_docker'), mock.patch( + 'node_cli.operations.base.prepare_host' + ), mock.patch('node_cli.operations.base.ensure_filestorage_mapping'), mock.patch( + 'node_cli.operations.base.link_env_file' + ), mock.patch('node_cli.operations.base.download_contracts'), mock.patch( + 'node_cli.operations.base.generate_nginx_config' + ), mock.patch('node_cli.operations.base.prepare_block_device'), mock.patch( + 'node_cli.operations.base.update_meta' + ), mock.patch('node_cli.operations.base.update_resource_allocation'), mock.patch( + 'node_cli.operations.base.update_images' + ), mock.patch('node_cli.operations.base.compose_up'), mock.patch( + 'node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE + ), mock.patch('node_cli.core.node.configure_firewall_rules'), mock.patch( + 'node_cli.utils.decorators.is_node_inited', return_value=False + ): result = run_command( - _init_sync, - ['./tests/test-env', '--archive', '--catchup', '--historic-state'] + _init_sync, ['./tests/test-env', '--archive', '--catchup', '--historic-state'] ) node_options = NodeOptions() @@ -87,30 +84,41 @@ def test_init_sync_archive_catchup(mocked_g_config, clean_node_options): def test_init_sync_historic_state_fail(mocked_g_config, clean_node_options): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) - with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.node.init_sync_op'), \ - mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ - mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ - mock.patch('node_cli.core.node.configure_firewall_rules'), \ - mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): - result = run_command( - _init_sync, - ['./tests/test-env', '--historic-state'] - ) + with mock.patch('subprocess.run', new=subprocess_run_mock), mock.patch( + 'node_cli.core.node.init_sync_op' + ), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch( + 'node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE + ), mock.patch('node_cli.core.node.configure_firewall_rules'), mock.patch( + 'node_cli.utils.decorators.is_node_inited', return_value=False + ): + result = run_command(_init_sync, ['./tests/test-env', '--historic-state']) assert result.exit_code == 1 assert '--historic-state can be used only' in result.output def test_update_sync(mocked_g_config): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) - with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.node.update_sync_op'), \ - mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), \ - mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ - mock.patch('node_cli.core.node.configure_firewall_rules'), \ - mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): - result = run_command( - _update_sync, - ['./tests/test-env', '--yes'] - ) - assert result.exit_code == 0 + + with mock.patch('subprocess.run', new=subprocess_run_mock), mock.patch( + 'node_cli.core.node.update_sync_op' + ), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch( + 'node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE + ), mock.patch('node_cli.core.node.configure_firewall_rules'), mock.patch( + 'node_cli.utils.decorators.is_node_inited', return_value=True + ): + result = run_command(_update_sync, ['./tests/test-env', '--yes']) + assert result.exit_code == CLIExitCodes.UNSAFE_UPDATE + assert 'Cannot update safetly' in result.output + + with mock.patch( + 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response() + ): + result = run_command(_update_sync, ['./tests/test-env', '--yes']) + assert result.exit_code == 0 + + with mock.patch( + 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response(False) + ): + result = run_command(_update_sync, ['./tests/test-env', '--yes']) + assert result.exit_code == CLIExitCodes.UNSAFE_UPDATE + assert 'Cannot update safetly' in result.output diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index 329e2669..7f699645 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -13,8 +13,9 @@ from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.core.node import BASE_CONTAINERS_AMOUNT, is_base_containers_alive from node_cli.core.node import init, pack_dir, update, is_update_safe +from node_cli.utils.exit_codes import CLIExitCodes -from tests.helper import response_mock, subprocess_run_mock +from tests.helper import response_mock, safe_update_api_response, subprocess_run_mock from tests.resources_test import BIG_DISK_SIZE dclient = docker.from_env() @@ -169,21 +170,17 @@ def test_update_node(mocked_g_config, resource_file): ), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch( 'node_cli.core.host.init_data_dir' ): - update(env_filepath, pull_config_for_schain=None) + with mock.patch('node_cli.utils.helper.requests.get', return_value=safe_update_api_response()): + result = update(env_filepath, pull_config_for_schain=None) + assert result is None def test_is_update_safe(): assert not is_update_safe() - safe_response = response_mock( - requests.codes.ok, - {'status': 'ok', 'payload': {'update_safe': True, 'unsafe_chains': []}}, - ) - with mock.patch('node_cli.utils.helper.requests.get', return_value=safe_response): + with mock.patch('node_cli.utils.helper.requests.get', return_value=safe_update_api_response()): assert is_update_safe() - unsafe_response = response_mock( - requests.codes.ok, - {'status': 'ok', 'payload': {'update_safe': False, 'unsafe_chains': ['test_chain']}}, - ) - with mock.patch('node_cli.utils.helper.requests.get', return_value=unsafe_response): + with mock.patch( + 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response(safe=False) + ): assert not is_update_safe() diff --git a/tests/helper.py b/tests/helper.py index 832ac577..c753e176 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -20,6 +20,8 @@ import mock import os + +import requests from click.testing import CliRunner from mock import Mock, MagicMock @@ -84,3 +86,16 @@ def subprocess_run_mock(*args, returncode=0, **kwargs): result.stdout = MagicMock() result.stderr = MagicMock() return result + + +def safe_update_api_response(safe: bool = True) -> dict: + if safe: + return response_mock( + requests.codes.ok, + {'status': 'ok', 'payload': {'update_safe': True, 'unsafe_chains': []}}, + ) + else: + return response_mock( + requests.codes.ok, + {'status': 'ok', 'payload': {'update_safe': False, 'unsafe_chains': ['test_chain']}}, + ) diff --git a/tests/routes_test.py b/tests/routes_test.py index 3cd2416e..9c00b8f1 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -13,6 +13,7 @@ '/api/v1/node/exit/start', '/api/v1/node/exit/status', '/api/v1/node/set-domain-name', + '/api/v1/node/update-safe', '/api/v1/health/containers', '/api/v1/health/schains', From bc3f44680f56171a61ef7ad2b4e52af2e2fdc1ca Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 8 Oct 2024 15:04:15 +0000 Subject: [PATCH 11/30] Fix linter --- tests/cli/node_test.py | 351 ++++++++++++++++++----------------- tests/core/core_node_test.py | 3 +- 2 files changed, 177 insertions(+), 177 deletions(-) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index ec09d768..d1bc5f34 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -36,7 +36,7 @@ version, _turn_off, _turn_on, - _set_domain_name + _set_domain_name, ) from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import init_default_logger @@ -46,7 +46,7 @@ run_command, run_command_mock, safe_update_api_response, - subprocess_run_mock + subprocess_run_mock, ) from tests.resources_test import BIG_DISK_SIZE @@ -55,18 +55,19 @@ def test_register_node(resource_alloc, mocked_g_config): - resp_mock = response_mock( - requests.codes.ok, - {'status': 'ok', 'payload': None} - ) + resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, register_node, - ['--name', 'test-node', '--ip', '0.0.0.0', '--port', '8080', '-d', 'skale.test']) + ['--name', 'test-node', '--ip', '0.0.0.0', '--port', '8080', '-d', 'skale.test'], + ) assert result.exit_code == 0 - assert result.output == 'Node registered in SKALE manager.\nFor more info run < skale node info >\n' # noqa + assert ( + result.output + == 'Node registered in SKALE manager.\nFor more info run < skale node info >\n' + ) # noqa def test_register_node_with_error(resource_alloc, mocked_g_config): @@ -79,98 +80,99 @@ def test_register_node_with_error(resource_alloc, mocked_g_config): 'node_cli.utils.helper.requests.post', resp_mock, register_node, - ['--name', 'test-node2', '--ip', '0.0.0.0', '--port', '80', '-d', 'skale.test']) + ['--name', 'test-node2', '--ip', '0.0.0.0', '--port', '80', '-d', 'skale.test'], + ) assert result.exit_code == 3 - assert result.output == f'Command failed with following errors:\n--------------------------------------------------\nStrange error\n--------------------------------------------------\nYou can find more info in {G_CONF_HOME}.skale/.skale-cli-log/debug-node-cli.log\n' # noqa + assert ( + result.output == f'Command failed with following errors:\n--------------------------------------------------\nStrange error\n--------------------------------------------------\nYou can find more info in {G_CONF_HOME}.skale/.skale-cli-log/debug-node-cli.log\n') # noqa def test_register_node_with_prompted_ip(resource_alloc, mocked_g_config): - resp_mock = response_mock( - requests.codes.ok, - {'status': 'ok', 'payload': None} - ) + resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, register_node, - ['--name', 'test-node', '--port', '8080', '-d', 'skale.test'], input='0.0.0.0\n') + ['--name', 'test-node', '--port', '8080', '-d', 'skale.test'], + input='0.0.0.0\n', + ) assert result.exit_code == 0 assert result.output == 'Enter node public IP: 0.0.0.0\nNode registered in SKALE manager.\nFor more info run < skale node info >\n' # noqa def test_register_node_with_default_port(resource_alloc, mocked_g_config): - resp_mock = response_mock( - requests.codes.ok, - {'status': 'ok', 'payload': None} - ) + resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, register_node, - ['--name', 'test-node', '-d', 'skale.test'], input='0.0.0.0\n') + ['--name', 'test-node', '-d', 'skale.test'], + input='0.0.0.0\n', + ) assert result.exit_code == 0 assert result.output == 'Enter node public IP: 0.0.0.0\nNode registered in SKALE manager.\nFor more info run < skale node info >\n' # noqa def test_register_with_no_alloc(mocked_g_config): - resp_mock = response_mock( - requests.codes.ok, - {'status': 'ok', 'payload': None} - ) + resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, register_node, - ['--name', 'test-node', '-d', 'skale.test'], input='0.0.0.0\n') + ['--name', 'test-node', '-d', 'skale.test'], + input='0.0.0.0\n', + ) assert result.exit_code == 8 - assert result.output == f'Enter node public IP: 0.0.0.0\nCommand failed with following errors:\n--------------------------------------------------\nNode hasn\'t been inited before.\nYou should run < skale node init >\n--------------------------------------------------\nYou can find more info in {G_CONF_HOME}.skale/.skale-cli-log/debug-node-cli.log\n' # noqa + assert result.output == f"Enter node public IP: 0.0.0.0\nCommand failed with following errors:\n--------------------------------------------------\nNode hasn't been inited before.\nYou should run < skale node init >\n--------------------------------------------------\nYou can find more info in {G_CONF_HOME}.skale/.skale-cli-log/debug-node-cli.log\n" # noqa def test_node_info_node_info(): payload = { 'node_info': { - 'name': 'test', 'ip': '0.0.0.0', + 'name': 'test', + 'ip': '0.0.0.0', 'publicIP': '1.1.1.1', 'port': 10001, 'publicKey': '0x7', 'start_date': 1570114466, 'leaving_date': 0, - 'last_reward_date': 1570628924, 'second_address': 0, - 'status': 0, 'id': 32, 'owner': '0x23', - 'domain_name': 'skale.test' + 'last_reward_date': 1570628924, + 'second_address': 0, + 'status': 0, + 'id': 32, + 'owner': '0x23', + 'domain_name': 'skale.test', } } - resp_mock = response_mock( - requests.codes.ok, - json_data={'payload': payload, 'status': 'ok'} - ) + resp_mock = response_mock(requests.codes.ok, json_data={'payload': payload, 'status': 'ok'}) result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 - assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Active\n--------------------------------------------------\n' # noqa + assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Active\n--------------------------------------------------\n') # noqa def test_node_info_node_info_not_created(): payload = { 'node_info': { - 'name': 'test', 'ip': '0.0.0.0', + 'name': 'test', + 'ip': '0.0.0.0', 'publicIP': '1.1.1.1', 'port': 10001, 'publicKey': '0x7', 'start_date': 1570114466, 'leaving_date': 0, - 'last_reward_date': 1570628924, 'second_address': 0, - 'status': 5, 'id': 32, 'owner': '0x23', - 'domain_name': 'skale.test' + 'last_reward_date': 1570628924, + 'second_address': 0, + 'status': 5, + 'id': 32, + 'owner': '0x23', + 'domain_name': 'skale.test', } } - resp_mock = response_mock( - requests.codes.ok, - json_data={'payload': payload, 'status': 'ok'} - ) + resp_mock = response_mock(requests.codes.ok, json_data={'payload': payload, 'status': 'ok'}) result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 assert result.output == 'This SKALE node is not registered on SKALE Manager yet\n' @@ -179,118 +181,127 @@ def test_node_info_node_info_not_created(): def test_node_info_node_info_frozen(): payload = { 'node_info': { - 'name': 'test', 'ip': '0.0.0.0', + 'name': 'test', + 'ip': '0.0.0.0', 'publicIP': '1.1.1.1', 'port': 10001, 'publicKey': '0x7', 'start_date': 1570114466, 'leaving_date': 0, - 'last_reward_date': 1570628924, 'second_address': 0, - 'status': 2, 'id': 32, 'owner': '0x23', - 'domain_name': 'skale.test' + 'last_reward_date': 1570628924, + 'second_address': 0, + 'status': 2, + 'id': 32, + 'owner': '0x23', + 'domain_name': 'skale.test', } } - resp_mock = response_mock( - requests.codes.ok, - json_data={'payload': payload, 'status': 'ok'} - ) + resp_mock = response_mock(requests.codes.ok, json_data={'payload': payload, 'status': 'ok'}) result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 - assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Frozen\n--------------------------------------------------\n' # noqa + assert ( + result.output + == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Frozen\n--------------------------------------------------\n' + ) # noqa def test_node_info_node_info_left(): payload = { 'node_info': { - 'name': 'test', 'ip': '0.0.0.0', + 'name': 'test', + 'ip': '0.0.0.0', 'publicIP': '1.1.1.1', 'port': 10001, 'publicKey': '0x7', 'start_date': 1570114466, 'leaving_date': 0, - 'last_reward_date': 1570628924, 'second_address': 0, - 'status': 4, 'id': 32, 'owner': '0x23', - 'domain_name': 'skale.test' + 'last_reward_date': 1570628924, + 'second_address': 0, + 'status': 4, + 'id': 32, + 'owner': '0x23', + 'domain_name': 'skale.test', } } - resp_mock = response_mock( - requests.codes.ok, - json_data={'payload': payload, 'status': 'ok'} - ) + resp_mock = response_mock(requests.codes.ok, json_data={'payload': payload, 'status': 'ok'}) result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 - assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Left\n--------------------------------------------------\n' # noqa + assert ( + result.output + == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Left\n--------------------------------------------------\n' + ) # noqa def test_node_info_node_info_leaving(): payload = { 'node_info': { - 'name': 'test', 'ip': '0.0.0.0', + 'name': 'test', + 'ip': '0.0.0.0', 'publicIP': '1.1.1.1', 'port': 10001, 'publicKey': '0x7', 'start_date': 1570114466, 'leaving_date': 0, - 'last_reward_date': 1570628924, 'second_address': 0, - 'status': 1, 'id': 32, 'owner': '0x23', - 'domain_name': 'skale.test' + 'last_reward_date': 1570628924, + 'second_address': 0, + 'status': 1, + 'id': 32, + 'owner': '0x23', + 'domain_name': 'skale.test', } } - resp_mock = response_mock( - requests.codes.ok, - json_data={'payload': payload, 'status': 'ok'} - ) + resp_mock = response_mock(requests.codes.ok, json_data={'payload': payload, 'status': 'ok'}) result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 - assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Leaving\n--------------------------------------------------\n' # noqa + assert ( + result.output + == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Leaving\n--------------------------------------------------\n' + ) # noqa def test_node_info_node_info_in_maintenance(): payload = { 'node_info': { - 'name': 'test', 'ip': '0.0.0.0', + 'name': 'test', + 'ip': '0.0.0.0', 'publicIP': '1.1.1.1', 'port': 10001, 'publicKey': '0x7', 'start_date': 1570114466, 'leaving_date': 0, - 'last_reward_date': 1570628924, 'second_address': 0, - 'status': 3, 'id': 32, 'owner': '0x23', - 'domain_name': 'skale.test' + 'last_reward_date': 1570628924, + 'second_address': 0, + 'status': 3, + 'id': 32, + 'owner': '0x23', + 'domain_name': 'skale.test', } } - resp_mock = response_mock( - requests.codes.ok, - json_data={'payload': payload, 'status': 'ok'} - ) + resp_mock = response_mock(requests.codes.ok, json_data={'payload': payload, 'status': 'ok'}) result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 - assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: In Maintenance\n--------------------------------------------------\n' # noqa + assert ( + result.output + == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: In Maintenance\n--------------------------------------------------\n' + ) # noqa def test_node_signature(): signature_sample = '0x1231231231' - response_data = { - 'status': 'ok', - 'payload': {'signature': signature_sample} - } + response_data = {'status': 'ok', 'payload': {'signature': signature_sample}} resp_mock = response_mock(requests.codes.ok, json_data=response_data) - result = run_command_mock('node_cli.utils.helper.requests.get', - resp_mock, signature, ['1']) + result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, signature, ['1']) assert result.exit_code == 0 assert result.output == f'Signature: {signature_sample}\n' def test_backup(): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) - result = run_command( - backup_node, - ['/tmp'] - ) + result = run_command(backup_node, ['/tmp']) assert result.exit_code == 0 print(result.output) assert 'Backup archive succesfully created ' in result.output @@ -298,21 +309,17 @@ def test_backup(): def test_restore(mocked_g_config): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) - result = run_command( - backup_node, - ['/tmp'] + result = run_command(backup_node, ['/tmp']) + backup_path = result.output.replace('Backup archive successfully created: ', '').replace( + '\n', '' ) - backup_path = result.output.replace( - 'Backup archive successfully created: ', '').replace('\n', '') - - with patch('node_cli.core.node.restore_op', MagicMock()) as mock_restore_op, \ - patch('subprocess.run', new=subprocess_run_mock), \ - patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ - patch('node_cli.utils.decorators.is_node_inited', return_value=False): - result = run_command( - restore_node, - [backup_path, './tests/test-env'] - ) + + with patch('node_cli.core.node.restore_op', MagicMock()) as mock_restore_op, patch( + 'subprocess.run', new=subprocess_run_mock + ), patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), patch( + 'node_cli.utils.decorators.is_node_inited', return_value=False + ): + result = run_command(restore_node, [backup_path, './tests/test-env']) assert result.exit_code == 0 assert 'Node is restored from backup\n' in result.output # noqa @@ -321,21 +328,17 @@ def test_restore(mocked_g_config): def test_restore_no_snapshot(mocked_g_config): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) - result = run_command( - backup_node, - ['/tmp'] + result = run_command(backup_node, ['/tmp']) + backup_path = result.output.replace('Backup archive successfully created: ', '').replace( + '\n', '' ) - backup_path = result.output.replace( - 'Backup archive successfully created: ', '').replace('\n', '') - - with patch('node_cli.core.node.restore_op', MagicMock()) as mock_restore_op, \ - patch('subprocess.run', new=subprocess_run_mock), \ - patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ - patch('node_cli.utils.decorators.is_node_inited', return_value=False): - result = run_command( - restore_node, - [backup_path, './tests/test-env', '--no-snapshot'] - ) + + with patch('node_cli.core.node.restore_op', MagicMock()) as mock_restore_op, patch( + 'subprocess.run', new=subprocess_run_mock + ), patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), patch( + 'node_cli.utils.decorators.is_node_inited', return_value=False + ): + result = run_command(restore_node, [backup_path, './tests/test-env', '--no-snapshot']) assert result.exit_code == 0 assert 'Node is restored from backup\n' in result.output # noqa @@ -343,109 +346,107 @@ def test_restore_no_snapshot(mocked_g_config): def test_maintenance_on(): - resp_mock = response_mock( - requests.codes.ok, - {'status': 'ok', 'payload': None} - ) + resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) result = run_command_mock( - 'node_cli.utils.helper.requests.post', - resp_mock, - set_node_in_maintenance, - ['--yes']) + 'node_cli.utils.helper.requests.post', resp_mock, set_node_in_maintenance, ['--yes'] + ) assert result.exit_code == 0 - assert result.output == 'Setting maintenance mode on...\nNode is successfully set in maintenance mode\n' # noqa + assert ( + result.output + == 'Setting maintenance mode on...\nNode is successfully set in maintenance mode\n' + ) # noqa def test_maintenance_off(mocked_g_config): - resp_mock = response_mock( - requests.codes.ok, - {'status': 'ok', 'payload': None} - ) + resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) result = run_command_mock( - 'node_cli.utils.helper.requests.post', - resp_mock, - remove_node_from_maintenance) + 'node_cli.utils.helper.requests.post', resp_mock, remove_node_from_maintenance + ) assert result.exit_code == 0 - assert result.output == 'Setting maintenance mode off...\nNode is successfully removed from maintenance mode\n' # noqa + assert ( + result.output + == 'Setting maintenance mode off...\nNode is successfully removed from maintenance mode\n' + ) # noqa def test_turn_off_maintenance_on(mocked_g_config): - resp_mock = response_mock( - requests.codes.ok, - {'status': 'ok', 'payload': None} - ) - with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.node.turn_off_op'), \ - mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): - with mock.patch('node_cli.utils.helper.requests.get', return_value=safe_update_api_response()): + resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) + with mock.patch('subprocess.run', new=subprocess_run_mock), mock.patch( + 'node_cli.core.node.turn_off_op' + ), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): + with mock.patch( + 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response() + ): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, _turn_off, - [ - '--maintenance-on', - '--yes' - ]) - assert result.output == 'Setting maintenance mode on...\nNode is successfully set in maintenance mode\n' # noqa + ['--maintenance-on', '--yes'], + ) + assert ( + result.output + == 'Setting maintenance mode on...\nNode is successfully set in maintenance mode\n' + ) # noqa assert result.exit_code == 0 result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, _turn_off, - [ - '--maintenance-on', - '--yes' - ]) + ['--maintenance-on', '--yes'], + ) assert 'Cannot turn off safetly' in result.output assert result.exit_code == CLIExitCodes.UNSAFE_UPDATE def test_turn_on_maintenance_off(mocked_g_config): - resp_mock = response_mock( - requests.codes.ok, - {'status': 'ok', 'payload': None} - ) - with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.node.get_flask_secret_key'), \ - mock.patch('node_cli.core.node.turn_on_op'), \ - mock.patch('node_cli.core.node.is_base_containers_alive'), \ - mock.patch('node_cli.core.node.is_node_inited', return_value=True): + resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) + with mock.patch('subprocess.run', new=subprocess_run_mock), mock.patch( + 'node_cli.core.node.get_flask_secret_key' + ), mock.patch('node_cli.core.node.turn_on_op'), mock.patch( + 'node_cli.core.node.is_base_containers_alive' + ), mock.patch('node_cli.core.node.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, _turn_on, - [ - './tests/test-env', - '--maintenance-off', - '--sync-schains', - '--yes' - ]) + ['./tests/test-env', '--maintenance-off', '--sync-schains', '--yes'], + ) assert result.exit_code == 0 - assert result.output == 'Setting maintenance mode off...\nNode is successfully removed from maintenance mode\n' # noqa, tmp fix + assert ( + result.output + == 'Setting maintenance mode off...\nNode is successfully removed from maintenance mode\n' + ) # noqa, tmp fix def test_set_domain_name(): - resp_mock = response_mock( - requests.codes.ok, - {'status': 'ok', 'payload': None} - ) + resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True): result = run_command_mock( 'node_cli.utils.helper.requests.post', resp_mock, - _set_domain_name, ['-d', 'skale.test', '--yes']) + _set_domain_name, + ['-d', 'skale.test', '--yes'], + ) assert result.exit_code == 0 - assert result.output == 'Setting new domain name: skale.test\nDomain name successfully changed\n' # noqa + assert ( + result.output == 'Setting new domain name: skale.test\nDomain name successfully changed\n' + ) # noqa def test_node_version(meta_file_v2): result = run_command(version) assert result.exit_code == 0 - assert result.output == '--------------------------------------------------\nVersion: 0.1.1\nConfig Stream: develop\nLvmpy stream: 1.1.2\n--------------------------------------------------\n' # noqa + assert ( + result.output + == '--------------------------------------------------\nVersion: 0.1.1\nConfig Stream: develop\nLvmpy stream: 1.1.2\n--------------------------------------------------\n' + ) # noqa result = run_command(version, ['--json']) print(repr(result.output)) assert result.exit_code == 0 - assert result.output == "{'version': '0.1.1', 'config_stream': 'develop', 'docker_lvmpy_stream': '1.1.2'}\n" # noqa + assert ( + result.output + == "{'version': '0.1.1', 'config_stream': 'develop', 'docker_lvmpy_stream': '1.1.2'}\n" + ) # noqa diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index 7f699645..c3c3e11d 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -13,7 +13,6 @@ from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.core.node import BASE_CONTAINERS_AMOUNT, is_base_containers_alive from node_cli.core.node import init, pack_dir, update, is_update_safe -from node_cli.utils.exit_codes import CLIExitCodes from tests.helper import response_mock, safe_update_api_response, subprocess_run_mock from tests.resources_test import BIG_DISK_SIZE @@ -170,7 +169,7 @@ def test_update_node(mocked_g_config, resource_file): ), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch( 'node_cli.core.host.init_data_dir' ): - with mock.patch('node_cli.utils.helper.requests.get', return_value=safe_update_api_response()): + with mock.patch('node_cli.utils.helper.requests.get', return_value=safe_update_api_response()): # noqa result = update(env_filepath, pull_config_for_schain=None) assert result is None From 4bf7b944ea02e7c86b1e0ea855a6e566f9db6bef Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 8 Oct 2024 15:17:56 +0000 Subject: [PATCH 12/30] Fix tests --- tests/cli/node_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index d1bc5f34..10816b33 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -150,7 +150,7 @@ def test_node_info_node_info(): resp_mock = response_mock(requests.codes.ok, json_data={'payload': payload, 'status': 'ok'}) result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 - assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Active\n--------------------------------------------------\n') # noqa + assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Active\n--------------------------------------------------\n' # noqa def test_node_info_node_info_not_created(): From dd82abe9bcc475b39831655537c3c9df11fb38fa Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 8 Oct 2024 15:32:58 +0000 Subject: [PATCH 13/30] Fix linter --- tests/cli/node_test.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 10816b33..7e393b27 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -200,10 +200,7 @@ def test_node_info_node_info_frozen(): resp_mock = response_mock(requests.codes.ok, json_data={'payload': payload, 'status': 'ok'}) result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 - assert ( - result.output - == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Frozen\n--------------------------------------------------\n' - ) # noqa + assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Frozen\n--------------------------------------------------\n' # noqa def test_node_info_node_info_left(): @@ -228,10 +225,7 @@ def test_node_info_node_info_left(): resp_mock = response_mock(requests.codes.ok, json_data={'payload': payload, 'status': 'ok'}) result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 - assert ( - result.output - == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Left\n--------------------------------------------------\n' - ) # noqa + assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Left\n--------------------------------------------------\n' # noqa def test_node_info_node_info_leaving(): @@ -256,10 +250,7 @@ def test_node_info_node_info_leaving(): resp_mock = response_mock(requests.codes.ok, json_data={'payload': payload, 'status': 'ok'}) result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 - assert ( - result.output - == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Leaving\n--------------------------------------------------\n' - ) # noqa + assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: Leaving\n--------------------------------------------------\n' # noqa def test_node_info_node_info_in_maintenance(): @@ -284,10 +275,7 @@ def test_node_info_node_info_in_maintenance(): resp_mock = response_mock(requests.codes.ok, json_data={'payload': payload, 'status': 'ok'}) result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, node_info) assert result.exit_code == 0 - assert ( - result.output - == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: In Maintenance\n--------------------------------------------------\n' - ) # noqa + assert result.output == '--------------------------------------------------\nNode info\nName: test\nID: 32\nIP: 0.0.0.0\nPublic IP: 1.1.1.1\nPort: 10001\nDomain name: skale.test\nStatus: In Maintenance\n--------------------------------------------------\n' # noqa def test_node_signature(): @@ -438,10 +426,7 @@ def test_set_domain_name(): def test_node_version(meta_file_v2): result = run_command(version) assert result.exit_code == 0 - assert ( - result.output - == '--------------------------------------------------\nVersion: 0.1.1\nConfig Stream: develop\nLvmpy stream: 1.1.2\n--------------------------------------------------\n' - ) # noqa + assert result.output == '--------------------------------------------------\nVersion: 0.1.1\nConfig Stream: develop\nLvmpy stream: 1.1.2\n--------------------------------------------------\n' # noqa result = run_command(version, ['--json']) print(repr(result.output)) From d2e33df1cb398299a4079e3f85bad0fb68084d78 Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 9 Oct 2024 12:35:12 +0000 Subject: [PATCH 14/30] Fix typo --- node_cli/core/node.py | 6 +++--- tests/cli/node_test.py | 2 +- tests/cli/sync_node_test.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index d47cb0b5..80fa3e3e 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -218,7 +218,7 @@ def init_sync( @check_user def update_sync(env_filepath: str, unsafe_ok: bool = False) -> None: if not unsafe_ok and not is_update_safe(): - error_msg = 'Cannot update safetly' + error_msg = 'Cannot update safely' error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) logger.info('Node update started') configure_firewall_rules() @@ -274,7 +274,7 @@ def get_node_env( @check_user def update(env_filepath: str, pull_config_for_schain: str, unsafe_ok: bool = False) -> None: if not unsafe_ok and not is_update_safe(): - error_msg = 'Cannot update safetly' + error_msg = 'Cannot update safety' error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) logger.info('Node update started') @@ -407,7 +407,7 @@ def set_maintenance_mode_off(): @check_user def turn_off(maintenance_on: bool = False, unsafe_ok: bool = False) -> None: if not unsafe_ok and not is_update_safe(): - error_msg = 'Cannot turn off safetly' + error_msg = 'Cannot turn off safely' error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) if maintenance_on: set_maintenance_mode_on() diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 7e393b27..9c86057c 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -382,7 +382,7 @@ def test_turn_off_maintenance_on(mocked_g_config): _turn_off, ['--maintenance-on', '--yes'], ) - assert 'Cannot turn off safetly' in result.output + assert 'Cannot turn off safely' in result.output assert result.exit_code == CLIExitCodes.UNSAFE_UPDATE diff --git a/tests/cli/sync_node_test.py b/tests/cli/sync_node_test.py index 36c7dd75..84b4a423 100644 --- a/tests/cli/sync_node_test.py +++ b/tests/cli/sync_node_test.py @@ -108,7 +108,7 @@ def test_update_sync(mocked_g_config): ): result = run_command(_update_sync, ['./tests/test-env', '--yes']) assert result.exit_code == CLIExitCodes.UNSAFE_UPDATE - assert 'Cannot update safetly' in result.output + assert 'Cannot update safely' in result.output with mock.patch( 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response() @@ -121,4 +121,4 @@ def test_update_sync(mocked_g_config): ): result = run_command(_update_sync, ['./tests/test-env', '--yes']) assert result.exit_code == CLIExitCodes.UNSAFE_UPDATE - assert 'Cannot update safetly' in result.output + assert 'Cannot update safely' in result.output From f3348a179b4921edca33220293dca4a39f33539c Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 9 Oct 2024 12:41:23 +0000 Subject: [PATCH 15/30] Fix typo --- node_cli/core/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 80fa3e3e..e8895d2a 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -274,7 +274,7 @@ def get_node_env( @check_user def update(env_filepath: str, pull_config_for_schain: str, unsafe_ok: bool = False) -> None: if not unsafe_ok and not is_update_safe(): - error_msg = 'Cannot update safety' + error_msg = 'Cannot update safely' error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) logger.info('Node update started') From 62ca678d3bd1ea3b8d91df0c3e455645edde0985 Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 9 Oct 2024 15:08:07 +0000 Subject: [PATCH 16/30] Fix typo --- node_cli/cli/node.py | 2 +- node_cli/cli/sync_node.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 3c907f42..ff781249 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -112,7 +112,7 @@ def init_node(env_file): @click.option( '--unsafe', 'unsafe_ok', - help='Allow unsafe turn-off', + help='Allow unsafe update', hidden=True, is_flag=True ) diff --git a/node_cli/cli/sync_node.py b/node_cli/cli/sync_node.py index 5a541e3f..a8ad1324 100644 --- a/node_cli/cli/sync_node.py +++ b/node_cli/cli/sync_node.py @@ -77,7 +77,7 @@ def _init_sync(env_file, archive, catchup, historic_state): @click.option( '--unsafe', 'unsafe_ok', - help='Allow unsafe turn-off', + help='Allow unsafe update', hidden=True, is_flag=True ) From 5ca8eeb3da30d6573100de148f8dbd2924c1357a Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 10 Oct 2024 20:31:36 +0000 Subject: [PATCH 17/30] Add --snapshot-from option for sync-node init --- node_cli/cli/sync_node.py | 16 +++++++++++++--- node_cli/core/node.py | 6 ++++-- node_cli/core/schains.py | 1 + node_cli/operations/base.py | 11 +++++++++-- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/node_cli/cli/sync_node.py b/node_cli/cli/sync_node.py index a8ad1324..b6247c1a 100644 --- a/node_cli/cli/sync_node.py +++ b/node_cli/cli/sync_node.py @@ -17,14 +17,17 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Optional + import click from node_cli.core.node import init_sync, update_sync from node_cli.utils.helper import ( abort_if_false, + error_exit, + IP_TYPE, safe_load_texts, streamed_cmd, - error_exit ) from node_cli.utils.exit_codes import CLIExitCodes @@ -60,14 +63,21 @@ def sync_node(): help=TEXTS['init']['historic_state'], is_flag=True ) +@click.option( + '--snapshot-from', + type=IP_TYPE, + default=None, + hidden=True, + help='Ip of the node from to download snapshot from' +) @streamed_cmd -def _init_sync(env_file, archive, catchup, historic_state): +def _init_sync(env_file, archive, catchup, historic_state, snapshot_from: Optional[str]): if historic_state and not archive: error_exit( '--historic-state can be used only is combination with --archive', exit_code=CLIExitCodes.FAILURE ) - init_sync(env_file, archive, catchup, historic_state) + init_sync(env_file, archive, catchup, historic_state, snapshot_from) @sync_node.command('update', help='Update sync node from .env file') diff --git a/node_cli/core/node.py b/node_cli/core/node.py index e8895d2a..e073081d 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -186,7 +186,8 @@ def init_sync( env_filepath: str, archive: bool, catchup: bool, - historic_state: bool + historic_state: bool, + snapshot_from: str ) -> None: configure_firewall_rules() env = get_node_env(env_filepath, sync_node=True) @@ -197,7 +198,8 @@ def init_sync( env, archive, catchup, - historic_state + historic_state, + snapshot_from ) if not inited_ok: error_exit( diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index 59798bda..2d40677e 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -96,6 +96,7 @@ def get_node_cli_schain_status_filepath(schain_name: str) -> str: def update_node_cli_schain_status(schain_name: str, status: dict) -> None: path = get_node_cli_schain_status_filepath(schain_name) + os.makedirs(os.path.dirname(path), exist_ok=True) save_json(path, status) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 3df09aa0..cf4a9eb1 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -20,7 +20,7 @@ import distro import functools import logging -from typing import Dict +from typing import Dict, Optional from node_cli.cli.info import VERSION from node_cli.configs import CONTAINER_CONFIG_PATH, CONTAINER_CONFIG_TMP_PATH @@ -47,6 +47,7 @@ from node_cli.operations.skale_node import download_skale_node, sync_skale_node, update_images from node_cli.core.checks import CheckType, run_checks as run_host_checks from node_cli.core.iptables import configure_iptables +from node_cli.core.schains import update_node_cli_schain_status from node_cli.utils.docker_utils import ( compose_rm, compose_up, @@ -184,7 +185,8 @@ def init_sync( env: dict, archive: bool, catchup: bool, - historic_state: bool + historic_state: bool, + snapshot_from: Optional[str] ) -> bool: cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) download_skale_node( @@ -224,6 +226,11 @@ def init_sync( distro.version() ) update_resource_allocation(env_type=env['ENV_TYPE']) + + schain_name = env['SCHAIN_NAME'] + if snapshot_from: + update_node_cli_schain_status(schain_name, {'snapshot_from': snapshot_from}) + update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True) compose_up(env, sync_node=True) From fc308fa995b2fbd83faf1857a984e7f04de37713 Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 11 Oct 2024 12:13:44 +0000 Subject: [PATCH 18/30] Fix update_sync test --- node_cli/core/node.py | 3 --- tests/cli/sync_node_test.py | 19 ++----------------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index e073081d..da06a11d 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -219,9 +219,6 @@ def init_sync( @check_inited @check_user def update_sync(env_filepath: str, unsafe_ok: bool = False) -> None: - if not unsafe_ok and not is_update_safe(): - error_msg = 'Cannot update safely' - error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) logger.info('Node update started') configure_firewall_rules() env = get_node_env(env_filepath, sync_node=True) diff --git a/tests/cli/sync_node_test.py b/tests/cli/sync_node_test.py index 84b4a423..4431a0a6 100644 --- a/tests/cli/sync_node_test.py +++ b/tests/cli/sync_node_test.py @@ -25,10 +25,9 @@ from node_cli.configs import SKALE_DIR, NODE_DATA_PATH from node_cli.core.node_options import NodeOptions from node_cli.cli.sync_node import _init_sync, _update_sync -from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import init_default_logger -from tests.helper import run_command, safe_update_api_response, subprocess_run_mock +from tests.helper import run_command, subprocess_run_mock from tests.resources_test import BIG_DISK_SIZE logger = logging.getLogger(__name__) @@ -107,18 +106,4 @@ def test_update_sync(mocked_g_config): 'node_cli.utils.decorators.is_node_inited', return_value=True ): result = run_command(_update_sync, ['./tests/test-env', '--yes']) - assert result.exit_code == CLIExitCodes.UNSAFE_UPDATE - assert 'Cannot update safely' in result.output - - with mock.patch( - 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response() - ): - result = run_command(_update_sync, ['./tests/test-env', '--yes']) - assert result.exit_code == 0 - - with mock.patch( - 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response(False) - ): - result = run_command(_update_sync, ['./tests/test-env', '--yes']) - assert result.exit_code == CLIExitCodes.UNSAFE_UPDATE - assert 'Cannot update safely' in result.output + assert result.exit_code == 0 From f9cefef70914f1ba267eb4b09d930033ddfa1ada Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 14 Oct 2024 18:33:40 +0000 Subject: [PATCH 19/30] Add sync-node repair command --- node_cli/cli/sync_node.py | 43 +++++++++++++++++++++++- node_cli/core/node.py | 22 ++++++++++++ node_cli/core/schains.py | 59 +++++++++++++++++++++++++++++---- node_cli/operations/__init__.py | 3 +- node_cli/operations/base.py | 31 +++++++++++++++-- node_cli/utils/docker_utils.py | 50 ++++++++++++++++++++++++++++ 6 files changed, 198 insertions(+), 10 deletions(-) diff --git a/node_cli/cli/sync_node.py b/node_cli/cli/sync_node.py index b6247c1a..304a20d5 100644 --- a/node_cli/cli/sync_node.py +++ b/node_cli/cli/sync_node.py @@ -21,7 +21,7 @@ import click -from node_cli.core.node import init_sync, update_sync +from node_cli.core.node import init_sync, update_sync, repair_sync from node_cli.utils.helper import ( abort_if_false, error_exit, @@ -95,3 +95,44 @@ def _init_sync(env_file, archive, catchup, historic_state, snapshot_from: Option @streamed_cmd def _update_sync(env_file, unsafe_ok): update_sync(env_file) + + +@sync_node.command('repair', help='Start sync node from empty database') +@click.option('--yes', is_flag=True, callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to start sync node from empty database?') +@click.option( + '--archive', + help=TEXTS['init']['archive'], + is_flag=True +) +@click.option( + '--catchup', + help=TEXTS['init']['catchup'], + is_flag=True +) +@click.option( + '--historic-state', + help=TEXTS['init']['historic_state'], + is_flag=True +) +@click.option( + '--snapshot-from', + type=IP_TYPE, + default=None, + hidden=True, + help='Ip of the node from to download snapshot from' +) +@streamed_cmd +def _repair_sync( + archive: str, + catchup: str, + historic_state: str, + snapshot_from: Optional[str] = None +) -> None: + repair_sync( + archive=archive, + catchup=catchup, + historic_state=historic_state, + snapshot_from=snapshot_from + ) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index da06a11d..bfd314b7 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -58,6 +58,7 @@ turn_on_op, restore_op, init_sync_op, + repair_sync_op, update_sync_op ) from node_cli.utils.print_formatters import ( @@ -234,6 +235,27 @@ def update_sync(env_filepath: str, unsafe_ok: bool = False) -> None: logger.info('Node update finished') +@check_inited +@check_user +def repair_sync( + archive: bool, + catchup: bool, + historic_state: bool, + snapshot_from: str +) -> None: + + env_params = extract_env_params(INIT_ENV_FILEPATH, sync_node=True) + schain_name = env_params['SCHAIN_NAME'] + repair_sync_op( + schain_name=schain_name, + archive=archive, + catchup=catchup, + historic_state=historic_state, + snapshot_from=snapshot_from + ) + logger.info('Schain was started from scratch') + + def get_node_env( env_filepath, inited_node=False, diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index 2d40677e..a27b41fc 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -1,3 +1,4 @@ +import glob import logging import os import pprint @@ -11,7 +12,8 @@ ALLOCATION_FILEPATH, NODE_CONFIG_PATH, NODE_CLI_STATUS_FILENAME, - SCHAIN_NODE_DATA_PATH + SCHAIN_NODE_DATA_PATH, + SCHAINS_MNT_DIR_SYNC ) from node_cli.configs.env import get_env_config @@ -94,20 +96,37 @@ def get_node_cli_schain_status_filepath(schain_name: str) -> str: return os.path.join(SCHAIN_NODE_DATA_PATH, schain_name, NODE_CLI_STATUS_FILENAME) -def update_node_cli_schain_status(schain_name: str, status: dict) -> None: +def update_node_cli_schain_status( + schain_name: str, + repair_ts: Optional[int] = None, + snapshot_from: Optional[str] = None +) -> None: path = get_node_cli_schain_status_filepath(schain_name) - os.makedirs(os.path.dirname(path), exist_ok=True) + if os.path.isdir(path): + orig_status = get_node_cli_schain_status(schain_name=schain_name) + orig_status.update({'repair_ts': repair_ts, 'snapshot_from': snapshot_from}) + status = orig_status + else: + status = { + 'schain_name': schain_name, + 'repair_ts': repair_ts, + 'snapshot_from': snapshot_from + } + os.makedirs(os.path.dirname(path), exist_ok=True) save_json(path, status) +def get_node_cli_schain_status(schain_name: str) -> dict: + path = get_node_cli_schain_status_filepath(schain_name) + return read_json(path) + + def toggle_schain_repair_mode( schain: str, snapshot_from: Optional[str] = None ) -> None: ts = int(time.time()) - status = {'schain_name': schain, 'repair_ts': ts} - status.update({'snapshot_from': snapshot_from}) - update_node_cli_schain_status(schain, status) + update_node_cli_schain_status(schain_name=schain, repair_ts=ts, snapshot_from=snapshot_from) print('Schain has been set for repair') @@ -168,6 +187,10 @@ def make_btrfs_snapshot(src: str, dst: str) -> None: run_cmd(['btrfs', 'subvolume', 'snapshot', src, dst]) +def rm_btrfs_subvolume(subvolume: str) -> None: + run_cmd(['btrfs', 'subvolume', 'delete', subvolume]) + + def fillin_snapshot_folder(src_path: str, block_number: int) -> None: snapshots_dirname = 'snapshots' snapshot_folder_path = os.path.join( @@ -224,3 +247,27 @@ def ensure_schain_volume(schain: str, schain_type: str, env_type: str) -> None: ensure_volume(schain, size) else: logger.warning('Volume %s already exists', schain) + + +def cleanup_sync_datadir(schain_name: str) -> None: + base_path = os.path.join(SCHAINS_MNT_DIR_SYNC, schain_name) + regular_folders_pattern = f'{base_path}/[!snapshots]*' + logger.info('Removing regular folders') + for filepath in glob.glob(regular_folders_pattern): + if os.path.isdir(filepath): + logger.debug('Removing recursively %s', filepath) + shutil.rmtree(filepath) + if os.path.isfile(filepath): + os.remove(filepath) + + logger.info('Removing subvolumes') + subvolumes_pattern = f'{base_path}/snapshots/*/*' + for filepath in glob.glob(subvolumes_pattern): + logger.debug('Deleting subvolume %s', filepath) + if os.path.isdir(filepath): + rm_btrfs_subvolume(filepath) + else: + os.remove(filepath) + logger.info('Cleaning up snapshots folder') + if os.path.isdir(base_path): + shutil.rmtree(base_path) diff --git a/node_cli/operations/__init__.py b/node_cli/operations/__init__.py index 11d3dd4d..ca1b076d 100644 --- a/node_cli/operations/__init__.py +++ b/node_cli/operations/__init__.py @@ -24,5 +24,6 @@ update_sync as update_sync_op, turn_off as turn_off_op, turn_on as turn_on_op, - restore as restore_op + restore as restore_op, + repair_sync as repair_sync_op ) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index cf4a9eb1..01c3d616 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -47,12 +47,15 @@ from node_cli.operations.skale_node import download_skale_node, sync_skale_node, update_images from node_cli.core.checks import CheckType, run_checks as run_host_checks from node_cli.core.iptables import configure_iptables -from node_cli.core.schains import update_node_cli_schain_status +from node_cli.core.schains import update_node_cli_schain_status, cleanup_sync_datadir from node_cli.utils.docker_utils import ( compose_rm, compose_up, docker_cleanup, - remove_dynamic_containers + remove_dynamic_containers, + remove_schain_container, + start_admin, + stop_admin ) from node_cli.utils.meta import get_meta_info, update_meta from node_cli.utils.print_formatters import print_failed_requirements_checks @@ -344,3 +347,27 @@ def restore(env, backup_path, config_only=False): print_failed_requirements_checks(failed_checks) return False return True + + +def repair_sync( + schain_name: str, + archive: bool, + catchup: bool, + historic_state: bool, + snapshot_from: Optional[str] +) -> None: + stop_admin(sync_node=True) + remove_schain_container(schain_name=schain_name) + + logger.info('Updating node options') + cleanup_sync_datadir(schain_name=schain_name) + + logger.info('Updating node options') + node_options = NodeOptions() + node_options.archive = archive + node_options.catchup = catchup + node_options.historic_state = historic_state + + logger.info('Updating cli status') + update_node_cli_schain_status(schain_name, snapshot_from=snapshot_from) + start_admin(sync_node=True) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index b48a3306..2f5e56a8 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -21,6 +21,7 @@ import itertools import os import logging +from typing import Optional import docker from docker.client import DockerClient @@ -39,6 +40,7 @@ logger = logging.getLogger(__name__) +ADMIN_REMOVE_TIMEOUT = 60 SCHAIN_REMOVE_TIMEOUT = 300 IMA_REMOVE_TIMEOUT = 20 TELEGRAF_REMOVE_TIMEOUT = 20 @@ -131,6 +133,54 @@ def safe_rm(container: Container, timeout=DOCKER_DEFAULT_STOP_TIMEOUT, **kwargs) logger.info(f'Container removed: {container_name}') +def stop_container( + container_name: str, + timeout: int = DOCKER_DEFAULT_STOP_TIMEOUT, + dclient: Optional[DockerClient] = None +) -> None: + dc = dclient or docker_client() + container = dc.containers.get(container_name) + logger.info('Stopping container: %s, timeout: %s', container_name, timeout) + container.stop(timeout=timeout) + + +def rm_container( + container_name: str, + timeout: int = DOCKER_DEFAULT_STOP_TIMEOUT, + dclient: Optional[DockerClient] = None +) -> None: + dc = dclient or docker_client() + container_names = [container.name for container in get_containers()] + if container_name in container_names: + container = dc.containers.get(container_name) + safe_rm(container) + + +def start_container( + container_name: str, + dclient: Optional[DockerClient] = None +) -> None: + dc = dclient or docker_client() + container = dc.containers.get(container_name) + logger.info('Starting container %s', container_name) + container.start() + + +def start_admin(sync_node: bool = False, dclient: Optional[DockerClient] = None) -> None: + container_name = 'skale_sync_admin' if sync_node else 'skale_admin' + start_container(container_name=container_name, dclient=dclient) + + +def stop_admin(sync_node: bool = False, dclient: Optional[DockerClient] = None) -> None: + container_name = 'skale_sync_admin' if sync_node else 'skale_admin' + stop_container(container_name=container_name, timeout=ADMIN_REMOVE_TIMEOUT, dclient=dclient) + + +def remove_schain_container(schain_name: str, dclient: Optional[DockerClient] = None) -> None: + container_name = f'skale_schain_{schain_name}' + rm_container(container_name, timeout=SCHAIN_REMOVE_TIMEOUT, dclient=dclient) + + def backup_container_logs( container: Container, head: int = DOCKER_DEFAULT_HEAD_LINES, From f044833ff8499f31b686efff89f6c7d70f9535b1 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 14 Oct 2024 19:26:51 +0000 Subject: [PATCH 20/30] Add cleanup_sync_datadir test --- node_cli/core/schains.py | 5 +-- tests/conftest.py | 12 ++++++- tests/core/core_schains_test.py | 61 +++++++++++++++++++++++++++++++-- tests/helper.py | 2 ++ 4 files changed, 74 insertions(+), 6 deletions(-) diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index a27b41fc..0d888835 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -249,8 +249,9 @@ def ensure_schain_volume(schain: str, schain_type: str, env_type: str) -> None: logger.warning('Volume %s already exists', schain) -def cleanup_sync_datadir(schain_name: str) -> None: - base_path = os.path.join(SCHAINS_MNT_DIR_SYNC, schain_name) +def cleanup_sync_datadir(schain_name: str, base_path: str = SCHAINS_MNT_DIR_SYNC) -> None: + base_path = os.path.join(base_path, schain_name) + print('HERE', base_path) regular_folders_pattern = f'{base_path}/[!snapshots]*' logger.info('Removing regular folders') for filepath in glob.glob(regular_folders_pattern): diff --git a/tests/conftest.py b/tests/conftest.py index 93d82521..677b27e6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,7 +44,7 @@ from node_cli.utils.docker_utils import docker_client from node_cli.utils.global_config import generate_g_config_file -from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3 +from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3, TEST_SCHAINS_MNT_DIR_SYNC TEST_ENV_PARAMS = """ @@ -312,3 +312,13 @@ def tmp_schains_dir(): yield SCHAIN_NODE_DATA_PATH finally: shutil.rmtree(SCHAIN_NODE_DATA_PATH) + + +@pytest.fixture +def tmp_sync_datadir(): + + os.makedirs(TEST_SCHAINS_MNT_DIR_SYNC) + try: + yield TEST_SCHAINS_MNT_DIR_SYNC + finally: + shutil.rmtree(TEST_SCHAINS_MNT_DIR_SYNC) diff --git a/tests/core/core_schains_test.py b/tests/core/core_schains_test.py index 0c27a7ad..1681ce20 100644 --- a/tests/core/core_schains_test.py +++ b/tests/core/core_schains_test.py @@ -1,9 +1,12 @@ import os import datetime +from unittest import mock +from pathlib import Path + import freezegun -from node_cli.core.schains import toggle_schain_repair_mode +from node_cli.core.schains import cleanup_sync_datadir, toggle_schain_repair_mode from node_cli.utils.helper import read_json @@ -13,11 +16,11 @@ @freezegun.freeze_time(CURRENT_DATETIME) def test_toggle_repair_mode(tmp_schains_dir): - schain_name = "test_schain" + schain_name = 'test_schain' schain_folder = os.path.join(tmp_schains_dir, schain_name) os.mkdir(schain_folder) toggle_schain_repair_mode(schain_name) - schain_status_path = os.path.join(schain_folder, "node_cli.status") + schain_status_path = os.path.join(schain_folder, 'node_cli.status') assert os.path.isfile(schain_status_path) assert read_json(schain_status_path) == { @@ -33,3 +36,55 @@ def test_toggle_repair_mode(tmp_schains_dir): 'schain_name': 'test_schain', 'snapshot_from': '127.0.0.1', } + + +@freezegun.freeze_time(CURRENT_DATETIME) +def test_cleanup_sync_datadir(tmp_sync_datadir): + schain_name = 'test_schain' + base_folder = Path(tmp_sync_datadir).joinpath(schain_name) + base_folder.mkdir() + folders = [ + '28e07f34', + 'block_sigshares_0.db', + 'da_proofs_0.db', + 'filestorage', + 'incoming_msgs_0.db', + 'proposal_hashes_0.db', + 'snapshots', + 'blocks_0.db', + 'da_sigshares_0.db', + 'historic_roots', + 'internal_info_0.db', + 'outgoing_msgs_0.db', + 'proposal_vectors_0.db', + 'block_proposals_0.db', + 'consensus_state_0.db', + 'diffs', + 'historic_state', + 'prices_0.db', + 'randoms_0.db', + ] + regular_files = ['HEALTH_CHECK', 'keys.info', 'keys.info.salt'] + snapshots = ['0', '100', '111'] + snapshot_content = ['28e07f34', 'blocks_0.db', 'filestorage', 'prices_0.db'] + + for folder_name in folders: + path = base_folder.joinpath(folder_name) + path.mkdir() + + for file_name in regular_files: + path = base_folder.joinpath(file_name) + path.touch() + + for snapshot_block in snapshots: + snapshot_folder = base_folder.joinpath('snapshots', snapshot_block) + snapshot_folder.mkdir() + for folder in snapshot_content: + content_path = snapshot_folder.joinpath(folder) + content_path.mkdir() + hash_path = snapshot_folder.joinpath('snapshot_hash.txt') + hash_path.touch() + + with mock.patch('node_cli.core.schains.rm_btrfs_subvolume'): + cleanup_sync_datadir(schain_name, base_path=tmp_sync_datadir) + assert not os.path.isdir(base_folder) diff --git a/tests/helper.py b/tests/helper.py index c753e176..805fcf51 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -27,6 +27,8 @@ BLOCK_DEVICE = os.getenv('BLOCK_DEVICE') +TEST_SCHAINS_MNT_DIR_SYNC = 'tests/tmp' + TEST_META_V1 = { 'version': '0.1.1', 'config_stream': 'develop' From 9cd26a7ef5589f34a111051d58f364e6bc505662 Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 15 Oct 2024 12:59:34 +0000 Subject: [PATCH 21/30] Add repair_sync test --- node_cli/core/schains.py | 1 - tests/core/core_node_test.py | 13 +++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index 0d888835..ab646b8f 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -251,7 +251,6 @@ def ensure_schain_volume(schain: str, schain_type: str, env_type: str) -> None: def cleanup_sync_datadir(schain_name: str, base_path: str = SCHAINS_MNT_DIR_SYNC) -> None: base_path = os.path.join(base_path, schain_name) - print('HERE', base_path) regular_folders_pattern = f'{base_path}/[!snapshots]*' logger.info('Removing regular folders') for filepath in glob.glob(regular_folders_pattern): diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index c3c3e11d..5bc02f97 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -12,7 +12,7 @@ from node_cli.configs import NODE_DATA_PATH from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.core.node import BASE_CONTAINERS_AMOUNT, is_base_containers_alive -from node_cli.core.node import init, pack_dir, update, is_update_safe +from node_cli.core.node import init, pack_dir, update, is_update_safe, repair_sync from tests.helper import response_mock, safe_update_api_response, subprocess_run_mock from tests.resources_test import BIG_DISK_SIZE @@ -169,7 +169,9 @@ def test_update_node(mocked_g_config, resource_file): ), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch( 'node_cli.core.host.init_data_dir' ): - with mock.patch('node_cli.utils.helper.requests.get', return_value=safe_update_api_response()): # noqa + with mock.patch( + 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response() + ): # noqa result = update(env_filepath, pull_config_for_schain=None) assert result is None @@ -183,3 +185,10 @@ def test_is_update_safe(): 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response(safe=False) ): assert not is_update_safe() + + +def test_repair_sync(tmp_sync_datadir, mocked_g_config, resource_file): + with mock.patch('node_cli.core.schains.rm_btrfs_subvolume'), \ + mock.patch('node_cli.utils.docker_utils.stop_container'), \ + mock.patch('node_cli.utils.docker_utils.start_container'): + repair_sync(archive=True, catchup=True, historic_state=True, snapshot_from='127.0.0.1') From c8611e9b57db7761735151436137b11d719f0f8a Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 15 Oct 2024 13:00:44 +0000 Subject: [PATCH 22/30] Fix linter --- tests/core/core_node_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index 5bc02f97..01f42867 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -190,5 +190,5 @@ def test_is_update_safe(): def test_repair_sync(tmp_sync_datadir, mocked_g_config, resource_file): with mock.patch('node_cli.core.schains.rm_btrfs_subvolume'), \ mock.patch('node_cli.utils.docker_utils.stop_container'), \ - mock.patch('node_cli.utils.docker_utils.start_container'): + mock.patch('node_cli.utils.docker_utils.start_container'): repair_sync(archive=True, catchup=True, historic_state=True, snapshot_from='127.0.0.1') From 363b8cd7e07bf8abe098b9c634b15e59f76cb830 Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 15 Oct 2024 14:47:08 +0000 Subject: [PATCH 23/30] Fix tests --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 677b27e6..54308b49 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -317,7 +317,7 @@ def tmp_schains_dir(): @pytest.fixture def tmp_sync_datadir(): - os.makedirs(TEST_SCHAINS_MNT_DIR_SYNC) + os.makedirs(TEST_SCHAINS_MNT_DIR_SYNC, exist_ok=True) try: yield TEST_SCHAINS_MNT_DIR_SYNC finally: From 110b4988d40ed7c3cabcb471ff2421a6e8c97580 Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 15 Oct 2024 15:01:33 +0000 Subject: [PATCH 24/30] Fix tests --- tests/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 54308b49..824ba93d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -307,7 +307,7 @@ def tmp_config_dir(): @pytest.fixture def tmp_schains_dir(): - os.makedirs(SCHAIN_NODE_DATA_PATH) + os.makedirs(SCHAIN_NODE_DATA_PATH, exist_ok=True) try: yield SCHAIN_NODE_DATA_PATH finally: @@ -316,7 +316,6 @@ def tmp_schains_dir(): @pytest.fixture def tmp_sync_datadir(): - os.makedirs(TEST_SCHAINS_MNT_DIR_SYNC, exist_ok=True) try: yield TEST_SCHAINS_MNT_DIR_SYNC From 22d67022af6973f2a4d858d482f9bfd4dd42ff5c Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 16 Oct 2024 15:53:57 +0000 Subject: [PATCH 25/30] Fix sync init --- node_cli/operations/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 01c3d616..c90a8d13 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -232,7 +232,7 @@ def init_sync( schain_name = env['SCHAIN_NAME'] if snapshot_from: - update_node_cli_schain_status(schain_name, {'snapshot_from': snapshot_from}) + update_node_cli_schain_status(schain_name, snapshot_from=snapshot_from) update_images(env.get('CONTAINER_CONFIGS_DIR') != '', sync_node=True) From d7060216c188511cebe71ea1760b89331c5b5ca7 Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 16 Oct 2024 16:29:27 +0000 Subject: [PATCH 26/30] Switch to URL_TYPE for --snapshot-from --- node_cli/cli/schains.py | 4 ++-- node_cli/cli/sync_node.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/node_cli/cli/schains.py b/node_cli/cli/schains.py index 06e289a6..c6ef4486 100644 --- a/node_cli/cli/schains.py +++ b/node_cli/cli/schains.py @@ -21,7 +21,7 @@ import click -from node_cli.utils.helper import abort_if_false, IP_TYPE +from node_cli.utils.helper import abort_if_false, URL_TYPE from node_cli.core.schains import ( describe, get_schain_firewall_rules, @@ -87,7 +87,7 @@ def show_rules(schain_name: str) -> None: prompt='Are you sure? Repair mode may corrupt working SKALE chain data.') @click.option( '--snapshot-from', - type=IP_TYPE, + type=URL_TYPE, default=None, hidden=True, help='Ip of the node from to download snapshot from' diff --git a/node_cli/cli/sync_node.py b/node_cli/cli/sync_node.py index 304a20d5..45623635 100644 --- a/node_cli/cli/sync_node.py +++ b/node_cli/cli/sync_node.py @@ -25,9 +25,9 @@ from node_cli.utils.helper import ( abort_if_false, error_exit, - IP_TYPE, safe_load_texts, streamed_cmd, + URL_TYPE ) from node_cli.utils.exit_codes import CLIExitCodes @@ -65,7 +65,7 @@ def sync_node(): ) @click.option( '--snapshot-from', - type=IP_TYPE, + type=URL_TYPE, default=None, hidden=True, help='Ip of the node from to download snapshot from' @@ -118,7 +118,7 @@ def _update_sync(env_file, unsafe_ok): ) @click.option( '--snapshot-from', - type=IP_TYPE, + type=URL_TYPE, default=None, hidden=True, help='Ip of the node from to download snapshot from' From 056571a3ef0f06975a9732a0c3aeae6a7d7922dc Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 25 Oct 2024 11:43:58 +0000 Subject: [PATCH 27/30] Set catchup True when --archive --- README.md | 1 - node_cli/cli/sync_node.py | 16 ++-------------- node_cli/core/node.py | 4 ---- node_cli/operations/base.py | 8 ++++---- tests/cli/sync_node_test.py | 10 ++++++++-- tests/core/core_node_test.py | 2 +- text.yml | 1 - 7 files changed, 15 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index e178c238..39762a67 100644 --- a/README.md +++ b/README.md @@ -560,7 +560,6 @@ Options: - `--archive` - Run sync node in an archive node (disable block rotation) - `--historic-state` - Enable historic state (works only in pair with --archive flag) -- `--catchup` - Add a flag to start sync node in catchup mode #### Sync node update diff --git a/node_cli/cli/sync_node.py b/node_cli/cli/sync_node.py index 45623635..81c5618e 100644 --- a/node_cli/cli/sync_node.py +++ b/node_cli/cli/sync_node.py @@ -53,11 +53,6 @@ def sync_node(): help=TEXTS['init']['archive'], is_flag=True ) -@click.option( - '--catchup', - help=TEXTS['init']['catchup'], - is_flag=True -) @click.option( '--historic-state', help=TEXTS['init']['historic_state'], @@ -71,13 +66,13 @@ def sync_node(): help='Ip of the node from to download snapshot from' ) @streamed_cmd -def _init_sync(env_file, archive, catchup, historic_state, snapshot_from: Optional[str]): +def _init_sync(env_file, archive, historic_state, snapshot_from: Optional[str]): if historic_state and not archive: error_exit( '--historic-state can be used only is combination with --archive', exit_code=CLIExitCodes.FAILURE ) - init_sync(env_file, archive, catchup, historic_state, snapshot_from) + init_sync(env_file, archive, historic_state, snapshot_from) @sync_node.command('update', help='Update sync node from .env file') @@ -106,11 +101,6 @@ def _update_sync(env_file, unsafe_ok): help=TEXTS['init']['archive'], is_flag=True ) -@click.option( - '--catchup', - help=TEXTS['init']['catchup'], - is_flag=True -) @click.option( '--historic-state', help=TEXTS['init']['historic_state'], @@ -126,13 +116,11 @@ def _update_sync(env_file, unsafe_ok): @streamed_cmd def _repair_sync( archive: str, - catchup: str, historic_state: str, snapshot_from: Optional[str] = None ) -> None: repair_sync( archive=archive, - catchup=catchup, historic_state=historic_state, snapshot_from=snapshot_from ) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index bfd314b7..fb213a79 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -186,7 +186,6 @@ def restore(backup_path, env_filepath, no_snapshot=False, config_only=False): def init_sync( env_filepath: str, archive: bool, - catchup: bool, historic_state: bool, snapshot_from: str ) -> None: @@ -198,7 +197,6 @@ def init_sync( env_filepath, env, archive, - catchup, historic_state, snapshot_from ) @@ -239,7 +237,6 @@ def update_sync(env_filepath: str, unsafe_ok: bool = False) -> None: @check_user def repair_sync( archive: bool, - catchup: bool, historic_state: bool, snapshot_from: str ) -> None: @@ -249,7 +246,6 @@ def repair_sync( repair_sync_op( schain_name=schain_name, archive=archive, - catchup=catchup, historic_state=historic_state, snapshot_from=snapshot_from ) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index c90a8d13..bc55406b 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -187,7 +187,6 @@ def init_sync( env_filepath: str, env: dict, archive: bool, - catchup: bool, historic_state: bool, snapshot_from: Optional[str] ) -> bool: @@ -208,7 +207,8 @@ def init_sync( node_options = NodeOptions() node_options.archive = archive - node_options.catchup = catchup + if archive: + node_options.catchup = True node_options.historic_state = historic_state ensure_filestorage_mapping() @@ -352,7 +352,6 @@ def restore(env, backup_path, config_only=False): def repair_sync( schain_name: str, archive: bool, - catchup: bool, historic_state: bool, snapshot_from: Optional[str] ) -> None: @@ -365,7 +364,8 @@ def repair_sync( logger.info('Updating node options') node_options = NodeOptions() node_options.archive = archive - node_options.catchup = catchup + if archive: + node_options.catchup = True node_options.historic_state = historic_state logger.info('Updating cli status') diff --git a/tests/cli/sync_node_test.py b/tests/cli/sync_node_test.py index 4431a0a6..4f0517a5 100644 --- a/tests/cli/sync_node_test.py +++ b/tests/cli/sync_node_test.py @@ -44,10 +44,16 @@ def test_init_sync(mocked_g_config): 'node_cli.utils.decorators.is_node_inited', return_value=False ): result = run_command(_init_sync, ['./tests/test-env']) + + node_options = NodeOptions() + assert not node_options.archive + assert not node_options.catchup + assert not node_options.historic_state + assert result.exit_code == 0 -def test_init_sync_archive_catchup(mocked_g_config, clean_node_options): +def test_init_sync_archive(mocked_g_config, clean_node_options): pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) # with mock.patch('subprocess.run', new=subprocess_run_mock), \ with mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch( @@ -70,7 +76,7 @@ def test_init_sync_archive_catchup(mocked_g_config, clean_node_options): 'node_cli.utils.decorators.is_node_inited', return_value=False ): result = run_command( - _init_sync, ['./tests/test-env', '--archive', '--catchup', '--historic-state'] + _init_sync, ['./tests/test-env', '--archive', '--historic-state'] ) node_options = NodeOptions() diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index 01f42867..b945f96f 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -191,4 +191,4 @@ def test_repair_sync(tmp_sync_datadir, mocked_g_config, resource_file): with mock.patch('node_cli.core.schains.rm_btrfs_subvolume'), \ mock.patch('node_cli.utils.docker_utils.stop_container'), \ mock.patch('node_cli.utils.docker_utils.start_container'): - repair_sync(archive=True, catchup=True, historic_state=True, snapshot_from='127.0.0.1') + repair_sync(archive=True, historic_state=True, snapshot_from='127.0.0.1') diff --git a/text.yml b/text.yml index 08bf65e8..cc5c5ec3 100644 --- a/text.yml +++ b/text.yml @@ -65,7 +65,6 @@ sync_node: help: Initialize sync SKALE node archive: Run sync node in an archive node (disable block rotation) historic_state: Enable historic state (works only in pair with --archive flag) - catchup: Add a flag to start sync node in catchup mode lvmpy: help: Lvmpy commands From 70e580235c8c14c94dca50f9b9adc73b17f0bbc4 Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 28 Oct 2024 13:17:12 +0000 Subject: [PATCH 28/30] Fix node-cli.status --- node_cli/operations/base.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index bc55406b..ea5ff162 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -207,8 +207,7 @@ def init_sync( node_options = NodeOptions() node_options.archive = archive - if archive: - node_options.catchup = True + node_options.catchup = archive node_options.historic_state = historic_state ensure_filestorage_mapping() @@ -364,8 +363,7 @@ def repair_sync( logger.info('Updating node options') node_options = NodeOptions() node_options.archive = archive - if archive: - node_options.catchup = True + node_options.catchup = archive node_options.historic_state = historic_state logger.info('Updating cli status') From 982c296725577c7afbca7276cbe3909a0c927ae1 Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 1 Nov 2024 17:15:05 +0000 Subject: [PATCH 29/30] Fix schains formatter --- node_cli/utils/print_formatters.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/node_cli/utils/print_formatters.py b/node_cli/utils/print_formatters.py index 4f0fdc7a..72225db9 100644 --- a/node_cli/utils/print_formatters.py +++ b/node_cli/utils/print_formatters.py @@ -101,19 +101,21 @@ def print_schains(schains): 'Deposit', 'Generation', 'Originator', + 'Type', ] rows = [] for schain in schains: - date = datetime.datetime.fromtimestamp(schain['startDate']) + date = datetime.datetime.fromtimestamp(schain['start_date']) rows.append([ schain['name'], - schain['mainnetOwner'], - schain['partOfNode'], + schain['mainnet_owner'], + schain['part_of_node'], schain['lifetime'], format_date(date), schain['deposit'], schain['generation'], schain['originator'], + schain['options']['allocation_type'] ]) print(Formatter().table(headers, rows)) From 60bdad041b03b4dec41af1def28cb5683280c572 Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 1 Nov 2024 19:10:14 +0000 Subject: [PATCH 30/30] Fix tests --- node_cli/core/health.py | 3 ++- tests/cli/health_test.py | 5 +++-- tests/cli/schains_test.py | 18 +++++++++--------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/node_cli/core/health.py b/node_cli/core/health.py index 623c20d7..247831db 100644 --- a/node_cli/core/health.py +++ b/node_cli/core/health.py @@ -72,7 +72,8 @@ def get_sgx_info(): ['Server URL', data['sgx_server_url']], ['SGXWallet Version', data['sgx_wallet_version']], ['Node SGX keyname', data['sgx_keyname']], - ['Status', data['status_name']] + ['Status HTTPS', data['status_https']], + ['Status ZMQ', data['status_zmq']] ] table = SingleTable(table_data) print(table.table) diff --git a/tests/cli/health_test.py b/tests/cli/health_test.py index f34433df..b2f2fa3f 100644 --- a/tests/cli/health_test.py +++ b/tests/cli/health_test.py @@ -87,7 +87,8 @@ def test_sgx_status(): 'sgx_server_url': 'https://127.0.0.1:1026', 'sgx_wallet_version': '1.50.1-stable.0', 'sgx_keyname': 'test_keyname', - 'status_name': 'CONNECTED' + 'status_zmq': True, + 'status_https': True } resp_mock = response_mock( requests.codes.ok, @@ -97,4 +98,4 @@ def test_sgx_status(): 'node_cli.utils.helper.requests.get', resp_mock, sgx) assert result.exit_code == 0 - assert result.output == '\x1b(0lqqqqqqqqqqqqqqqqqqqwqqqqqqqqqqqqqqqqqqqqqqqqk\x1b(B\n\x1b(0x\x1b(B SGX info \x1b(0x\x1b(B \x1b(0x\x1b(B\n\x1b(0tqqqqqqqqqqqqqqqqqqqnqqqqqqqqqqqqqqqqqqqqqqqqu\x1b(B\n\x1b(0x\x1b(B Server URL \x1b(0x\x1b(B https://127.0.0.1:1026 \x1b(0x\x1b(B\n\x1b(0x\x1b(B SGXWallet Version \x1b(0x\x1b(B 1.50.1-stable.0 \x1b(0x\x1b(B\n\x1b(0x\x1b(B Node SGX keyname \x1b(0x\x1b(B test_keyname \x1b(0x\x1b(B\n\x1b(0x\x1b(B Status \x1b(0x\x1b(B CONNECTED \x1b(0x\x1b(B\n\x1b(0mqqqqqqqqqqqqqqqqqqqvqqqqqqqqqqqqqqqqqqqqqqqqj\x1b(B\n' # noqa + assert result.output == '\x1b(0lqqqqqqqqqqqqqqqqqqqwqqqqqqqqqqqqqqqqqqqqqqqqk\x1b(B\n\x1b(0x\x1b(B SGX info \x1b(0x\x1b(B \x1b(0x\x1b(B\n\x1b(0tqqqqqqqqqqqqqqqqqqqnqqqqqqqqqqqqqqqqqqqqqqqqu\x1b(B\n\x1b(0x\x1b(B Server URL \x1b(0x\x1b(B https://127.0.0.1:1026 \x1b(0x\x1b(B\n\x1b(0x\x1b(B SGXWallet Version \x1b(0x\x1b(B 1.50.1-stable.0 \x1b(0x\x1b(B\n\x1b(0x\x1b(B Node SGX keyname \x1b(0x\x1b(B test_keyname \x1b(0x\x1b(B\n\x1b(0x\x1b(B Status HTTPS \x1b(0x\x1b(B True \x1b(0x\x1b(B\n\x1b(0x\x1b(B Status ZMQ \x1b(0x\x1b(B True \x1b(0x\x1b(B\n\x1b(0mqqqqqqqqqqqqqqqqqqqvqqqqqqqqqqqqqqqqqqqqqqqqj\x1b(B\n' # noqa diff --git a/tests/cli/schains_test.py b/tests/cli/schains_test.py index 3126976a..d12e7277 100644 --- a/tests/cli/schains_test.py +++ b/tests/cli/schains_test.py @@ -33,17 +33,17 @@ def test_ls(): time.tzset() payload = [ { - 'name': 'test_schain1', 'mainnetOwner': '0x123', - 'indexInOwnerList': 3, 'partOfNode': 0, - 'lifetime': 5, 'startDate': 1570115385, - 'deposit': 1000000000000000000, 'index': 3, 'generation': 1, 'originator': '0x465' + 'name': 'test_schain1', 'mainnet_owner': '0x123', + 'index_owner_list': 3, 'part_of_node': 0, + 'lifetime': 5, 'start_date': 1570115385, + 'deposit': 1000000000000000000, 'index': 3, 'generation': 1, 'originator': '0x465', 'options': {'allocation_type': 0} # noqa }, { 'name': 'crazy_cats1', - 'mainnetOwner': '0x321', - 'indexInOwnerList': 8, 'partOfNode': 0, - 'lifetime': 5, 'startDate': 1570469410, - 'deposit': 1000000000000000000, 'index': 8, 'generation': 0, 'originator': '0x0' + 'mainnet_owner': '0x321', + 'index_owner_list': 8, 'part_of_node': 0, + 'lifetime': 5, 'start_date': 1570469410, + 'deposit': 1000000000000000000, 'index': 8, 'generation': 0, 'originator': '0x0', 'options': {'allocation_type': 0} # noqa } ] resp_mock = response_mock( @@ -52,7 +52,7 @@ def test_ls(): ) result = run_command_mock('node_cli.utils.helper.requests.get', resp_mock, ls) assert result.exit_code == 0 - assert result.output == ' Name Owner Size Lifetime Created At Deposit Generation Originator\n-------------------------------------------------------------------------------------------------------------\ntest_schain1 0x123 0 5 Oct 03 2019 16:09:45 1000000000000000000 1 0x465 \ncrazy_cats1 0x321 0 5 Oct 07 2019 18:30:10 1000000000000000000 0 0x0 \n' # noqa + assert result.output == ' Name Owner Size Lifetime Created At Deposit Generation Originator Type\n--------------------------------------------------------------------------------------------------------------------\ntest_schain1 0x123 0 5 Oct 03 2019 16:09:45 1000000000000000000 1 0x465 0 \ncrazy_cats1 0x321 0 5 Oct 07 2019 18:30:10 1000000000000000000 0 0x0 0 \n' # noqa def test_dkg():