diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..060cae42 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +tests +helper-scripts +dist +build +.github +.gitmodules diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 645b1ecb..41e6a727 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,7 +12,7 @@ jobs: create_release: if: github.event.pull_request.merged name: Create release - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 outputs: upload_url: ${{ steps.create_release.outputs.upload_url }} version: ${{ steps.export_outputs.outputs.version }} @@ -20,6 +20,9 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + with: + submodules: true + - name: Checkout submodules run: git submodule update --init - name: Install ubuntu dependencies @@ -61,37 +64,53 @@ jobs: strategy: matrix: include: - - os: ubuntu-18.04 + - os: ubuntu-20.04 asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64 steps: - uses: actions/checkout@v2 - - name: Set up Python 3.7 + - name: Set up Python 3.11 uses: actions/setup-python@v1 with: - python-version: 3.7 + python-version: 3.11 + - name: Install ubuntu dependencies - if: matrix.os == 'ubuntu-18.04' + if: matrix.os == 'ubuntu-20.04' run: | sudo apt-get update - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install -e . - pip install -e .[dev] - pip install wheel - pip install --upgrade 'setuptools<45.0.0' + - name: Checkout submodules run: git submodule update --init - - name: Build library + + - name: Build binary + run: | + mkdir ./dist + docker build . -t node-cli-builder + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} + ls -altr /home/ubuntu/dist/ + docker rm -f $(docker ps -aq) + + - name: Save sha512sum run: | - bash ./scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} - - name: Upload Release Asset + sudo sha512sum /home/ubuntu/dist/${{ matrix.asset_name }} | sudo tee > /dev/null /home/ubuntu/dist/sha512sum + + - name: Upload release binary id: upload-release-asset uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: ./dist/${{ matrix.asset_name }} + asset_path: /home/ubuntu/dist/${{ matrix.asset_name }} asset_name: ${{ matrix.asset_name }} asset_content_type: application/octet-stream + + - name: Upload release checksum + id: upload-release-checksum + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: /home/ubuntu/dist/sha512sum + asset_name: ${{ matrix.asset_name }}.sha512 + asset_content_type: text/plain diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b69634d7..3e45e8d8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,32 +3,55 @@ on: [push, pull_request] jobs: test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 strategy: matrix: - python-version: [3.7] + python-version: [3.11] steps: - uses: actions/checkout@v2 + with: + submodules: true + + - name: Checkout submodules + run: git submodule update --init + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + - name: Install ubuntu dependencies run: | sudo apt-get update - sudo apt-get install python-setuptools + sudo apt-get install python-setuptools iptables + - name: Install python dependencies run: | python -m pip install --upgrade pip pip install -e . pip install -e .[dev] pip install --upgrade 'setuptools<45.0.0' + - name: Lint with flake8 run: | flake8 . - - name: Build binary + + - name: Build binary in Ubuntu 18.04 environment run: | - bash scripts/build.sh 1.0.0 test-branch - - name: Run tests + mkdir ./dist + docker build . -t node-cli-builder + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test + docker rm -f $(docker ps -aq) + + - name: Check build + run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 + + - name: Build binary in Ubuntu 20.04 environment run: | - bash ./scripts/run_tests.sh + scripts/build.sh test test + + - name: Check build + run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 + + - name: Run tests + run: bash ./scripts/run_tests.sh diff --git a/.gitmodules b/.gitmodules index d54a2086..b9ae3738 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,9 @@ [submodule "helper-scripts"] path = helper-scripts url = https://github.com/skalenetwork/helper-scripts.git + branch = develop + +[submodule "lvmpy"] + path = lvmpy + url = https://github.com/skalenetwork/docker-lvmpy branch = develop diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..67a6a42d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM ubuntu:18.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get install -y \ + git \ + python3.11 \ + libpython3.11-dev \ + python3.11-venv \ + python3.11-distutils \ + python3.11-dev \ + build-essential \ + zlib1g-dev \ + libssl-dev \ + libffi-dev \ + swig \ + iptables + +RUN mkdir /app +WORKDIR /app + +COPY . . + +ENV PATH=/app/buildvenv/bin:$PATH +RUN python3.11 -m venv /app/buildvenv && \ + pip install --upgrade pip && \ + pip install wheel setuptools==63.2.0 && \ + pip install -e '.[dev]' diff --git a/lvmpy b/lvmpy new file mode 160000 index 00000000..8ee051bf --- /dev/null +++ b/lvmpy @@ -0,0 +1 @@ +Subproject commit 8ee051bf24aa3feecc0ef97fb5eec970eb068512 diff --git a/main.spec b/main.spec index 3c776386..a4dbe395 100644 --- a/main.spec +++ b/main.spec @@ -1,9 +1,5 @@ # -*- mode: python -*- -# import distutils -# if distutils.distutils_path.endswith('__init__.py'): -# distutils.distutils_path = os.path.dirname(distutils.distutils_path) - import importlib.util libxtwrapper_path = importlib.util.find_spec('libxtwrapper').origin diff --git a/node_cli/cli/__init__.py b/node_cli/cli/__init__.py index 680cf87f..18f56790 100644 --- a/node_cli/cli/__init__.py +++ b/node_cli/cli/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.2.0' +__version__ = '2.3.0' if __name__ == "__main__": print(__version__) diff --git a/node_cli/cli/lvmpy.py b/node_cli/cli/lvmpy.py new file mode 100644 index 00000000..473defa8 --- /dev/null +++ b/node_cli/cli/lvmpy.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2020 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click + +from node_cli.utils.helper import abort_if_false +from node_cli.utils.texts import Texts +from lvmpy.src.app import run as run_lvmpy +from lvmpy.src.health import heal_service + +G_TEXTS = Texts() +TEXTS = G_TEXTS['lvmpy'] + + +@click.group() +def lvmpy_cli(): + pass + + +@lvmpy_cli.group('lvmpy', help=TEXTS['help']) +def health(): + pass + + +@health.command(help=TEXTS['run']['help']) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt=TEXTS['run']['prompt'] +) +def run(): + run_lvmpy() + + +@health.command(help=TEXTS['heal']['help']) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt=TEXTS['heal']['prompt'] +) +def heal(): + heal_service() diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 4c3618a2..eec6c59a 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -131,9 +131,15 @@ def backup_node(backup_folder_path): @node.command('restore', help="Restore SKALE node on another machine") @click.argument('backup_path') @click.argument('env_file') +@click.option( + '--no-snapshot', + help='Do not restore sChains from snapshot', + is_flag=True, + hidden=True +) @streamed_cmd -def restore_node(backup_path, env_file): - restore(backup_path, env_file) +def restore_node(backup_path, env_file, no_snapshot): + restore(backup_path, env_file, no_snapshot) @node.command('maintenance-on', help="Set SKALE node into maintenance mode") diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index cb7a1b36..c8f71d35 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -35,6 +35,7 @@ FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage') SNAPSHOTS_SHARED_VOLUME = 'shared-space' SCHAINS_MNT_DIR = '/mnt' +VOLUME_GROUP = 'schains' SKALE_DIR = os.path.join(G_CONF_HOME, '.skale') SKALE_TMP_DIR = os.path.join(SKALE_DIR, '.tmp') @@ -73,6 +74,16 @@ FILEBEAT_CONFIG_PATH = os.path.join(NODE_DATA_PATH, 'filebeat.yml') DOCKER_LVMPY_PATH = os.path.join(SKALE_DIR, 'docker-lvmpy') +DOCKER_LVMPY_BIN_PATH = '/usr/local/bin/skale' + +LVMPY_CMD = f'{DOCKER_LVMPY_BIN_PATH} lvmpy' +LVMPY_RUN_CMD = f'{LVMPY_CMD} run --yes' +LVMPY_HEAL_CMD = f'{LVMPY_CMD} heal --yes' + +LVMPY_CRON_LOG_PATH = '/var/log/docker-lvmpy/cron.log' +LVMPY_CRON_SCHEDULE_MINUTES = 3 + +LVMPY_LOG_DIR = '/var/log/docker-lvmpy' IPTABLES_DIR = '/etc/iptables/' IPTABLES_RULES_STATE_FILEPATH = os.path.join(IPTABLES_DIR, 'rules.v4') diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 3a4601a7..b8aa637d 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -143,13 +143,17 @@ def init(env_filepath): @check_not_inited -def restore(backup_path, env_filepath): +def restore(backup_path, env_filepath, no_snapshot=False): env = get_node_env(env_filepath) if env is None: return save_env_params(env_filepath) env['SKALE_DIR'] = SKALE_DIR - env['BACKUP_RUN'] = 'True' # should be str + + if not no_snapshot: + logger.info('Adding BACKUP_RUN to env ...') + env['BACKUP_RUN'] = 'True' # should be str + restored_ok = restore_op(env, backup_path) if not restored_ok: error_exit( diff --git a/node_cli/main.py b/node_cli/main.py index 6a0abe83..aa1a7e54 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -29,6 +29,7 @@ from node_cli.cli.health import health_cli from node_cli.cli.info import BUILD_DATETIME, COMMIT, BRANCH, OS, VERSION from node_cli.cli.logs import logs_cli +from node_cli.cli.lvmpy import lvmpy_cli from node_cli.cli.node import node_cli from node_cli.cli.schains import schains_cli from node_cli.cli.wallet import wallet_cli @@ -97,6 +98,7 @@ def handle_exception(exc_type, exc_value, exc_traceback): health_cli, schains_cli, logs_cli, + lvmpy_cli, resources_allocation_cli, node_cli, wallet_cli, diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 1d7bd27e..81c1dab0 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import distro import functools import logging from typing import Dict, Optional @@ -36,7 +37,7 @@ configure_filebeat, configure_flask, unpack_backup_archive ) -from node_cli.operations.docker_lvmpy import docker_lvmpy_update, docker_lvmpy_install +from node_cli.operations.docker_lvmpy import lvmpy_install # noqa from node_cli.operations.skale_node import download_skale_node, sync_skale_node, update_images from node_cli.core.checks import CheckType, run_checks as run_host_checks from node_cli.core.iptables import configure_iptables @@ -103,7 +104,7 @@ def update(env_filepath: str, env: Dict) -> None: backup_old_contracts() download_contracts(env) - docker_lvmpy_update(env) + lvmpy_install(env) generate_nginx_config() prepare_host( @@ -128,7 +129,9 @@ def update(env_filepath: str, env: Dict) -> None: update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_images(env.get('CONTAINER_CONFIGS_DIR') != '') compose_up(env) @@ -157,7 +160,7 @@ def init(env_filepath: str, env: Dict, snapshot_from: Optional[str] = None) -> b configure_iptables() generate_nginx_config() - docker_lvmpy_install(env) + lvmpy_install(env) init_shared_space_volume(env['ENV_TYPE']) node_options = NodeOptions() @@ -166,7 +169,9 @@ def init(env_filepath: str, env: Dict, snapshot_from: Optional[str] = None) -> b update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_resource_allocation( disk_device=env['DISK_MOUNTPOINT'], @@ -207,13 +212,15 @@ def restore(env, backup_path): link_env_file() configure_iptables() - docker_lvmpy_install(env) + lvmpy_install(env) init_shared_space_volume(env['ENV_TYPE']) update_meta( VERSION, env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'] + env['DOCKER_LVMPY_STREAM'], + distro.id(), + distro.version() ) update_resource_allocation( disk_device=env['DISK_MOUNTPOINT'], diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py index 00f2deb2..31036c95 100644 --- a/node_cli/operations/docker_lvmpy.py +++ b/node_cli/operations/docker_lvmpy.py @@ -21,10 +21,21 @@ import os import shutil -from node_cli.utils.helper import run_cmd +import crontab + from node_cli.utils.git_utils import sync_repo -from node_cli.configs import (DOCKER_LVMPY_PATH, DOCKER_LVMPY_REPO_URL, - FILESTORAGE_MAPPING, SCHAINS_MNT_DIR) +from node_cli.configs import ( + DOCKER_LVMPY_PATH, + DOCKER_LVMPY_REPO_URL, + FILESTORAGE_MAPPING, + LVMPY_RUN_CMD, + LVMPY_HEAL_CMD, + LVMPY_CRON_LOG_PATH, + LVMPY_CRON_SCHEDULE_MINUTES, + SCHAINS_MNT_DIR, + VOLUME_GROUP +) +from lvmpy.src.install import setup as setup_lvmpy logger = logging.getLogger(__name__) @@ -33,7 +44,7 @@ def update_docker_lvmpy_env(env): env['PHYSICAL_VOLUME'] = env['DISK_MOUNTPOINT'] env['VOLUME_GROUP'] = 'schains' env['FILESTORAGE_MAPPING'] = FILESTORAGE_MAPPING - env['SCHAINS_MNT_DIR'] = SCHAINS_MNT_DIR + env['MNT_DIR'] = SCHAINS_MNT_DIR env['PATH'] = os.environ.get('PATH', None) return env @@ -53,24 +64,29 @@ def sync_docker_lvmpy_repo(env): ) -def docker_lvmpy_update(env): - sync_docker_lvmpy_repo(env) +def lvmpy_install(env): ensure_filestorage_mapping() - logger.info('Running docker-lvmpy update script') - update_docker_lvmpy_env(env) - run_cmd( - cmd=f'sudo -H -E {DOCKER_LVMPY_PATH}/scripts/update.sh'.split(), - env=env + logging.info('Configuring and starting lvmpy') + setup_lvmpy( + block_device=env['DISK_MOUNTPOINT'], + volume_group=VOLUME_GROUP, + exec_start=LVMPY_RUN_CMD ) - logger.info('docker-lvmpy update done') + init_healing_cron() + logger.info('docker-lvmpy is configured and started') -def docker_lvmpy_install(env): - sync_docker_lvmpy_repo(env) - ensure_filestorage_mapping() - update_docker_lvmpy_env(env) - run_cmd( - cmd=f'sudo -H -E {DOCKER_LVMPY_PATH}/scripts/install.sh'.split(), - env=env - ) - logger.info('docker-lvmpy installed') +def init_healing_cron(): + logger.info('Configuring cron job for healing lvmpy') + cron_line = f'{LVMPY_HEAL_CMD} >> {LVMPY_CRON_LOG_PATH} 2>&1' + legacy_line = f'cd /opt/docker-lvmpy && venv/bin/python -c "import health; health.heal_service()" >> {LVMPY_CRON_LOG_PATH} 2>&1' # noqa + + with crontab.CronTab(user='root') as c: + jobs = [c.command for c in c] + if legacy_line in jobs: + c.remove_all(command=legacy_line) + if cron_line not in jobs: + job = c.new( + command=cron_line + ) + job.minute.every(LVMPY_CRON_SCHEDULE_MINUTES) diff --git a/node_cli/utils/meta.py b/node_cli/utils/meta.py index b49d9f61..69078af2 100644 --- a/node_cli/utils/meta.py +++ b/node_cli/utils/meta.py @@ -6,20 +6,23 @@ DEFAULT_VERSION = '1.0.0' DEFAULT_CONFIG_STREAM = '1.1.0' DEFAULT_DOCKER_LVMPY_STREAM = '1.0.0' +DEFAULT_OS_ID = 'ubuntu' +DEFAULT_OS_VERSION = '18.04' class CliMeta( namedtuple( 'Node', - ['version', 'config_stream', 'docker_lvmpy_stream'] + ['version', 'config_stream', 'docker_lvmpy_stream', 'os_id', 'os_version'] ) ): __slots__ = () def __new__(cls, version=DEFAULT_VERSION, config_stream=DEFAULT_CONFIG_STREAM, - docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM): + docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM, os_id=DEFAULT_OS_ID, + os_version=DEFAULT_OS_VERSION): return super(CliMeta, cls).__new__( - cls, version, config_stream, docker_lvmpy_stream + cls, version, config_stream, docker_lvmpy_stream, os_id, os_version ) @@ -41,7 +44,8 @@ def save_meta(meta: CliMeta) -> None: def compose_default_meta() -> CliMeta: return CliMeta(version=DEFAULT_VERSION, docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM, - config_stream=DEFAULT_CONFIG_STREAM) + config_stream=DEFAULT_CONFIG_STREAM, os_id=DEFAULT_OS_ID, + os_version=DEFAULT_OS_VERSION) def ensure_meta(meta: CliMeta = None) -> None: @@ -51,7 +55,7 @@ def ensure_meta(meta: CliMeta = None) -> None: def update_meta(version: str, config_stream: str, - docker_lvmpy_stream: str) -> None: + docker_lvmpy_stream: str, os_id: str, os_version: str) -> None: ensure_meta() - meta = CliMeta(version, config_stream, docker_lvmpy_stream) + meta = CliMeta(version, config_stream, docker_lvmpy_stream, os_id, os_version) save_meta(meta) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 12cce17a..afbb2068 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -3,4 +3,9 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" PROJECT_DIR=$(dirname $DIR) -HIDE_STREAM_LOG=true TEST_HOME_DIR="$PROJECT_DIR/tests/" GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" DOTENV_FILEPATH='tests/test-env' py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ +LVMPY_LOG_DIR="$PROJECT_DIR/tests/" \ + HIDE_STREAM_LOG=true \ + TEST_HOME_DIR="$PROJECT_DIR/tests/" \ + GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" \ + DOTENV_FILEPATH='tests/test-env' \ + py.test --cov=$PROJECT_DIR/ tests/ --ignore=tests/operations/ $@ diff --git a/setup.py b/setup.py index 3dfcc3f5..dc335888 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ def find_version(*file_paths): install_requires=[ "click==8.1.3", "PyInstaller==5.6.2", + "distro==1.4.0", "docker==6.0.1", "texttable==1.6.4", "python-dateutil==2.8.2", @@ -60,14 +61,18 @@ def find_version(*file_paths): "python-dotenv==0.21.0", "terminaltables==3.1.10", "requests==2.28.1", - "GitPython==3.1.27", + "GitPython==3.1.30", "packaging==21.3", "python-debian==0.1.48", - "python-iptables==1.0.0", + "python-iptables==1.0.1", "PyYAML==6.0", "MarkupSafe==2.1.1", + 'Flask==2.2.2', + 'itsdangerous==2.0.1', + 'sh==1.14.2', + 'python-crontab==2.6.0' ], - python_requires='>=3.6,<4', + python_requires='>=3.8,<4', extras_require=extras_require, keywords=['skale', 'cli'], @@ -77,6 +82,6 @@ def find_version(*file_paths): 'Intended Audience :: Developers', 'License :: OSI Approved :: GNU Affero General Public License v3', 'Natural Language :: English', - 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.11', ], ) diff --git a/tests/cli/main_test.py b/tests/cli/main_test.py index 2703f0b5..5ce570ad 100644 --- a/tests/cli/main_test.py +++ b/tests/cli/main_test.py @@ -18,14 +18,13 @@ # along with this program. If not, see . -from node_cli.cli import info from node_cli.main import version from tests.helper import run_command def test_version(): result = run_command(version, []) - expected = f'SKALE Node CLI version: {info.VERSION}\n' + expected = 'SKALE Node CLI version: test\n' assert result.output == expected result = run_command(version, ['--short']) - assert result.output == f'{info.VERSION}\n' + assert result.output == 'test\n' diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 3b9fe1d0..319137ed 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -20,6 +20,7 @@ import pathlib import mock +from unittest.mock import MagicMock, patch import requests import logging @@ -302,10 +303,11 @@ def test_restore(mocked_g_config): ) backup_path = result.output.replace( 'Backup archive successfully created: ', '').replace('\n', '') - with mock.patch('subprocess.run', new=subprocess_run_mock), \ - mock.patch('node_cli.core.node.restore_op'), \ - mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ - mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False): + + with patch('node_cli.core.node.restore_op', MagicMock()) as mock_restore_op, \ + patch('subprocess.run', new=subprocess_run_mock), \ + patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ + patch('node_cli.utils.decorators.is_node_inited', return_value=False): result = run_command( restore_node, [backup_path, './tests/test-env'] @@ -313,6 +315,31 @@ def test_restore(mocked_g_config): assert result.exit_code == 0 assert 'Node is restored from backup\n' in result.output # noqa + assert mock_restore_op.call_args[0][0].get('BACKUP_RUN') == 'True' + + +def test_restore_no_snapshot(mocked_g_config): + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + result = run_command( + backup_node, + ['/tmp'] + ) + backup_path = result.output.replace( + 'Backup archive successfully created: ', '').replace('\n', '') + + with patch('node_cli.core.node.restore_op', MagicMock()) as mock_restore_op, \ + patch('subprocess.run', new=subprocess_run_mock), \ + patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), \ + patch('node_cli.utils.decorators.is_node_inited', return_value=False): + result = run_command( + restore_node, + [backup_path, './tests/test-env', '--no-snapshot'] + ) + assert result.exit_code == 0 + assert 'Node is restored from backup\n' in result.output # noqa + + assert mock_restore_op.call_args[0][0].get('BACKUP_RUN') is None + def test_maintenance_on(): resp_mock = response_mock( diff --git a/tests/conftest.py b/tests/conftest.py index 70aada8f..13b8ca65 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,7 +39,7 @@ from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.utils.global_config import generate_g_config_file -from tests.helper import TEST_META_V1, TEST_META_V2 +from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3 TEST_ENV_PARAMS = """ @@ -221,6 +221,16 @@ def meta_file_v2(): os.remove(META_FILEPATH) +@pytest.fixture +def meta_file_v3(): + with open(META_FILEPATH, 'w') as f: + json.dump(TEST_META_V3, f) + try: + yield META_FILEPATH + finally: + os.remove(META_FILEPATH) + + @pytest.fixture def ensure_meta_removed(): try: diff --git a/tests/helper.py b/tests/helper.py index 16290b8d..b065b397 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -34,6 +34,14 @@ } +TEST_META_V3 = { + 'version': '0.1.1', + 'config_stream': 'develop', + 'docker_lvmpy_stream': '1.1.2', + 'os_id': 'ubuntu', + 'os_version': '18.04' +} + def response_mock(status_code=0, json_data=None, headers=None, raw=None): diff --git a/tests/tools_meta_test.py b/tests/tools_meta_test.py index 95ab7658..431533db 100644 --- a/tests/tools_meta_test.py +++ b/tests/tools_meta_test.py @@ -7,7 +7,7 @@ ensure_meta, get_meta_info, save_meta, update_meta ) -from tests.helper import TEST_META_V1, TEST_META_V2 +from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3 def test_get_meta_info_v1(meta_file_v1): @@ -24,6 +24,15 @@ def test_get_meta_info_v2(meta_file_v2): assert meta.docker_lvmpy_stream == TEST_META_V2['docker_lvmpy_stream'] +def test_get_meta_info_v3(meta_file_v3): + meta = get_meta_info() + assert meta.version == TEST_META_V3['version'] + assert meta.config_stream == TEST_META_V3['config_stream'] + assert meta.docker_lvmpy_stream == TEST_META_V3['docker_lvmpy_stream'] + assert meta.os_id == TEST_META_V3['os_id'] + assert meta.os_version == TEST_META_V3['os_version'] + + def test_get_meta_info_empty(): meta = get_meta_info() assert meta is None @@ -34,6 +43,8 @@ def test_compose_default_meta(): assert meta.version == '1.0.0' assert meta.config_stream == '1.1.0' assert meta.docker_lvmpy_stream == '1.0.0' + assert meta.os_id == 'ubuntu' + assert meta.os_version == '18.04' def test_save_meta(meta_file_v2): @@ -44,28 +55,45 @@ def test_save_meta(meta_file_v2): assert saved_json == { 'version': '1.1.2', 'config_stream': '2.2.2', - 'docker_lvmpy_stream': '1.0.0' + 'docker_lvmpy_stream': '1.0.0', + 'os_id': 'ubuntu', + 'os_version': '18.04', } -def test_update_meta(meta_file_v2): +def test_update_meta_from_v2_to_v3(meta_file_v2): old_meta = get_meta_info() update_meta(version='3.3.3', config_stream='1.1.1', - docker_lvmpy_stream='1.2.2') + docker_lvmpy_stream='1.2.2', os_id='debian', os_version='11') meta = get_meta_info() assert meta.version == '3.3.3' assert meta.config_stream == '1.1.1' assert meta.docker_lvmpy_stream == '1.2.2' + assert meta.os_id == 'debian' + assert meta.os_version == '11' assert meta != old_meta def test_update_meta_from_v1(meta_file_v1): update_meta(version='4.4.4', config_stream='beta', - docker_lvmpy_stream='1.3.3') + docker_lvmpy_stream='1.3.3', os_id='debian', os_version='11') meta = get_meta_info() assert meta.version == '4.4.4' assert meta.config_stream == 'beta' assert meta.docker_lvmpy_stream == '1.3.3' + assert meta.os_id == 'debian' + assert meta.os_version == '11' + + +def test_update_meta_from_v3(meta_file_v3): + update_meta(version='5.5.5', config_stream='stable', + docker_lvmpy_stream='1.2.3', os_id='ubuntu', os_version='20.04') + meta = get_meta_info() + assert meta.version == '5.5.5' + assert meta.config_stream == 'stable' + assert meta.docker_lvmpy_stream == '1.2.3' + assert meta.os_id == 'ubuntu' + assert meta.os_version == '20.04' def test_ensure_meta(ensure_meta_removed): diff --git a/text.yml b/text.yml index d1a660fb..2c074a6f 100644 --- a/text.yml +++ b/text.yml @@ -59,3 +59,12 @@ exit: in_progress: "Node exiting is in progress" wait_for_rotations: "Node is waiting to finish rotations" completed: "Node exiting is completed" + +lvmpy: + help: Lvmpy commands + run: + help: Run lvmpy http server + prompt: Are you sure you want to run lvmpy server? + heal: + help: Run healing procedure for lvmpy server + prompt: Are you sure you want run healing procedure?