Merge pull request #723 from skalenetwork/beta

2.3.x stable (2.1.1 release)

.dockerignore

@@ -0,0 +1,6 @@
+tests
+helper-scripts
+dist
+build
+.github
+.gitmodules

.github/workflows/publish.yml

@@ -12,14 +12,17 @@ jobs:
   create_release:
     if: github.event.pull_request.merged
     name: Create release
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     outputs:
       upload_url: ${{ steps.create_release.outputs.upload_url }}
       version: ${{ steps.export_outputs.outputs.version }}
       branch: ${{ steps.export_outputs.outputs.branch }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
+        with:
+          submodules: true
+
       - name: Checkout submodules
         run: git submodule update --init
       - name: Install ubuntu dependencies
@@ -61,37 +64,53 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: ubuntu-18.04
+          - os: ubuntu-20.04
             asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.7
+    - name: Set up Python 3.11
       uses: actions/setup-python@v1
       with:
-        python-version: 3.7
+        python-version: 3.11
+
     - name: Install ubuntu dependencies
-      if: matrix.os == 'ubuntu-18.04'
+      if: matrix.os == 'ubuntu-20.04'
       run: |
         sudo apt-get update
-    - name: Install python dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -e .
-        pip install -e .[dev]
-        pip install wheel
-        pip install --upgrade 'setuptools<45.0.0'
+
     - name: Checkout submodules
       run: git submodule update --init
-    - name: Build library
+
+    - name: Build binary
+      run: |
+        mkdir ./dist
+        docker build . -t node-cli-builder
+        docker run  -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }}
+        ls -altr /home/ubuntu/dist/
+        docker rm -f $(docker ps -aq)
+
+    - name: Save sha512sum
       run: |
-        bash ./scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }}
-    - name: Upload Release Asset
+        sudo sha512sum /home/ubuntu/dist/${{ matrix.asset_name }} | sudo tee > /dev/null /home/ubuntu/dist/sha512sum
+
+    - name: Upload release binary
       id: upload-release-asset
       uses: actions/upload-release-asset@v1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./dist/${{ matrix.asset_name }}
+        asset_path: /home/ubuntu/dist/${{ matrix.asset_name }}
         asset_name: ${{ matrix.asset_name }}
         asset_content_type: application/octet-stream
+
+    - name: Upload release checksum
+      id: upload-release-checksum
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.create_release.outputs.upload_url }}
+        asset_path: /home/ubuntu/dist/sha512sum
+        asset_name: ${{ matrix.asset_name }}.sha512
+        asset_content_type: text/plain

.github/workflows/test.yml

@@ -3,32 +3,55 @@ on: [push, pull_request]
 
 jobs:
   test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.11]
     steps:
     - uses: actions/checkout@v2
+      with:
+        submodules: true
+
+    - name: Checkout submodules
+      run: git submodule update --init
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
+
     - name: Install ubuntu dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install python-setuptools
+        sudo apt-get install python-setuptools iptables
+
     - name: Install python dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -e .
         pip install -e .[dev]
         pip install --upgrade 'setuptools<45.0.0'
+
     - name: Lint with flake8
       run: |
         flake8 .
-    - name: Build binary
+
+    - name: Build binary in Ubuntu 18.04 environment
       run: |
-        bash scripts/build.sh 1.0.0 test-branch
-    - name: Run tests
+        mkdir ./dist
+        docker build . -t node-cli-builder
+        docker run  -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test
+        docker rm -f $(docker ps -aq)
+
+    - name: Check build
+      run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64
+
+    - name: Build binary in Ubuntu 20.04 environment
       run: |
-        bash ./scripts/run_tests.sh
+        scripts/build.sh test test
+
+    - name: Check build
+      run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64
+
+    - name: Run tests
+      run: bash ./scripts/run_tests.sh

.gitmodules

@@ -1,4 +1,9 @@
 [submodule "helper-scripts"]
 	path = helper-scripts
 	url = https://github.com/skalenetwork/helper-scripts.git
+    branch = develop
+
+[submodule "lvmpy"]
+	path = lvmpy
+	url = https://github.com/skalenetwork/docker-lvmpy
 	branch = develop

Dockerfile

@@ -0,0 +1,29 @@
+FROM ubuntu:18.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa
+RUN apt-get install -y  \
+                       git \
+                       python3.11 \
+                       libpython3.11-dev \
+                       python3.11-venv \
+                       python3.11-distutils \
+                       python3.11-dev \
+                       build-essential \
+                       zlib1g-dev \
+                       libssl-dev \
+                       libffi-dev \
+                       swig \
+                       iptables
+
+RUN mkdir /app
+WORKDIR /app
+
+COPY . .
+
+ENV PATH=/app/buildvenv/bin:$PATH
+RUN python3.11 -m venv /app/buildvenv && \
+    pip install --upgrade pip && \
+    pip install wheel setuptools==63.2.0 && \
+    pip install -e '.[dev]' 

main.spec

@@ -1,9 +1,5 @@
 # -*- mode: python -*-
 
-# import distutils
-# if distutils.distutils_path.endswith('__init__.py'):
-#    distutils.distutils_path = os.path.dirname(distutils.distutils_path)
-
 import importlib.util
 
 libxtwrapper_path = importlib.util.find_spec('libxtwrapper').origin

node_cli/cli/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '2.2.0'
+__version__ = '2.3.0'
 
 if __name__ == "__main__":
     print(__version__)

node_cli/cli/lvmpy.py

@@ -0,0 +1,62 @@
+#   -*- coding: utf-8 -*-
+#
+#   This file is part of node-cli
+#
+#   Copyright (C) 2020 SKALE Labs
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import click
+
+from node_cli.utils.helper import abort_if_false
+from node_cli.utils.texts import Texts
+from lvmpy.src.app import run as run_lvmpy
+from lvmpy.src.health import heal_service
+
+G_TEXTS = Texts()
+TEXTS = G_TEXTS['lvmpy']
+
+
+@click.group()
+def lvmpy_cli():
+    pass
+
+
+@lvmpy_cli.group('lvmpy', help=TEXTS['help'])
+def health():
+    pass
+
+
+@health.command(help=TEXTS['run']['help'])
+@click.option(
+    '--yes',
+    is_flag=True,
+    callback=abort_if_false,
+    expose_value=False,
+    prompt=TEXTS['run']['prompt']
+)
+def run():
+    run_lvmpy()
+
+
+@health.command(help=TEXTS['heal']['help'])
+@click.option(
+    '--yes',
+    is_flag=True,
+    callback=abort_if_false,
+    expose_value=False,
+    prompt=TEXTS['heal']['prompt']
+)
+def heal():
+    heal_service()

node_cli/cli/node.py

@@ -131,9 +131,15 @@ def backup_node(backup_folder_path):
 @node.command('restore', help="Restore SKALE node on another machine")
 @click.argument('backup_path')
 @click.argument('env_file')
+@click.option(
+    '--no-snapshot',
+    help='Do not restore sChains from snapshot',
+    is_flag=True,
+    hidden=True
+)
 @streamed_cmd
-def restore_node(backup_path, env_file):
-    restore(backup_path, env_file)
+def restore_node(backup_path, env_file, no_snapshot):
+    restore(backup_path, env_file, no_snapshot)
 
 
 @node.command('maintenance-on', help="Set SKALE node into maintenance mode")

node_cli/configs/__init__.py

@@ -35,6 +35,7 @@
 FILESTORAGE_MAPPING = os.path.join(SKALE_STATE_DIR, 'filestorage')
 SNAPSHOTS_SHARED_VOLUME = 'shared-space'
 SCHAINS_MNT_DIR = '/mnt'
+VOLUME_GROUP = 'schains'
 
 SKALE_DIR = os.path.join(G_CONF_HOME, '.skale')
 SKALE_TMP_DIR = os.path.join(SKALE_DIR, '.tmp')
@@ -73,6 +74,16 @@
 FILEBEAT_CONFIG_PATH = os.path.join(NODE_DATA_PATH, 'filebeat.yml')
 
 DOCKER_LVMPY_PATH = os.path.join(SKALE_DIR, 'docker-lvmpy')
+DOCKER_LVMPY_BIN_PATH = '/usr/local/bin/skale'
+
+LVMPY_CMD = f'{DOCKER_LVMPY_BIN_PATH} lvmpy'
+LVMPY_RUN_CMD = f'{LVMPY_CMD} run --yes'
+LVMPY_HEAL_CMD = f'{LVMPY_CMD} heal --yes'
+
+LVMPY_CRON_LOG_PATH = '/var/log/docker-lvmpy/cron.log'
+LVMPY_CRON_SCHEDULE_MINUTES = 3
+
+LVMPY_LOG_DIR = '/var/log/docker-lvmpy'
 
 IPTABLES_DIR = '/etc/iptables/'
 IPTABLES_RULES_STATE_FILEPATH = os.path.join(IPTABLES_DIR, 'rules.v4')

node_cli/core/node.py

@@ -143,13 +143,17 @@ def init(env_filepath):
 
 
 @check_not_inited
-def restore(backup_path, env_filepath):
+def restore(backup_path, env_filepath, no_snapshot=False):
     env = get_node_env(env_filepath)
     if env is None:
         return
     save_env_params(env_filepath)
     env['SKALE_DIR'] = SKALE_DIR
-    env['BACKUP_RUN'] = 'True'  # should be str
+
+    if not no_snapshot:
+        logger.info('Adding BACKUP_RUN to env ...')
+        env['BACKUP_RUN'] = 'True'  # should be str
+
     restored_ok = restore_op(env, backup_path)
     if not restored_ok:
         error_exit(

node_cli/main.py

@@ -29,6 +29,7 @@
 from node_cli.cli.health import health_cli
 from node_cli.cli.info import BUILD_DATETIME, COMMIT, BRANCH, OS, VERSION
 from node_cli.cli.logs import logs_cli
+from node_cli.cli.lvmpy import lvmpy_cli
 from node_cli.cli.node import node_cli
 from node_cli.cli.schains import schains_cli
 from node_cli.cli.wallet import wallet_cli
@@ -97,6 +98,7 @@ def handle_exception(exc_type, exc_value, exc_traceback):
             health_cli,
             schains_cli,
             logs_cli,
+            lvmpy_cli,
             resources_allocation_cli,
             node_cli,
             wallet_cli,