Merge branch 'beta' into beta-to-stable

VERSION

@@ -1 +1 @@
-2.4.0
+2.5.0

core/schains/checks.py

@@ -36,11 +36,12 @@
 from core.schains.config.main import schain_config_version_match
 from core.schains.dkg.utils import get_secret_key_share_filepath
 from core.schains.firewall.types import IRuleController
+from core.schains.ima import get_migration_ts as get_ima_migration_ts
 from core.schains.process_manager_helper import is_monitor_process_alive
 from core.schains.rpc import (
     check_endpoint_alive, check_endpoint_blocks, get_endpoint_alive_check_timeout
 )
-from core.schains.runner import get_container_name
+from core.schains.runner import get_container_name, get_image_name, is_new_image_pulled
 from core.schains.skaled_exit_codes import SkaledExitCodes
 
 from tools.configs.containers import IMA_CONTAINER, SCHAIN_CONTAINER
@@ -160,8 +161,31 @@ def exit_code_ok(self) -> CheckRes:
     @property
     def ima_container(self) -> CheckRes:
         """Checks that IMA container is running"""
-        name = get_container_name(IMA_CONTAINER, self.name)
-        return CheckRes(self.dutils.is_container_running(name))
+        container_name = get_container_name(IMA_CONTAINER, self.name)
+        new_image_pulled = is_new_image_pulled(type=IMA_CONTAINER, dutils=self.dutils)
+
+        migration_ts = get_ima_migration_ts(self.name)
+        new = time.time() > migration_ts
+
+        container_running = self.dutils.is_container_running(container_name)
+
+        updated_image = False
+        if container_running:
+            expected_image = get_image_name(type=IMA_CONTAINER, new=new)
+            image = self.dutils.get_container_image_name(container_name)
+            updated_image = image == expected_image
+
+        data = {
+            'container_running': container_running,
+            'updated_image': updated_image,
+            'new_image_pulled': new_image_pulled
+        }
+        logger.debug(
+            '%s, IMA check - %s',
+            self.name, data
+        )
+        result: bool = container_running and updated_image and new_image_pulled
+        return CheckRes(result, data=data)
 
     @property
     def rpc(self) -> CheckRes:

core/schains/firewall/rule_controller.py

@@ -112,7 +112,8 @@ def internal_ports(self) -> Iterable[int]:
                 self.port_allocation.CATCHUP,
                 self.port_allocation.PROPOSAL,
                 self.port_allocation.BINARY_CONSENSUS,
-                self.port_allocation.ZMQ_BROADCAST
+                self.port_allocation.ZMQ_BROADCAST,
+                self.port_allocation.IMA_RPC
             )
         )
 

core/schains/ima.py

@@ -17,17 +17,20 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+import json
+import logging
+import os
 from dataclasses import dataclass
 
+from flask import g
+from skale.dataclasses.skaled_ports import SkaledPorts
+from websocket import create_connection
+
 from core.schains.config.directory import schain_config_dir
 from core.schains.config.helper import get_schain_ports, get_schain_config, get_chain_id
 from core.ima.schain import get_schain_ima_abi_filepath
-
-import json
-import logging
-import os
-from tools.configs import SGX_SSL_KEY_FILEPATH, SGX_SSL_CERT_FILEPATH, SGX_SERVER_URL
-from tools.configs.containers import CONTAINERS_INFO
+from tools.configs import ENV_TYPE, SGX_SSL_KEY_FILEPATH, SGX_SSL_CERT_FILEPATH, SGX_SERVER_URL
+from tools.configs.containers import CONTAINERS_INFO, IMA_MIGRATION_PATH
 from tools.configs.db import REDIS_URI
 from tools.configs.ima import (
     IMA_ENDPOINT,
@@ -37,8 +40,7 @@
 )
 from tools.configs.schains import SCHAINS_DIR_PATH
 from tools.configs.web3 import ABI_FILEPATH
-from flask import g
-from websocket import create_connection
+from tools.helper import safe_load_yml
 
 logger = logging.getLogger(__name__)
 
@@ -67,6 +69,7 @@ class ImaEnv:
 
     sgx_url: str
     ecdsa_key_name: str
+    bls_key_name: str
     sgx_ssl_key_path: str
     sgx_ssl_cert_path: str
     node_address: str
@@ -80,6 +83,8 @@ class ImaEnv:
 
     time_framing: int
 
+    rpc_port: int
+
     def to_dict(self):
         """Returns upper-case representation of the ImaEnv object"""
         return {
@@ -95,13 +100,15 @@ def to_dict(self):
             'NODES_COUNT': self.nodes_count,
             'SGX_URL': self.sgx_url,
             'ECDSA_KEY_NAME': self.ecdsa_key_name,
+            'BLS_KEY_NAME': self.bls_key_name,
             'SGX_SSL_KEY_PATH': self.sgx_ssl_key_path,
             'SGX_SSL_CERT_PATH': self.sgx_ssl_cert_path,
             'NODE_ADDRESS': self.node_address,
             'TM_URL_MAIN_NET': self.tm_url_mainnet,
             'CID_MAIN_NET': self.cid_main_net,
             'CID_SCHAIN': self.cid_schain,
             'MONITORING_PORT': self.monitoring_port,
+            'RPC_PORT': self.rpc_port,
             'TIME_FRAMING': self.time_framing
         }
 
@@ -134,6 +141,7 @@ def schain_index_to_node_number(node):
 def get_ima_env(schain_name: str, mainnet_chain_id: int) -> ImaEnv:
     schain_config = get_schain_config(schain_name)
     node_info = schain_config["skaleConfig"]["nodeInfo"]
+    bls_key_name = node_info['wallets']['ima']['keyShareName']
     schain_nodes = schain_config["skaleConfig"]["sChain"]
     public_node_info = get_current_node_from_nodes(node_info['nodeID'], schain_nodes)
 
@@ -155,13 +163,15 @@ def get_ima_env(schain_name: str, mainnet_chain_id: int) -> ImaEnv:
         nodes_count=len(schain_nodes['nodes']),
         sgx_url=SGX_SERVER_URL,
         ecdsa_key_name=node_info['ecdsaKeyName'],
+        bls_key_name=bls_key_name,
         sgx_ssl_key_path=SGX_SSL_KEY_FILEPATH,
         sgx_ssl_cert_path=SGX_SSL_CERT_FILEPATH,
         node_address=node_address,
         tm_url_mainnet=REDIS_URI,
         cid_main_net=mainnet_chain_id,
         cid_schain=schain_chain_id,
         monitoring_port=node_info['imaMonitoringPort'],
+        rpc_port=get_ima_rpc_port(schain_name),
         time_framing=IMA_TIME_FRAMING
     )
 
@@ -179,6 +189,12 @@ def get_ima_monitoring_port(schain_name):
         return None
 
 
+def get_ima_rpc_port(schain_name):
+    config = get_schain_config(schain_name)
+    base_port = config['skaleConfig']['nodeInfo']['basePort']
+    return base_port + SkaledPorts.IMA_RPC.value
+
+
 def get_ima_container_statuses():
     containers_list = g.docker_utils.get_all_ima_containers(all=True, format=True)
     ima_containers = [{'name': container['name'], 'state': container['state']['Status']}
@@ -244,3 +260,11 @@ def get_ima_log_checks():
                                                    'last_ima_errors': errors,
                                                    'error_categories': categories}})
     return all_ima_healthchecks
+
+
+def get_migration_schedule() -> dict:
+    return safe_load_yml(IMA_MIGRATION_PATH)[ENV_TYPE]
+
+
+def get_migration_ts(name: str) -> int:
+    return get_migration_schedule().get(name, 0)

core/schains/monitor/base_monitor.py

@@ -43,7 +43,10 @@
 from core.schains.monitor.containers import monitor_schain_container, monitor_ima_container
 from core.schains.monitor.rpc import handle_failed_schain_rpc
 from core.schains.runner import (
-    restart_container, is_container_exists, get_container_name
+    pull_new_image,
+    restart_container,
+    is_container_exists,
+    get_container_name
 )
 from core.schains.config import init_schain_config, init_schain_config_dir
 from core.schains.config.directory import get_schain_config
@@ -52,13 +55,13 @@
     get_node_ips_from_config,
     get_own_ip_from_config
 )
-from core.schains.ima import ImaData
+from core.schains.ima import get_migration_ts as get_ima_migration_ts, ImaData
 from core.schains.skaled_status import init_skaled_status
 
 from tools.docker_utils import DockerUtils
 from tools.notifications.messages import notify_checks, is_checks_passed
 from tools.str_formatters import arguments_list_string
-from tools.configs.containers import SCHAIN_CONTAINER
+from tools.configs.containers import IMA_CONTAINER, SCHAIN_CONTAINER
 
 from web.models.schain import upsert_schain_record, set_first_run, SChainRecord
 
@@ -310,10 +313,14 @@ def skaled_rpc(self) -> bool:
     @monitor_block
     def ima_container(self) -> bool:
         initial_status = self.checks.ima_container.status
+        migration_ts = get_ima_migration_ts(self.name)
+        logger.debug('Migration time for %s IMA - %d', self.name, migration_ts)
         if not initial_status:
+            pull_new_image(type=IMA_CONTAINER, dutils=self.dutils)
             monitor_ima_container(
                 self.schain,
                 self.ima_data,
+                migration_ts=migration_ts,
                 dutils=self.dutils
             )
         else:

core/schains/monitor/containers.py

@@ -18,14 +18,18 @@
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import logging
+import time
 
 from core.schains.volume import is_volume_exists
 from core.schains.runner import (
+    get_container_image,
+    get_image_name,
+    is_container_exists,
     is_schain_container_failed,
+    remove_container,
     restart_container,
-    run_schain_container,
-    is_container_exists,
-    run_ima_container
+    run_ima_container,
+    run_schain_container
 )
 from core.ima.schain import copy_schain_ima_abi
 from core.schains.ima import ImaData
@@ -99,6 +103,7 @@ def monitor_schain_container(
 def monitor_ima_container(
     schain: dict,
     ima_data: ImaData,
+    migration_ts: int = 0,
     dutils: DockerUtils = None
 ) -> None:
     schain_name = schain["name"]
@@ -113,12 +118,28 @@ def monitor_ima_container(
 
     copy_schain_ima_abi(schain_name)
 
-    if not is_container_exists(schain_name, container_type=IMA_CONTAINER, dutils=dutils):
-        logger.info(f'sChain {schain_name}: IMA container doesn\'t exits, creating...')
+    container_exists = is_container_exists(schain_name, container_type=IMA_CONTAINER, dutils=dutils)
+    container_image = get_container_image(schain_name, IMA_CONTAINER, dutils)
+    new_image = get_image_name(type=IMA_CONTAINER, new=True)
+
+    expected_image = get_image_name(type=IMA_CONTAINER)
+    logger.debug('%s IMA image %s, expected %s', schain_name, container_image, expected_image)
+
+    if time.time() > migration_ts:
+        logger.debug('%s IMA migration time passed', schain_name)
+        expected_image = new_image
+        if container_exists and expected_image != container_image:
+            logger.info('%s Removing old container as part of IMA migration', schain_name)
+            remove_container(schain_name, IMA_CONTAINER, dutils)
+            container_exists = False
+
+    if not container_exists:
+        logger.info('%s No IMA container, creating, image %s', schain_name, expected_image)
         run_ima_container(
             schain,
             ima_data.chain_id,
+            image=expected_image,
             dutils=dutils
         )
     else:
-        logger.warning(f'sChain {schain_name}: IMA container exists, but not running, skipping')
+        logger.debug('sChain %s: IMA container exists, but not running, skipping', schain_name)

core/schains/runner.py

@@ -17,8 +17,9 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-import logging
 import copy
+import logging
+
 from docker.types import LogConfig, Ulimit
 
 from core.schains.volume import get_schain_volume_config
@@ -55,6 +56,16 @@ def is_container_exists(
     return dutils.is_container_exists(container_name)
 
 
+def get_container_image(
+    schain_name,
+    container_type=SCHAIN_CONTAINER,
+    dutils=None
+):
+    dutils = dutils or DockerUtils()
+    container_name = get_container_name(container_type, schain_name)
+    return dutils.get_container_image_name(container_name)
+
+
 def is_container_running(
     schain_name,
     container_type=SCHAIN_CONTAINER,
@@ -65,9 +76,14 @@ def is_container_running(
     return dutils.is_container_running(container_name)
 
 
-def get_image_name(type):
+def get_image_name(type: str, new: bool = False) -> str:
+    tag_field = 'version'
+    if type == IMA_CONTAINER and new:
+        tag_field = 'new_version'
     container_info = CONTAINERS_INFO[type]
-    return f'{container_info["name"]}:{container_info["version"]}'
+    image_base_name = container_info['name']
+    tag = container_info[tag_field]
+    return f'{image_base_name}:{tag}'
 
 
 def get_container_name(type, schain_name):
@@ -104,13 +120,16 @@ def run_container(
     volume_config=None,
     cpu_shares_limit=None,
     mem_limit=None,
+    image=None,
     dutils=None,
     volume_mode=None
 ):
     dutils = dutils or DockerUtils()
-    image_name, container_name, run_args, custom_args = get_container_info(
+    default_image, container_name, run_args, custom_args = get_container_info(
         type, schain_name)
 
+    image_name = image or default_image
+
     add_config_volume(run_args, schain_name, mode=volume_mode)
 
     if custom_args.get('logs', None):
@@ -188,6 +207,7 @@ def run_schain_container(
 def run_ima_container(
     schain: dict,
     mainnet_chain_id: int,
+    image: str,
     dutils: DockerUtils = None
 ) -> None:
     dutils = dutils or DockerUtils()
@@ -203,6 +223,7 @@ def run_ima_container(
         env=env.to_dict(),
         cpu_shares_limit=cpu_limit,
         mem_limit=mem_limit,
+        image=image,
         dutils=dutils
     )
 
@@ -252,3 +273,20 @@ def is_schain_container_failed(
     if bad_state:
         logger.warning(f'{name} is in bad state - exited: {exited}, created: {created}')
     return bad_state
+
+
+def is_new_image_pulled(type: str, dutils: DockerUtils) -> bool:
+    image = get_image_name(type, new=True)
+    return dutils.pulled(image)
+
+
+def remove_container(schain_name: str, type: str, dutils: DockerUtils):
+    container = get_container_name(type=type, schain_name=schain_name)
+    dutils.safe_rm(container)
+
+
+def pull_new_image(type: str, dutils: DockerUtils) -> None:
+    image = get_image_name(type, new=True)
+    if not dutils.pulled(image):
+        logger.info('Pulling new image %s', image)
+        dutils.pull(image)

requirements-dev.txt

@@ -5,4 +5,4 @@ mock==4.0.2
 blinker==1.4
 
 pytest-cov==2.9.0
-codecov==2.1.9
+codecov==2.1.13