Skip to content

Commit c7b3611

Browse files
authored
Merge pull request #636 from skalenetwork/no-volume-limits
Create btrfs volume without lvmpy for syncnode
2 parents a690b1e + 34bc52d commit c7b3611

File tree

8 files changed

+207
-88
lines changed

8 files changed

+207
-88
lines changed

node_cli/configs/env.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
'DEFAULT_GAS_PRICE_WEI': '',
4545
'DISABLE_IMA': '',
4646
'SKIP_DOCKER_CONFIG': '',
47+
'ENFORCE_BTRFS': '',
4748
'SKIP_DOCKER_CLEANUP': ''
4849
}
4950

node_cli/core/host.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
SCHAINS_DATA_PATH, LOG_PATH,
3434
REMOVED_CONTAINERS_FOLDER_PATH,
3535
IMA_CONTRACTS_FILEPATH, MANAGER_CONTRACTS_FILEPATH,
36-
SKALE_RUN_DIR, SKALE_TMP_DIR
36+
SKALE_RUN_DIR, SKALE_STATE_DIR, SKALE_TMP_DIR
3737
)
3838
from node_cli.configs.resource_allocation import (
3939
RESOURCE_ALLOCATION_FILEPATH
@@ -92,7 +92,7 @@ def make_dirs():
9292
REMOVED_CONTAINERS_FOLDER_PATH,
9393
SGX_CERTS_PATH, SCHAINS_DATA_PATH, LOG_PATH,
9494
REPORTS_PATH, REDIS_DATA_PATH,
95-
SKALE_RUN_DIR, SKALE_TMP_DIR
95+
SKALE_RUN_DIR, SKALE_STATE_DIR, SKALE_TMP_DIR
9696
):
9797
safe_mkdir(dir_path)
9898

node_cli/core/node.py

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,13 @@
5050
from node_cli.core.checks import run_checks as run_host_checks
5151
from node_cli.core.resources import update_resource_allocation
5252
from node_cli.operations import (
53-
update_op, init_op, turn_off_op, turn_on_op, restore_op, init_sync_op, update_sync_op
53+
update_op,
54+
init_op,
55+
turn_off_op,
56+
turn_on_op,
57+
restore_op,
58+
init_sync_op,
59+
update_sync_op
5460
)
5561
from node_cli.utils.print_formatters import (
5662
print_failed_requirements_checks, print_node_cmd_error, print_node_info
@@ -202,9 +208,11 @@ def restore(backup_path, env_filepath):
202208

203209
def get_node_env(env_filepath, inited_node=False, sync_schains=None, sync_node=False):
204210
if env_filepath is not None:
205-
env_params = extract_env_params(env_filepath, sync_node=sync_node)
206-
if env_params is None:
207-
return
211+
env_params = extract_env_params(
212+
env_filepath,
213+
sync_node=sync_node,
214+
raise_for_status=True
215+
)
208216
save_env_params(env_filepath)
209217
else:
210218
env_params = extract_env_params(INIT_ENV_FILEPATH, sync_node=sync_node)

node_cli/operations/base.py

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,12 @@
3333
backup_old_contracts, download_contracts, configure_filebeat,
3434
configure_flask, unpack_backup_archive
3535
)
36-
from node_cli.operations.docker_lvmpy import docker_lvmpy_update, docker_lvmpy_install
36+
from node_cli.operations.volume import (
37+
cleanup_volume_artifacts,
38+
docker_lvmpy_update,
39+
docker_lvmpy_install,
40+
prepare_block_device
41+
)
3742
from node_cli.operations.skale_node import download_skale_node, sync_skale_node, update_images
3843
from node_cli.core.checks import CheckType, run_checks as run_host_checks
3944
from node_cli.core.iptables import configure_iptables
@@ -163,8 +168,9 @@ def init(env_filepath: str, env: str) -> bool:
163168

164169

165170
def init_sync(env_filepath: str, env: str) -> bool:
171+
cleanup_volume_artifacts(env['DISK_MOUNTPOINT'])
166172
download_skale_node(
167-
env['CONTAINER_CONFIGS_STREAM'],
173+
env.get('CONTAINER_CONFIGS_STREAM'),
168174
env.get('CONTAINER_CONFIGS_DIR')
169175
)
170176
sync_skale_node()
@@ -180,7 +186,10 @@ def init_sync(env_filepath: str, env: str) -> bool:
180186
download_contracts(env)
181187

182188
generate_nginx_config()
183-
docker_lvmpy_install(env)
189+
prepare_block_device(
190+
env['DISK_MOUNTPOINT'],
191+
force=env['ENFORCE_BTRFS'] == 'True'
192+
)
184193

185194
update_meta(
186195
VERSION,
@@ -196,7 +205,7 @@ def init_sync(env_filepath: str, env: str) -> bool:
196205
def update_sync(env_filepath: str, env: Dict) -> None:
197206
compose_rm(env, sync_node=True)
198207
remove_dynamic_containers()
199-
208+
cleanup_volume_artifacts(env['DISK_MOUNTPOINT'])
200209
download_skale_node(
201210
env['CONTAINER_CONFIGS_STREAM'],
202211
env.get('CONTAINER_CONFIGS_DIR')
@@ -209,7 +218,10 @@ def update_sync(env_filepath: str, env: Dict) -> None:
209218
backup_old_contracts()
210219
download_contracts(env)
211220

212-
docker_lvmpy_update(env)
221+
prepare_block_device(
222+
env['DISK_MOUNTPOINT'],
223+
force=env['ENFORCE_BTRFS'] == 'True'
224+
)
213225
generate_nginx_config()
214226

215227
prepare_host(

node_cli/operations/docker_lvmpy.py

Lines changed: 0 additions & 76 deletions
This file was deleted.

node_cli/operations/volume.py

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# This file is part of node-cli
4+
#
5+
# Copyright (C) 2021 SKALE Labs
6+
#
7+
# This program is free software: you can redistribute it and/or modify
8+
# it under the terms of the GNU Affero General Public License as published by
9+
# the Free Software Foundation, either version 3 of the License, or
10+
# (at your option) any later version.
11+
#
12+
# This program is distributed in the hope that it will be useful,
13+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
14+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15+
# GNU Affero General Public License for more details.
16+
#
17+
# You should have received a copy of the GNU Affero General Public License
18+
# along with this program. If not, see <https://www.gnu.org/licenses/>.
19+
20+
import glob
21+
import logging
22+
import os
23+
import shutil
24+
import tempfile
25+
26+
from node_cli.utils.helper import run_cmd
27+
from node_cli.utils.git_utils import sync_repo
28+
from node_cli.configs import (
29+
DOCKER_LVMPY_PATH,
30+
DOCKER_LVMPY_REPO_URL,
31+
FILESTORAGE_MAPPING,
32+
SCHAINS_MNT_DIR,
33+
SKALE_STATE_DIR
34+
)
35+
36+
logger = logging.getLogger(__name__)
37+
38+
39+
class FilesystemExistsError(Exception):
40+
pass
41+
42+
43+
def update_docker_lvmpy_env(env):
44+
env['PHYSICAL_VOLUME'] = env['DISK_MOUNTPOINT']
45+
env['VOLUME_GROUP'] = 'schains'
46+
env['FILESTORAGE_MAPPING'] = FILESTORAGE_MAPPING
47+
env['SCHAINS_MNT_DIR'] = SCHAINS_MNT_DIR
48+
env['PATH'] = os.environ.get('PATH', None)
49+
return env
50+
51+
52+
def ensure_filestorage_mapping(mapping_dir=FILESTORAGE_MAPPING):
53+
if not os.path.isdir(FILESTORAGE_MAPPING):
54+
os.makedirs(FILESTORAGE_MAPPING)
55+
56+
57+
def sync_docker_lvmpy_repo(env):
58+
if os.path.isdir(DOCKER_LVMPY_PATH):
59+
shutil.rmtree(DOCKER_LVMPY_PATH)
60+
sync_repo(
61+
DOCKER_LVMPY_REPO_URL,
62+
DOCKER_LVMPY_PATH,
63+
env["DOCKER_LVMPY_STREAM"]
64+
)
65+
66+
67+
def docker_lvmpy_update(env):
68+
sync_docker_lvmpy_repo(env)
69+
ensure_filestorage_mapping()
70+
logger.info('Running docker-lvmpy update script')
71+
update_docker_lvmpy_env(env)
72+
run_cmd(
73+
cmd=f'sudo -H -E {DOCKER_LVMPY_PATH}/scripts/update.sh'.split(),
74+
env=env
75+
)
76+
logger.info('docker-lvmpy update done')
77+
78+
79+
def docker_lvmpy_install(env):
80+
sync_docker_lvmpy_repo(env)
81+
ensure_filestorage_mapping()
82+
update_docker_lvmpy_env(env)
83+
run_cmd(
84+
cmd=f'sudo -H -E {DOCKER_LVMPY_PATH}/scripts/install.sh'.split(),
85+
env=env
86+
)
87+
logger.info('docker-lvmpy installed')
88+
89+
90+
def block_device_mounted(block_device):
91+
with open('/proc/mounts') as mounts:
92+
return any(block_device in mount for mount in mounts.readlines())
93+
94+
95+
def ensure_not_mounted(block_device):
96+
logger.info('Making sure %s is not mounted', block_device)
97+
if block_device_mounted(block_device):
98+
run_cmd(['umount', block_device])
99+
100+
101+
def cleanup_static_path(filestorage_mapping=FILESTORAGE_MAPPING):
102+
logger.info('Removing all links from filestorage mapping')
103+
for dir_link in glob.glob(os.path.join(filestorage_mapping, '*')):
104+
logger.debug('Unlinking %s', dir_link)
105+
if os.path.islink(dir_link):
106+
os.unlink(dir_link)
107+
108+
109+
def cleanup_volume_artifacts(block_device):
110+
ensure_not_mounted(block_device)
111+
cleanup_static_path()
112+
113+
114+
def get_block_device_filesystem(block_device):
115+
r = run_cmd(['blkid', '-o', 'udev', block_device])
116+
output = r.stdout.decode('utf-8')
117+
logger.debug('blkid output %s', output)
118+
fs_line = next(filter(lambda s: s.startswith('ID_FS_TYPE'), output.split('\n')))
119+
return fs_line.split('=')[1]
120+
121+
122+
def format_as_btrfs(block_device):
123+
logger.info('Formating %s as btrfs', block_device)
124+
run_cmd(['mkfs.btrfs', block_device, '-f'])
125+
126+
127+
def mount_device(block_device, mountpoint):
128+
os.makedirs(mountpoint, exist_ok=True)
129+
logger.info('Mounting %s device', block_device)
130+
run_cmd(['mount', block_device, mountpoint])
131+
132+
133+
def prepare_block_device(block_device, force=False):
134+
filesystem = None
135+
try:
136+
filesystem = get_block_device_filesystem(block_device)
137+
except Exception as e:
138+
logger.info('Cannot get filesystem type %s', e)
139+
logger.debug('Cannot get filesystem type', exc_info=True)
140+
if not force:
141+
raise
142+
143+
if filesystem == 'btrfs':
144+
logger.info('%s already formatted as btrfs', block_device)
145+
ensure_btrfs_for_all_space(block_device)
146+
else:
147+
logger.info('%s contains %s filesystem', block_device, filesystem)
148+
format_as_btrfs(block_device)
149+
mountpoint = os.path.join(SKALE_STATE_DIR, 'schains')
150+
mount_device(block_device, mountpoint)
151+
152+
153+
def max_resize_btrfs(path):
154+
run_cmd(['btrfs', 'filesystem', 'resize', 'max', path])
155+
156+
157+
def ensure_btrfs_for_all_space(block_device):
158+
with tempfile.TemporaryDirectory(dir=SKALE_STATE_DIR) as mountpoint:
159+
try:
160+
mount_device(block_device, mountpoint)
161+
logger.info('Resizing btrfs filesystem for %s', block_device)
162+
max_resize_btrfs(mountpoint)
163+
finally:
164+
ensure_not_mounted(block_device)

node_cli/utils/helper.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,10 @@
7171
}
7272

7373

74+
class InvalidEnvFileError(Exception):
75+
pass
76+
77+
7478
def read_json(path):
7579
with open(path, encoding='utf-8') as data_file:
7680
return json.loads(data_file.read())
@@ -141,7 +145,7 @@ def get_username():
141145
return os.environ.get('USERNAME') or os.environ.get('USER')
142146

143147

144-
def extract_env_params(env_filepath, sync_node=False):
148+
def extract_env_params(env_filepath, sync_node=False, raise_for_status=True):
145149
env_params = get_env_config(env_filepath, sync_node=sync_node)
146150
absent_params = ', '.join(absent_env_params(env_params))
147151
if absent_params:
@@ -150,6 +154,8 @@ def extract_env_params(env_filepath, sync_node=False):
150154
f"You should specify them to make sure that "
151155
f"all services are working",
152156
err=True)
157+
if raise_for_status:
158+
raise InvalidEnvFileError(f'Missing params: {absent_params}')
153159
return None
154160
return env_params
155161

tests/helper.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,13 @@
1818
# along with this program. If not, see <https://www.gnu.org/licenses/>.
1919

2020
import mock
21+
import os
22+
2123
from click.testing import CliRunner
2224
from mock import Mock, MagicMock
2325

26+
BLOCK_DEVICE = os.getenv('BLOCK_DEVICE')
27+
2428

2529
def response_mock(status_code=0, json_data=None,
2630
headers=None, raw=None):

0 commit comments

Comments
 (0)