From 71bff33f602940f7c39d8d41568b96149288f7ed Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Thu, 15 Feb 2024 17:00:36 +0100 Subject: [PATCH 01/28] CTX-5430: Created new configuration for user, saving changes... --- coretex/configuration/node.py | 6 ++ coretex/configuration/user.py | 174 ++++++++++++++++++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 coretex/configuration/node.py create mode 100644 coretex/configuration/user.py diff --git a/coretex/configuration/node.py b/coretex/configuration/node.py new file mode 100644 index 00000000..62c6a8e6 --- /dev/null +++ b/coretex/configuration/node.py @@ -0,0 +1,6 @@ +def initializeConfig() + +class NodeConfiguration: + + def __init__(self) -> None: + self._raw = initializeConfig() \ No newline at end of file diff --git a/coretex/configuration/user.py b/coretex/configuration/user.py new file mode 100644 index 00000000..93d49d5a --- /dev/null +++ b/coretex/configuration/user.py @@ -0,0 +1,174 @@ +from typing import Dict, Any, Optional, TypeVar +from pathlib import Path +from datetime import datetime, timezone + +import os +import json + +from ..utils import decodeDate +from ..configuration import CONFIG_DIR + + +PropertyType = TypeVar("PropertyType") + + +USER_CONFIG_PATH = CONFIG_DIR / "user_config.json" + + +class InvalidUserConfiguration(Exception): + pass + + +def initializeConfig() -> Dict[str, Any]: + config = { + "username": os.environ.get("CTX_USERNAME"), + "password": os.environ.get("CTX_PASSWORD"), + "token": None, + "refreshToken": None, + "tokenExpirationDate": None, + "refreshTokenExpirationDate": None, + "serverUrl": os.environ.get("CTX_API_URL", "https://api.coretex.ai/"), + "projectId": os.environ.get("CTX_PROJECT_ID") + } + + if not USER_CONFIG_PATH.exists(): + with USER_CONFIG_PATH.open("w") as configFile: + json.dump(config, configFile, indent = 4) + else: + with open(USER_CONFIG_PATH, "r") as file: + config = json.load(file) + + if not isinstance(config, dict): + raise InvalidUserConfiguration(f"Invalid config type \"{type(config)}\", expected: \"dict\".") + + return config + + +def hasExpired(tokenExpirationDate: Optional[str]) -> bool: + if tokenExpirationDate is None: + return False + + currentDate = datetime.utcnow().replace(tzinfo = timezone.utc) + return currentDate >= decodeDate(tokenExpirationDate) + + +class UserConfiguration: + + def __init__(self) -> None: + self._raw = initializeConfig() + + @property + def username(self) -> str: + return self.getStrValue("username", "CTX_USERNAME") + + @username.setter + def username(self, value: Optional[str]) -> None: + self._raw["username"] = value + + @property + def password(self) -> str: + return self.getStrValue("password", "CTX_PASSWORD") + + @password.setter + def password(self, value: Optional[str]) -> None: + self._raw["password"] = value + + @property + def token(self) -> str: + return self.getStrValue("token") + + @token.setter + def token(self, value: Optional[str]) -> None: + self._raw["token"] = value + + @property + def refreshToken(self) -> str: + return self.getStrValue("refreshToken") + + @refreshToken.setter + def refreshToken(self, value: Optional[str]) -> None: + self._raw["refreshToken"] = value + + @property + def tokenExpirationDate(self) -> str: + return self.getStrValue("tokenExpirationDate") + + @tokenExpirationDate.setter + def tokenExpirationDate(self, value: Optional[str]) -> None: + self._raw["tokenExpirationDate"] = value + + @property + def refreshTokenExpirationDate(self) -> str: + return self.getStrValue("refreshTokenExpirationDate") + + @refreshTokenExpirationDate.setter + def refreshTokenExpirationDate(self, value: Optional[str]) -> None: + self._raw["refreshTokenExpirationDate"] = value + + @property + def serverUrl(self) -> str: + return self.getStrValue("serverUrl", "CTX_API_URL") + + @serverUrl.setter + def serverUrl(self, value: Optional[str]) -> None: + self._raw["serverUrl"] = value + + @property + def projectId(self) -> int: + return self.getIntValue("projectId", "CTX_PROJECT_ID") + + @projectId.setter + def projectId(self, value: Optional[int]) -> None: + self._raw["projectId"] = value + + @property + def isValid(self) -> bool: + return self._raw.get("username") is not None and self._raw.get("password") is not None + + @property + def hasTokenExpired(self) -> bool: + if self._raw.get("token") is None: + return True + + tokenExpirationDate = self._raw.get("tokenExpirationDate") + if tokenExpirationDate is None: + return True + + return hasExpired(tokenExpirationDate) + + @property + def hasRefreshTokenExpired(self) -> bool: + if self._raw.get("refreshToken") is None: + return True + + refreshTokenExpirationDate = self._raw.get("refreshTokenExpirationDate") + if refreshTokenExpirationDate is None: + return True + + return hasExpired(refreshTokenExpirationDate) + + def save(self) -> None: + with USER_CONFIG_PATH.open("w") as configFile: + json.dump(self.__dict__, configFile, indent = 4) + + def getStrValue(self, configKey: str, envKey: Optional[str] = None) -> str: + if envKey is not None and envKey in os.environ: + return os.environ[envKey] + + value = self._raw.get(configKey) + + if not isinstance(value, str): + raise InvalidUserConfiguration(f"Invalid username type \"{type(value)}\", expected: \"str\".") + + return value + + def getIntValue(self, configKey: str, envKey: Optional[str] = None) -> int: + if envKey is not None and envKey in os.environ: + return int(os.environ[envKey]) + + value = self._raw.get(configKey) + + if not isinstance(value, int): + raise InvalidUserConfiguration(f"Invalid username type \"{type(value)}\", expected: \"str\".") + + return value From 3b9c70dabc5986e99972db3e4999c706915ccf00 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 27 Mar 2024 13:44:32 +0100 Subject: [PATCH 02/28] CTX-5430: Saving changes related to configuration refactor. --- coretex/configuration/__init__.py | 0 coretex/configuration/base.py | 2 + coretex/configuration/node.py | 174 +++++++++++++++++++++++++++++- coretex/configuration/user.py | 10 +- 4 files changed, 177 insertions(+), 9 deletions(-) create mode 100644 coretex/configuration/__init__.py create mode 100644 coretex/configuration/base.py diff --git a/coretex/configuration/__init__.py b/coretex/configuration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/coretex/configuration/base.py b/coretex/configuration/base.py new file mode 100644 index 00000000..efcceb97 --- /dev/null +++ b/coretex/configuration/base.py @@ -0,0 +1,2 @@ +class BaseConfiguration: + pass diff --git a/coretex/configuration/node.py b/coretex/configuration/node.py index 62c6a8e6..fde895bc 100644 --- a/coretex/configuration/node.py +++ b/coretex/configuration/node.py @@ -1,6 +1,176 @@ -def initializeConfig() +from typing import Dict, Any, Optional + +import os +import json + +from ..configuration import CONFIG_DIR + + +NODE_CONFIG_PATH = CONFIG_DIR / "node_config.json" + + +class InvalidNodeConfiguration(Exception): + pass + + +def initializeConfig() -> Dict[str, Any]: + config = { + "nodeName": os.environ.get("CTX_NODE_NAME"), + "nodeAccessToken": None, + "image": "coretexai/coretex-node:latest-cpu", + "allowGpu": False, + "nodeRam": None, + "nodeSharedMemory": None, + "cpuCount": None, + "nodeMode": None, + "allowDocker": False, + "secretsKey": None, + "initScript": None + } + + if not NODE_CONFIG_PATH.exists(): + with NODE_CONFIG_PATH.open("w") as configFile: + json.dump(config, configFile, indent = 4) + else: + with open(NODE_CONFIG_PATH, "r") as file: + config = json.load(file) + + if not isinstance(config, dict): + raise InvalidNodeConfiguration(f"Invalid config type \"{type(config)}\", expected: \"dict\".") + + return config + class NodeConfiguration: def __init__(self) -> None: - self._raw = initializeConfig() \ No newline at end of file + self._raw = initializeConfig() + + @property + def nodeName(self) -> str: + return self.getStrValue("nodeName", "CTX_NODE_NAME") + + @nodeName.setter + def nodeName(self, value: Optional[str]) -> None: + self._raw["nodeName"] = value + + @property + def nodeAccessToken(self) -> str: + return self.getStrValue("nodeAccessToken") + + @nodeAccessToken.setter + def nodeAccessToken(self, value: Optional[str]) -> None: + self._raw["nodeAccessToken"] = value + + @property + def image(self) -> str: + return self.getStrValue("image", "CTX_NODE_NAME") + + @image.setter + def image(self, value: Optional[str]) -> None: + self._raw["image"] = value + + @property + def allowGpu(self) -> str: + return self.getStrValue("allowGpu") + + @allowGpu.setter + def allowGpu(self, value: Optional[bool]) -> None: + self._raw["allowGpu"] = value + + @property + def nodeRam(self) -> int: + return self.getIntValue("nodeRam") + + @nodeRam.setter + def nodeRam(self, value: Optional[int]) -> None: + self._raw["nodeRam"] = value + + @property + def nodeSwap(self) -> int: + return self.getIntValue("nodeSwap") + + @nodeSwap.setter + def nodeSwap(self, value: Optional[int]) -> None: + self._raw["nodeSwap"] = value + + @property + def nodeSharedMemory(self) -> int: + return self.getIntValue("nodeSharedMemory") + + @nodeSharedMemory.setter + def nodeSharedMemory(self, value: Optional[int]) -> None: + self._raw["nodeSharedMemory"] = value + + @property + def cpuCount(self) -> int: + return self.getIntValue("cpuCount") + + @cpuCount.setter + def cpuCount(self, value: Optional[int]) -> None: + self._raw["cpuCount"] = value + + @property + def nodeMode(self) -> int: + return self.getIntValue("nodeMode") + + @nodeMode.setter + def nodeMode(self, value: Optional[int]) -> None: + self._raw["nodeMode"] = value + + @property + def allowDocker(self) -> bool: + return self.getBoolValue("allowDocker") + + @allowDocker.setter + def allowDocker(self, value: Optional[bool]) -> None: + self._raw["allowDocker"] = value + + @property + def secretsKey(self) -> str: + return self.getStrValue("secretsKey") + + @secretsKey.setter + def secretsKey(self, value: Optional[str]) -> None: + self._raw["secretsKey"] = value + + @property + def initScript(self) -> str: + return self.getStrValue("initScript") + + @initScript.setter + def initScript(self, value: Optional[str]) -> None: + self._raw["initScript"] = value + + def getStrValue(self, configKey: str, envKey: Optional[str] = None) -> str: + if envKey is not None and envKey in os.environ: + return os.environ[envKey] + + value = self._raw.get(configKey) + + if not isinstance(value, str): + raise InvalidNodeConfiguration(f"Invalid f{configKey} type \"{type(value)}\", expected: \"str\".") + + return value + + def getIntValue(self, configKey: str, envKey: Optional[str] = None) -> int: + if envKey is not None and envKey in os.environ: + return int(os.environ[envKey]) + + value = self._raw.get(configKey) + + if not isinstance(value, int): + raise InvalidNodeConfiguration(f"Invalid f{configKey} type \"{type(value)}\", expected: \"int\".") + + return value + + def getBoolValue(self, configKey: str, envKey: Optional[str] = None) -> bool: + if envKey is not None and envKey in os.environ: + return bool(os.environ[envKey]) + + value = self._raw.get(configKey) + + if not isinstance(value, bool): + raise InvalidNodeConfiguration(f"Invalid f{configKey} type \"{type(value)}\", expected: \"bool\".") + + return value diff --git a/coretex/configuration/user.py b/coretex/configuration/user.py index 93d49d5a..7e977870 100644 --- a/coretex/configuration/user.py +++ b/coretex/configuration/user.py @@ -1,5 +1,4 @@ -from typing import Dict, Any, Optional, TypeVar -from pathlib import Path +from typing import Dict, Any, Optional from datetime import datetime, timezone import os @@ -9,9 +8,6 @@ from ..configuration import CONFIG_DIR -PropertyType = TypeVar("PropertyType") - - USER_CONFIG_PATH = CONFIG_DIR / "user_config.json" @@ -158,7 +154,7 @@ def getStrValue(self, configKey: str, envKey: Optional[str] = None) -> str: value = self._raw.get(configKey) if not isinstance(value, str): - raise InvalidUserConfiguration(f"Invalid username type \"{type(value)}\", expected: \"str\".") + raise InvalidUserConfiguration(f"Invalid f{configKey} type \"{type(value)}\", expected: \"str\".") return value @@ -169,6 +165,6 @@ def getIntValue(self, configKey: str, envKey: Optional[str] = None) -> int: value = self._raw.get(configKey) if not isinstance(value, int): - raise InvalidUserConfiguration(f"Invalid username type \"{type(value)}\", expected: \"str\".") + raise InvalidUserConfiguration(f"Invalid f{configKey} type \"{type(value)}\", expected: \"str\".") return value From f785e2d677cdfa5ea58846f00417fcdd3a10f3de Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Fri, 29 Mar 2024 14:00:01 +0100 Subject: [PATCH 03/28] CTX-5430: First part of refactor finished, saving changes (stop and start for node working). --- coretex/__init__.py | 3 +- coretex/cli/commands/login.py | 16 ++- coretex/cli/commands/model.py | 54 ++++----- coretex/cli/commands/node.py | 38 ++++--- coretex/cli/main.py | 8 +- coretex/cli/modules/node.py | 102 +++++++++-------- coretex/cli/modules/ui.py | 31 ++--- coretex/cli/modules/update.py | 41 ++++--- coretex/cli/modules/user.py | 62 +++------- coretex/configuration.py | 137 ----------------------- coretex/configuration/__init__.py | 10 ++ coretex/configuration/base.py | 56 +++++++++- coretex/configuration/node.py | 180 ++++++++++++++++-------------- coretex/configuration/user.py | 135 +++++++++++----------- coretex/old_configuration.py | 137 +++++++++++++++++++++++ coretex/utils/misc.py | 27 +++++ coretex/utils/process.py | 10 +- 17 files changed, 558 insertions(+), 489 deletions(-) delete mode 100644 coretex/configuration.py create mode 100644 coretex/old_configuration.py create mode 100644 coretex/utils/misc.py diff --git a/coretex/__init__.py b/coretex/__init__.py index 079d56dd..4f711967 100644 --- a/coretex/__init__.py +++ b/coretex/__init__.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +print('sync1') # Internal - not for outside use from .configuration import _syncConfigWithEnv _syncConfigWithEnv() @@ -22,7 +23,7 @@ # Internal - not for outside use from ._logger import _initializeDefaultLogger, _initializeCLILogger -from .configuration import isCliRuntime +from .utils.misc import isCliRuntime if isCliRuntime(): diff --git a/coretex/cli/commands/login.py b/coretex/cli/commands/login.py index 7f7e0efc..3f6e10de 100644 --- a/coretex/cli/commands/login.py +++ b/coretex/cli/commands/login.py @@ -17,17 +17,17 @@ import click -from ..modules.user import authenticate, saveLoginData +from ..modules.user import authenticate from ..modules.ui import clickPrompt, stdEcho, successEcho -from ...configuration import loadConfig, saveConfig, isUserConfigured +from ...configuration import UserConfiguration @click.command() def login() -> None: - config = loadConfig() - if isUserConfigured(config): + config = UserConfiguration() + if config.isUserConfigured(): if not clickPrompt( - f"User already logged in with username {config['username']}.\nWould you like to log in with a different user (Y/n)?", + f"User already logged in with username {config.username}.\nWould you like to log in with a different user (Y/n)?", type = bool, default = True, show_default = False @@ -36,8 +36,6 @@ def login() -> None: stdEcho("Please enter your credentials:") loginInfo = authenticate() - config = saveLoginData(loginInfo, config) + config.saveLoginData(loginInfo) - saveConfig(config) - - successEcho(f"User {config['username']} successfully logged in.") + successEcho(f"User {config.username} successfully logged in.") diff --git a/coretex/cli/commands/model.py b/coretex/cli/commands/model.py index 462a49f8..7962ed8b 100644 --- a/coretex/cli/commands/model.py +++ b/coretex/cli/commands/model.py @@ -1,39 +1,39 @@ -from typing import Optional +# from typing import Optional -import click +# import click -from ..modules import project_utils, user, utils, ui -from ...entities import Model -from ...configuration import loadConfig +# from ..modules import project_utils, user, utils, ui +# from ...entities import Model +# from ...configuration import loadConfig -@click.command() -@click.argument("path", type = click.Path(exists = True, file_okay = False, dir_okay = True)) -@click.option("-n", "--name", type = str, required = True) -@click.option("-p", "--project", type = str, required = False, default = None) -@click.option("-a", "--accuracy", type = click.FloatRange(0, 1), required = False, default = 1) -def create(name: str, path: str, project: Optional[str], accuracy: float) -> None: - config = loadConfig() +# @click.command() +# @click.argument("path", type = click.Path(exists = True, file_okay = False, dir_okay = True)) +# @click.option("-n", "--name", type = str, required = True) +# @click.option("-p", "--project", type = str, required = False, default = None) +# @click.option("-a", "--accuracy", type = click.FloatRange(0, 1), required = False, default = 1) +# def create(name: str, path: str, project: Optional[str], accuracy: float) -> None: +# config = loadConfig() - # If project was provided used that, otherwise get the one from config - # If project that was provided does not exist prompt user to create a new - # one with that name - ctxProject = project_utils.getProject(project, config) - if ctxProject is None: - return +# # If project was provided used that, otherwise get the one from config +# # If project that was provided does not exist prompt user to create a new +# # one with that name +# ctxProject = project_utils.getProject(project, config) +# if ctxProject is None: +# return - ui.progressEcho("Creating the model...") +# ui.progressEcho("Creating the model...") - model = Model.createProjectModel(name, ctxProject.id, accuracy) - model.upload(path) +# model = Model.createProjectModel(name, ctxProject.id, accuracy) +# model.upload(path) - ui.successEcho(f"Model \"{model.name}\" created successfully") +# ui.successEcho(f"Model \"{model.name}\" created successfully") -@click.group() -@utils.onBeforeCommandExecute(user.initializeUserSession) -def model() -> None: - pass +# @click.group() +# @utils.onBeforeCommandExecute(user.initializeUserSession) +# def model() -> None: +# pass -model.add_command(create) +# model.add_command(create) diff --git a/coretex/cli/commands/node.py b/coretex/cli/commands/node.py index d5048458..e72062d5 100644 --- a/coretex/cli/commands/node.py +++ b/coretex/cli/commands/node.py @@ -24,7 +24,7 @@ from ..modules.update import NodeStatus, getNodeStatus, activateAutoUpdate, dumpScript, UPDATE_SCRIPT_NAME from ..modules.utils import onBeforeCommandExecute from ..modules.user import initializeUserSession -from ...configuration import loadConfig, saveConfig, CONFIG_DIR, isNodeConfigured +from ...configuration import UserConfiguration, NodeConfiguration, CONFIG_DIR from ...utils import docker @@ -46,20 +46,21 @@ def start(image: Optional[str]) -> None: if node_module.exists(): node_module.clean() - config = loadConfig() + nodeConfig = NodeConfiguration() + userConfig = UserConfiguration() if image is not None: - config["image"] = image # store forced image (flagged) so we can run autoupdate afterwards - saveConfig(config) + nodeConfig.image = image # store forced image (flagged) so we can run autoupdate afterwards + nodeConfig.save() - dockerImage = config["image"] + dockerImage = nodeConfig.image if node_module.shouldUpdate(dockerImage): node_module.pull(dockerImage) - node_module.start(dockerImage, config) + node_module.start(dockerImage, userConfig, nodeConfig) - activateAutoUpdate(CONFIG_DIR, config) + activateAutoUpdate(CONFIG_DIR, userConfig, nodeConfig) @click.command() @@ -74,8 +75,8 @@ def stop() -> None: @click.command() @onBeforeCommandExecute(node_module.initializeNodeConfiguration) def update() -> None: - config = loadConfig() - dockerImage = config["image"] + userConfig = UserConfiguration() + nodeConfig = NodeConfiguration() nodeStatus = getNodeStatus() @@ -98,11 +99,11 @@ def update() -> None: node_module.stop() - if not node_module.shouldUpdate(dockerImage): + if not node_module.shouldUpdate(nodeConfig.image): successEcho("Node is already up to date.") return - node_module.pull(dockerImage) + node_module.pull(nodeConfig.image) if getNodeStatus() == NodeStatus.busy: if not clickPrompt( @@ -115,7 +116,7 @@ def update() -> None: node_module.stop() - node_module.start(dockerImage, config) + node_module.start(nodeConfig.image, userConfig, nodeConfig) @click.command() @@ -133,9 +134,10 @@ def config(verbose: bool) -> None: node_module.stop() - config = loadConfig() + userConfig = UserConfiguration() + nodeConfig = NodeConfiguration() - if isNodeConfigured(config): + if nodeConfig.isNodeConfigured(): if not clickPrompt( "Node configuration already exists. Would you like to update? (Y/n)", type = bool, @@ -144,12 +146,12 @@ def config(verbose: bool) -> None: ): return - node_module.configureNode(config, verbose) - saveConfig(config) - previewConfig(config) + node_module.configureNode(nodeConfig, verbose) + nodeConfig.save() + previewConfig(userConfig, nodeConfig) # Updating auto-update script since node configuration is changed - dumpScript(CONFIG_DIR / UPDATE_SCRIPT_NAME, config) + dumpScript(CONFIG_DIR / UPDATE_SCRIPT_NAME, userConfig, nodeConfig) successEcho("Node successfully configured.") diff --git a/coretex/cli/main.py b/coretex/cli/main.py index a3d09166..0b2ad40c 100644 --- a/coretex/cli/main.py +++ b/coretex/cli/main.py @@ -18,9 +18,9 @@ import click from .commands.login import login -from .commands.model import model +# from .commands.model import model from .commands.node import node -from .commands.project import project +# from .commands.project import project from .modules.intercept import ClickExceptionInterceptor @@ -30,6 +30,6 @@ def cli() -> None: pass cli.add_command(login) -cli.add_command(model) -cli.add_command(project) +# cli.add_command(model) +# cli.add_command(project) cli.add_command(node) diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index 1572d011..6ebc3759 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -26,10 +26,9 @@ from .ui import clickPrompt, arrowPrompt, highlightEcho, errorEcho, progressEcho, successEcho, stdEcho from .node_mode import NodeMode from ...networking import networkManager, NetworkRequestError -from ...configuration import loadConfig, saveConfig, isNodeConfigured, getInitScript from ...utils import CommandException, docker from ...entities.model import Model - +from ...configuration import UserConfiguration, NodeConfiguration class NodeException(Exception): pass @@ -59,45 +58,44 @@ def exists() -> bool: return docker.containerExists(config_defaults.DOCKER_CONTAINER_NAME) -def start(dockerImage: str, config: Dict[str, Any]) -> None: +def start(dockerImage: str, userConfig: UserConfiguration, nodeConfig: NodeConfiguration) -> None: try: progressEcho("Starting Coretex Node...") docker.createNetwork(config_defaults.DOCKER_CONTAINER_NETWORK) environ = { - "CTX_API_URL": config["serverUrl"], + "CTX_API_URL": userConfig.serverUrl, "CTX_STORAGE_PATH": "/root/.coretex", - "CTX_NODE_ACCESS_TOKEN": config["nodeAccessToken"], - "CTX_NODE_MODE": config["nodeMode"] + "CTX_NODE_ACCESS_TOKEN": nodeConfig.nodeAccessToken, + "CTX_NODE_MODE": str(nodeConfig.nodeMode) } - modelId = config.get("modelId") - if isinstance(modelId, int): - environ["CTX_MODEL_ID"] = modelId + if isinstance(nodeConfig.modelId, int): + environ["CTX_MODEL_ID"] = str(nodeConfig.modelId) - secretsKey = config.get("secretsKey", config_defaults.DEFAULT_SECRETS_KEY) + secretsKey = nodeConfig.secretsKey if isinstance(secretsKey, str) and secretsKey != config_defaults.DEFAULT_SECRETS_KEY: environ["CTX_SECRETS_KEY"] = secretsKey volumes = [ - (config["storagePath"], "/root/.coretex") + (nodeConfig.storagePath, "/root/.coretex") ] - if config.get("allowDocker", False): + if nodeConfig.allowDocker: volumes.append(("/var/run/docker.sock", "/var/run/docker.sock")) - initScript = getInitScript(config) + initScript = nodeConfig.getInitScriptPath() if initScript is not None: volumes.append((str(initScript), "/script/init.sh")) docker.start( config_defaults.DOCKER_CONTAINER_NAME, dockerImage, - config["allowGpu"], - config["nodeRam"], - config["nodeSwap"], - config["nodeSharedMemory"], - config["cpuCount"], + nodeConfig.allowGpu, + nodeConfig.nodeRam, + nodeConfig.nodeSwap, + nodeConfig.nodeSharedMemory, + nodeConfig.cpuCount, environ, volumes ) @@ -255,58 +253,58 @@ def _configureInitScript() -> str: return str(path) -def configureNode(config: Dict[str, Any], verbose: bool) -> None: +def configureNode(config: NodeConfiguration, verbose: bool) -> None: highlightEcho("[Node Configuration]") - config["nodeName"] = clickPrompt("Node name", type = str) - config["nodeAccessToken"] = registerNode(config["nodeName"]) + config.nodeName = clickPrompt("Node name", type = str) + config.nodeAccessToken = registerNode(config.nodeName) imageType = selectImageType() if imageType == ImageType.custom: - config["image"] = clickPrompt("Specify URL of docker image that you want to use:", type = str) + config.image = clickPrompt("Specify URL of docker image that you want to use:", type = str) else: - config["image"] = "coretexai/coretex-node" + config.image = "coretexai/coretex-node" if isGPUAvailable(): - config["allowGpu"] = clickPrompt("Do you want to allow the Node to access your GPU? (Y/n)", type = bool, default = True) + config.allowGpu = clickPrompt("Do you want to allow the Node to access your GPU? (Y/n)", type = bool, default = True) else: - config["allowGpu"] = False + config.allowGpu = False if imageType == ImageType.official: - tag = "gpu" if config["allowGpu"] else "cpu" - config["image"] += f":latest-{tag}" - - config["storagePath"] = config_defaults.DEFAULT_STORAGE_PATH - config["nodeRam"] = config_defaults.DEFAULT_RAM_MEMORY - config["nodeSwap"] = config_defaults.DEFAULT_SWAP_MEMORY - config["nodeSharedMemory"] = config_defaults.DEFAULT_SHARED_MEMORY - config["cpuCount"] = config_defaults.DEFAULT_CPU_COUNT - config["nodeMode"] = config_defaults.DEFAULT_NODE_MODE - config["allowDocker"] = config_defaults.DEFAULT_ALLOW_DOCKER - config["secretsKey"] = config_defaults.DEFAULT_SECRETS_KEY - config["initScript"] = config_defaults.DEFAULT_INIT_SCRIPT + tag = "gpu" if config.allowGpu else "cpu" + config.image += f":latest-{tag}" + + config.storagePath = config_defaults.DEFAULT_STORAGE_PATH + config.nodeRam = config_defaults.DEFAULT_RAM_MEMORY + config.nodeSwap = config_defaults.DEFAULT_SWAP_MEMORY + config.nodeSharedMemory = config_defaults.DEFAULT_SHARED_MEMORY + config.cpuCount = config_defaults.DEFAULT_CPU_COUNT if config_defaults.DEFAULT_CPU_COUNT is not None else 0 # is this fine? + config.nodeMode = config_defaults.DEFAULT_NODE_MODE + config.allowDocker = config_defaults.DEFAULT_ALLOW_DOCKER + config.secretsKey = config_defaults.DEFAULT_SECRETS_KEY + config.initScript = config_defaults.DEFAULT_INIT_SCRIPT if verbose: - config["storagePath"] = clickPrompt("Storage path (press enter to use default)", config_defaults.DEFAULT_STORAGE_PATH, type = str) - config["nodeRam"] = clickPrompt("Node RAM memory limit in GB (press enter to use default)", config_defaults.DEFAULT_RAM_MEMORY, type = int) - config["nodeSwap"] = clickPrompt("Node swap memory limit in GB, make sure it is larger than mem limit (press enter to use default)", config_defaults.DEFAULT_SWAP_MEMORY, type = int) - config["nodeSharedMemory"] = clickPrompt("Node POSIX shared memory limit in GB (press enter to use default)", config_defaults.DEFAULT_SHARED_MEMORY, type = int) - config["cpuCount"] = clickPrompt("Enter the number of CPUs the container will use (press enter to use default)", config_defaults.DEFAULT_CPU_COUNT, type = int) - config["allowDocker"] = clickPrompt("Allow Node to access system docker? This is a security risk! (Y/n)", config_defaults.DEFAULT_ALLOW_DOCKER, type = bool) - config["secretsKey"] = clickPrompt("Enter a key used for decrypting your Coretex Secrets", config_defaults.DEFAULT_SECRETS_KEY, type = str, hide_input = True) - config["initScript"] = _configureInitScript() - - nodeMode, modelId = selectNodeMode(config["storagePath"]) - config["nodeMode"] = nodeMode + config.storagePath = clickPrompt("Storage path (press enter to use default)", config_defaults.DEFAULT_STORAGE_PATH, type = str) + config.nodeRam = clickPrompt("Node RAM memory limit in GB (press enter to use default)", config_defaults.DEFAULT_RAM_MEMORY, type = int) + config.nodeSwap = clickPrompt("Node swap memory limit in GB, make sure it is larger than mem limit (press enter to use default)", config_defaults.DEFAULT_SWAP_MEMORY, type = int) + config.nodeSharedMemory = clickPrompt("Node POSIX shared memory limit in GB (press enter to use default)", config_defaults.DEFAULT_SHARED_MEMORY, type = int) + config.cpuCount = clickPrompt("Enter the number of CPUs the container will use (press enter to use default)", config_defaults.DEFAULT_CPU_COUNT, type = int) + config.allowDocker = clickPrompt("Allow Node to access system docker? This is a security risk! (Y/n)", config_defaults.DEFAULT_ALLOW_DOCKER, type = bool) + config.secretsKey = clickPrompt("Enter a key used for decrypting your Coretex Secrets", config_defaults.DEFAULT_SECRETS_KEY, type = str, hide_input = True) + config.initScript = _configureInitScript() + + nodeMode, modelId = selectNodeMode(config.storagePath) + config.nodeMode = nodeMode if modelId is not None: - config["modelId"] = modelId + config.modelId = modelId else: stdEcho("To configure node manually run coretex node config with --verbose flag.") def initializeNodeConfiguration() -> None: - config = loadConfig() + config = NodeConfiguration() - if isNodeConfigured(config): + if config.isNodeConfigured(): return errorEcho("Node configuration not found.") @@ -325,4 +323,4 @@ def initializeNodeConfiguration() -> None: stop() configureNode(config, verbose = False) - saveConfig(config) + config.save() diff --git a/coretex/cli/modules/ui.py b/coretex/cli/modules/ui.py index 6beba8ca..f67dab57 100644 --- a/coretex/cli/modules/ui.py +++ b/coretex/cli/modules/ui.py @@ -24,6 +24,7 @@ from .node_mode import NodeMode from .config_defaults import DEFAULT_CPU_COUNT +from ...configuration import UserConfiguration, NodeConfiguration def clickPrompt( @@ -49,30 +50,30 @@ def arrowPrompt(choices: List[Any], message: str) -> Any: return answers["option"] -def previewConfig(config: Dict[str, Any]) -> None: - allowDocker = "Yes" if config.get("allowDocker", False) else "No" +def previewConfig(userConfig: UserConfiguration, nodeConfig: NodeConfiguration) -> None: + allowDocker = "Yes" if nodeConfig.allowDocker else "No" - if config.get("secretsKey") is None or config.get("secretsKey") == "": + if nodeConfig.secretsKey is None or nodeConfig.secretsKey == "": secretsKey = "" else: secretsKey = "********" table = [ - ["Node name", config["nodeName"]], - ["Server URL", config["serverUrl"]], - ["Coretex Node type", config["image"]], - ["Storage path", config["storagePath"]], - ["RAM", f"{config['nodeRam']}GB"], - ["SWAP memory", f"{config['nodeSwap']}GB"], - ["POSIX shared memory", f"{config['nodeSharedMemory']}GB"], - ["CPU cores allocated", config.get("cpuCount", DEFAULT_CPU_COUNT)], - ["Coretex Node mode", f"{NodeMode(config['nodeMode']).name}"], + ["Node name", nodeConfig.nodeName], + ["Server URL", userConfig.serverUrl], + ["Coretex Node type", nodeConfig.image], + ["Storage path", nodeConfig.storagePath], + ["RAM", f"{nodeConfig.nodeRam}GB"], + ["SWAP memory", f"{nodeConfig.nodeSwap}GB"], + ["POSIX shared memory", f"{nodeConfig.nodeSharedMemory}GB"], + ["CPU cores allocated", f"{nodeConfig.cpuCount}"], + ["Coretex Node mode", f"{NodeMode(nodeConfig.nodeMode).name}"], ["Docker access", allowDocker], ["Secrets key", secretsKey], - ["Node init script", config.get("initScript", "")] + ["Node init script", nodeConfig.initScript if nodeConfig.initScript is not None else ""] ] - if config.get("modelId") is not None: - table.append(["Coretex Model ID", config["modelId"]]) + if nodeConfig.modelId is not None: + table.append(["Coretex Model ID", f"{nodeConfig.modelId}"]) stdEcho(tabulate(table)) diff --git a/coretex/cli/modules/update.py b/coretex/cli/modules/update.py index f284cdd8..cdaa9ab8 100644 --- a/coretex/cli/modules/update.py +++ b/coretex/cli/modules/update.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Dict, Any from enum import IntEnum from pathlib import Path @@ -26,7 +25,7 @@ from .node import getRepoFromImageUrl, getTagFromImageUrl from ..resources import RESOURCES_DIR from ...utils import command -from ...configuration import CONFIG_DIR, getInitScript +from ...configuration import CONFIG_DIR, UserConfiguration, NodeConfiguration UPDATE_SCRIPT_NAME = "ctx_node_update.sh" @@ -41,7 +40,7 @@ class NodeStatus(IntEnum): reconnecting = 5 -def generateUpdateScript(config: Dict[str, Any]) -> str: +def generateUpdateScript(userConfig: UserConfiguration, nodeConfig: NodeConfiguration) -> str: _, dockerPath, _ = command(["which", "docker"], ignoreStdout = True, ignoreStderr = True) bashScriptTemplatePath = RESOURCES_DIR / "update_script_template.sh" @@ -50,38 +49,38 @@ def generateUpdateScript(config: Dict[str, Any]) -> str: return bashScriptTemplate.format( dockerPath = dockerPath.strip(), - repository = getRepoFromImageUrl(config["image"]), - tag = getTagFromImageUrl(config["image"]), - serverUrl = config["serverUrl"], - storagePath = config["storagePath"], - nodeAccessToken = config["nodeAccessToken"], - nodeMode = config["nodeMode"], - modelId = config.get("modelId"), + repository = getRepoFromImageUrl(nodeConfig.image), + tag = getTagFromImageUrl(nodeConfig.image), + serverUrl = userConfig.serverUrl, + storagePath = nodeConfig.storagePath, + nodeAccessToken = nodeConfig.nodeAccessToken, + nodeMode = nodeConfig.nodeMode, + modelId = nodeConfig.modelId, containerName = config_defaults.DOCKER_CONTAINER_NAME, networkName = config_defaults.DOCKER_CONTAINER_NETWORK, restartPolicy = "always", ports = "21000:21000", capAdd = "SYS_PTRACE", - ramMemory = config["nodeRam"], - swapMemory = config["nodeSwap"], - sharedMemory = config["nodeSharedMemory"], - cpuCount = config.get("cpuCount", config_defaults.DEFAULT_CPU_COUNT), - imageType = "cpu" if config["allowGpu"] is False else "gpu", - allowDocker = config.get("allowDocker", False), - initScript = getInitScript(config) + ramMemory = nodeConfig.nodeRam, + swapMemory = nodeConfig.nodeSwap, + sharedMemory = nodeConfig.nodeSharedMemory, + cpuCount = nodeConfig.cpuCount, + imageType = "cpu" if nodeConfig.allowGpu is False else "gpu", + allowDocker = nodeConfig.allowDocker, + initScript = nodeConfig.getInitScriptPath() ) -def dumpScript(updateScriptPath: Path, config: Dict[str, Any]) -> None: +def dumpScript(updateScriptPath: Path, userConfig: UserConfiguration, nodeConfig: NodeConfiguration) -> None: with updateScriptPath.open("w") as scriptFile: - scriptFile.write(generateUpdateScript(config)) + scriptFile.write(generateUpdateScript(userConfig, nodeConfig)) command(["chmod", "+x", str(updateScriptPath)], ignoreStdout = True) -def activateAutoUpdate(configDir: Path, config: Dict[str, Any]) -> None: +def activateAutoUpdate(configDir: Path, userConfig: UserConfiguration, nodeConfig: NodeConfiguration) -> None: updateScriptPath = CONFIG_DIR / UPDATE_SCRIPT_NAME - dumpScript(updateScriptPath, config) + dumpScript(updateScriptPath, userConfig, nodeConfig) if not jobExists(UPDATE_SCRIPT_NAME): scheduleJob(configDir, UPDATE_SCRIPT_NAME) diff --git a/coretex/cli/modules/user.py b/coretex/cli/modules/user.py index c101474e..c0dec9f4 100644 --- a/coretex/cli/modules/user.py +++ b/coretex/cli/modules/user.py @@ -15,25 +15,12 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Dict, Any -from dataclasses import dataclass from datetime import datetime, timezone from .ui import clickPrompt, errorEcho, progressEcho from ...utils import decodeDate from ...networking import networkManager, NetworkResponse, NetworkRequestError -from ...configuration import loadConfig, saveConfig - - -@dataclass -class LoginInfo: - - username: str - password: str - token: str - tokenExpirationDate: str - refreshToken: str - refreshTokenExpirationDate: str +from ...configuration import LoginInfo, UserConfiguration def authenticateUser(username: str, password: str) -> NetworkResponse: @@ -49,17 +36,6 @@ def authenticateUser(username: str, password: str) -> NetworkResponse: return response -def saveLoginData(loginInfo: LoginInfo, config: Dict[str, Any]) -> Dict[str, Any]: - config["username"] = loginInfo.username - config["password"] = loginInfo.password - config["token"] = loginInfo.token - config["tokenExpirationDate"] = loginInfo.tokenExpirationDate - config["refreshToken"] = loginInfo.refreshToken - config["refreshTokenExpirationDate"] = loginInfo.refreshTokenExpirationDate - - return config - - def authenticate(retryCount: int = 0) -> LoginInfo: if retryCount >= 3: raise RuntimeError("Failed to authenticate. Terminating...") @@ -87,42 +63,38 @@ def authenticate(retryCount: int = 0) -> LoginInfo: def initializeUserSession() -> None: - config = loadConfig() + config = UserConfiguration() - if config.get("username") is None or config.get("password") is None: + if config.username is None or config.password is None: errorEcho("User configuration not found. Please authenticate with your credentials.") loginInfo = authenticate() - config = saveLoginData(loginInfo, config) + config.saveLoginData(loginInfo) else: - tokenExpirationDate = config.get("tokenExpirationDate") - refreshTokenExpirationDate = config.get("refreshTokenExpirationDate") - - if tokenExpirationDate is not None and refreshTokenExpirationDate is not None: - tokenExpirationDate = decodeDate(tokenExpirationDate) - refreshTokenExpirationDate = decodeDate(refreshTokenExpirationDate) + if config.tokenExpirationDate is not None and config.refreshTokenExpirationDate is not None: + tokenExpirationDate = decodeDate(config.tokenExpirationDate) + refreshTokenExpirationDate = decodeDate(config.refreshTokenExpirationDate) currentDate = datetime.utcnow().replace(tzinfo = timezone.utc) if currentDate < tokenExpirationDate: return - if currentDate < refreshTokenExpirationDate: - refreshToken = config["refreshToken"] - response = networkManager.authenticateWithRefreshToken(refreshToken) + if currentDate < refreshTokenExpirationDate and config.refreshToken is not None: + response = networkManager.authenticateWithRefreshToken(config.refreshToken) if response.hasFailed(): if response.statusCode >= 500: raise NetworkRequestError(response, "Something went wrong, please try again later.") if response.statusCode >= 400: - response = authenticateUser(config["username"], config["password"]) + response = authenticateUser(config.username, config.password) else: - response = authenticateUser(config["username"], config["password"]) + response = authenticateUser(config.username, config.password) else: - response = authenticateUser(config["username"], config["password"]) + response = authenticateUser(config.username, config.password) jsonResponse = response.getJson(dict) - config["token"] = jsonResponse["token"] - config["tokenExpirationDate"] = jsonResponse["expires_on"] - config["refreshToken"] = jsonResponse.get("refresh_token") - config["refreshTokenExpirationDate"] = jsonResponse.get("refresh_expires_on") + config.token = jsonResponse["token"] + config.tokenExpirationDate = jsonResponse["expires_on"] + config.refreshToken = jsonResponse.get("refresh_token") + config.refreshTokenExpirationDate = jsonResponse.get("refresh_expires_on") - saveConfig(config) + config.save() diff --git a/coretex/configuration.py b/coretex/configuration.py deleted file mode 100644 index ca755f7f..00000000 --- a/coretex/configuration.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright (C) 2023 Coretex LLC - -# This file is part of Coretex.ai - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from typing import Dict, Any, Optional -from pathlib import Path - -import os -import json -import sys - - -def isCliRuntime() -> bool: - executablePath = sys.argv[0] - return ( - executablePath.endswith("/bin/coretex") and - os.access(executablePath, os.X_OK) - ) - - -def getEnvVar(key: str, default: str) -> str: - if os.environ.get(key) is None: - return default - - return os.environ[key] - - -CONFIG_DIR = Path.home().joinpath(".config", "coretex") -DEFAULT_CONFIG_PATH = CONFIG_DIR / "config.json" - - -DEFAULT_CONFIG: Dict[str, Any] = { - # os.environ used directly here since we don't wanna - # set those variables to any value if they don't exist - # in the os.environ the same way we do for properties which - # call genEnvVar - "username": os.environ.get("CTX_USERNAME"), - "password": os.environ.get("CTX_PASSWORD"), - "token": None, - "refreshToken": None, - "serverUrl": getEnvVar("CTX_API_URL", "https://api.coretex.ai/"), - "storagePath": getEnvVar("CTX_STORAGE_PATH", "~/.coretex") -} - - -def loadConfig() -> Dict[str, Any]: - with DEFAULT_CONFIG_PATH.open("r") as configFile: - try: - config: Dict[str, Any] = json.load(configFile) - except json.JSONDecodeError: - config = {} - - for key, value in DEFAULT_CONFIG.items(): - if not key in config: - config[key] = value - - return config - - -def _syncConfigWithEnv() -> None: - # If configuration does not exist create default one - if not DEFAULT_CONFIG_PATH.exists(): - DEFAULT_CONFIG_PATH.parent.mkdir(parents = True, exist_ok = True) - config = DEFAULT_CONFIG.copy() - else: - config = loadConfig() - - saveConfig(config) - - if not "CTX_API_URL" in os.environ: - os.environ["CTX_API_URL"] = config["serverUrl"] - - secretsKey = config.get("secretsKey") - if isinstance(secretsKey, str) and secretsKey != "": - os.environ["CTX_SECRETS_KEY"] = secretsKey - - if not isCliRuntime(): - os.environ["CTX_STORAGE_PATH"] = config["storagePath"] - else: - os.environ["CTX_STORAGE_PATH"] = f"{CONFIG_DIR}/data" - - -def saveConfig(config: Dict[str, Any]) -> None: - configPath = DEFAULT_CONFIG_PATH.expanduser() - with configPath.open("w+") as configFile: - json.dump(config, configFile, indent = 4) - - -def isUserConfigured(config: Dict[str, Any]) -> bool: - return ( - config.get("username") is not None and - config.get("password") is not None and - config.get("storagePath") is not None - ) - - -def isNodeConfigured(config: Dict[str, Any]) -> bool: - return ( - config.get("nodeName") is not None and - config.get("storagePath") is not None and - config.get("image") is not None and - config.get("serverUrl") is not None and - config.get("nodeAccessToken") is not None and - config.get("nodeRam") is not None and - config.get("nodeSwap") is not None and - config.get("nodeSharedMemory") is not None and - config.get("nodeMode") is not None - ) - - -def getInitScript(config: Dict[str, Any]) -> Optional[Path]: - value = config.get("initScript") - - if not isinstance(value, str): - return None - - if value == "": - return None - - path = Path(value).expanduser().absolute() - if not path.exists(): - return None - - return path diff --git a/coretex/configuration/__init__.py b/coretex/configuration/__init__.py index e69de29b..2d985d90 100644 --- a/coretex/configuration/__init__.py +++ b/coretex/configuration/__init__.py @@ -0,0 +1,10 @@ +from .base import CONFIG_DIR +from .user import UserConfiguration, LoginInfo +from .node import NodeConfiguration + + +def _syncConfigWithEnv() -> None: + # TODO: explain this ! + print("sync") + UserConfiguration() + NodeConfiguration() diff --git a/coretex/configuration/base.py b/coretex/configuration/base.py index efcceb97..7e515254 100644 --- a/coretex/configuration/base.py +++ b/coretex/configuration/base.py @@ -1,2 +1,56 @@ -class BaseConfiguration: +from typing import Dict, Any, Optional, Type, TypeVar +from abc import abstractmethod +from pathlib import Path + +import os +import json + +T = TypeVar("T", int, str, bool) + +CONFIG_DIR = Path.home().joinpath(".config", "coretex") + + +class InvalidConfiguration(Exception): pass + +class BaseConfiguration: + def __init__(self, path: Path) -> None: + self._path = path + + if not path.exists(): + self._raw = self.getDefaultConfig() + self.save() + else: + with path.open("r") as file: + self._raw = json.load(file) + + @classmethod + @abstractmethod + def getDefaultConfig(cls) -> Dict[str, Any]: + pass + + def _value(self, configKey: str, valueType: Type[T], envKey: Optional[str] = None) -> Optional[T]: + if envKey is not None and envKey in os.environ: + return valueType(os.environ[envKey]) + + return self._raw.get(configKey) + + def getValue(self, configKey: str, valueType: Type[T], envKey: Optional[str] = None) -> T: + value = self._value(configKey, valueType, envKey) + + if not isinstance(value, valueType): + raise InvalidConfiguration(f"Invalid {configKey} type \"{type(value)}\", expected: \"{valueType.__name__}\".") + + return value + + def getOptValue(self, configKey: str, valueType: Type[T], envKey: Optional[str] = None) -> Optional[T]: + value = self._value(configKey, valueType, envKey) + + if not isinstance(value, valueType): + return None + + return value + + def save(self) -> None: + with self._path.open("w") as configFile: + json.dump(self._raw, configFile, indent = 4) diff --git a/coretex/configuration/node.py b/coretex/configuration/node.py index fde895bc..d017c91b 100644 --- a/coretex/configuration/node.py +++ b/coretex/configuration/node.py @@ -1,176 +1,194 @@ -from typing import Dict, Any, Optional +from typing import Dict, Any, Optional, TypeVar +from pathlib import Path import os -import json +import sys -from ..configuration import CONFIG_DIR +from .base import BaseConfiguration, CONFIG_DIR -NODE_CONFIG_PATH = CONFIG_DIR / "node_config.json" -class InvalidNodeConfiguration(Exception): - pass +def getEnvVar(key: str, default: str) -> str: + if os.environ.get(key) is None: + return default + return os.environ[key] -def initializeConfig() -> Dict[str, Any]: - config = { - "nodeName": os.environ.get("CTX_NODE_NAME"), - "nodeAccessToken": None, - "image": "coretexai/coretex-node:latest-cpu", - "allowGpu": False, - "nodeRam": None, - "nodeSharedMemory": None, - "cpuCount": None, - "nodeMode": None, - "allowDocker": False, - "secretsKey": None, - "initScript": None - } - if not NODE_CONFIG_PATH.exists(): - with NODE_CONFIG_PATH.open("w") as configFile: - json.dump(config, configFile, indent = 4) - else: - with open(NODE_CONFIG_PATH, "r") as file: - config = json.load(file) +NODE_CONFIG_PATH = CONFIG_DIR / "node_config.json" +NODE_DEFAULT_CONFIG = { + "nodeName": os.environ.get("CTX_NODE_NAME"), + "nodeAccessToken": None, + "storagePath": getEnvVar("CTX_STORAGE_PATH", "~/.coretex"), + "image": "coretexai/coretex-node:latest-cpu", + "allowGpu": False, + "nodeRam": None, + "nodeSharedMemory": None, + "cpuCount": None, + "nodeMode": None, + "allowDocker": False, + "secretsKey": None, + "initScript": None, + "modelId": None, +} + + +def isCliRuntime() -> bool: + executablePath = sys.argv[0] + return ( + executablePath.endswith("/bin/coretex") and + os.access(executablePath, os.X_OK) + ) - if not isinstance(config, dict): - raise InvalidNodeConfiguration(f"Invalid config type \"{type(config)}\", expected: \"dict\".") - return config +class InvalidNodeConfiguration(Exception): + pass -class NodeConfiguration: +class NodeConfiguration(BaseConfiguration): def __init__(self) -> None: - self._raw = initializeConfig() + super().__init__(NODE_CONFIG_PATH) + secretsKey = self.secretsKey + if isinstance(secretsKey, str) and secretsKey != "": + os.environ["CTX_SECRETS_KEY"] = secretsKey + + if not isCliRuntime(): + os.environ["CTX_STORAGE_PATH"] = self.storagePath + else: + os.environ["CTX_STORAGE_PATH"] = f"{CONFIG_DIR}/data" + + @classmethod + def getDefaultConfig(cls) -> Dict[str, Any]: + return NODE_DEFAULT_CONFIG @property def nodeName(self) -> str: - return self.getStrValue("nodeName", "CTX_NODE_NAME") + return self.getValue("nodeName", str, "CTX_NODE_NAME") @nodeName.setter - def nodeName(self, value: Optional[str]) -> None: + def nodeName(self, value: str) -> None: self._raw["nodeName"] = value @property def nodeAccessToken(self) -> str: - return self.getStrValue("nodeAccessToken") + return self.getValue("nodeAccessToken", str) @nodeAccessToken.setter - def nodeAccessToken(self, value: Optional[str]) -> None: + def nodeAccessToken(self, value: str) -> None: self._raw["nodeAccessToken"] = value + @property + def storagePath(self) -> str: + return self.getValue("storagePath", str, "CTX_STORAGE_PATH") + + @storagePath.setter + def storagePath(self, value: Optional[str]) -> None: + self._raw["storagePath"] = value + @property def image(self) -> str: - return self.getStrValue("image", "CTX_NODE_NAME") + return self.getValue("image", str) @image.setter - def image(self, value: Optional[str]) -> None: + def image(self, value: str) -> None: self._raw["image"] = value @property - def allowGpu(self) -> str: - return self.getStrValue("allowGpu") + def allowGpu(self) -> bool: + return self.getValue("allowGpu", bool) @allowGpu.setter - def allowGpu(self, value: Optional[bool]) -> None: + def allowGpu(self, value: bool) -> None: self._raw["allowGpu"] = value @property def nodeRam(self) -> int: - return self.getIntValue("nodeRam") + return self.getValue("nodeRam", int) @nodeRam.setter - def nodeRam(self, value: Optional[int]) -> None: + def nodeRam(self, value: int) -> None: self._raw["nodeRam"] = value @property def nodeSwap(self) -> int: - return self.getIntValue("nodeSwap") + return self.getValue("nodeSwap", int) @nodeSwap.setter - def nodeSwap(self, value: Optional[int]) -> None: + def nodeSwap(self, value: int) -> None: self._raw["nodeSwap"] = value @property def nodeSharedMemory(self) -> int: - return self.getIntValue("nodeSharedMemory") + return self.getValue("nodeSharedMemory", int) @nodeSharedMemory.setter - def nodeSharedMemory(self, value: Optional[int]) -> None: + def nodeSharedMemory(self, value: int) -> None: self._raw["nodeSharedMemory"] = value @property def cpuCount(self) -> int: - return self.getIntValue("cpuCount") + return self.getValue("cpuCount", int) @cpuCount.setter - def cpuCount(self, value: Optional[int]) -> None: + def cpuCount(self, value: int) -> None: self._raw["cpuCount"] = value @property def nodeMode(self) -> int: - return self.getIntValue("nodeMode") + return self.getValue("nodeMode", int) @nodeMode.setter - def nodeMode(self, value: Optional[int]) -> None: + def nodeMode(self, value: int) -> None: self._raw["nodeMode"] = value @property def allowDocker(self) -> bool: - return self.getBoolValue("allowDocker") + return self.getValue("allowDocker", bool) @allowDocker.setter - def allowDocker(self, value: Optional[bool]) -> None: + def allowDocker(self, value: bool) -> None: self._raw["allowDocker"] = value @property - def secretsKey(self) -> str: - return self.getStrValue("secretsKey") + def secretsKey(self) -> Optional[str]: + return self.getOptValue("secretsKey", str) @secretsKey.setter def secretsKey(self, value: Optional[str]) -> None: self._raw["secretsKey"] = value @property - def initScript(self) -> str: - return self.getStrValue("initScript") + def initScript(self) -> Optional[str]: + return self.getOptValue("initScript", str) @initScript.setter def initScript(self, value: Optional[str]) -> None: self._raw["initScript"] = value - def getStrValue(self, configKey: str, envKey: Optional[str] = None) -> str: - if envKey is not None and envKey in os.environ: - return os.environ[envKey] - - value = self._raw.get(configKey) - - if not isinstance(value, str): - raise InvalidNodeConfiguration(f"Invalid f{configKey} type \"{type(value)}\", expected: \"str\".") - - return value - - def getIntValue(self, configKey: str, envKey: Optional[str] = None) -> int: - if envKey is not None and envKey in os.environ: - return int(os.environ[envKey]) + @property + def modelId(self) -> Optional[int]: + return self.getOptValue("modelId", int) - value = self._raw.get(configKey) + @modelId.setter + def modelId(self, value: Optional[int]) -> None: + self._raw["modelId"] = value - if not isinstance(value, int): - raise InvalidNodeConfiguration(f"Invalid f{configKey} type \"{type(value)}\", expected: \"int\".") + def isNodeConfigured(self) -> bool: + return False - return value + def getInitScriptPath(self) -> Optional[Path]: + value = self._raw.get("initScript") - def getBoolValue(self, configKey: str, envKey: Optional[str] = None) -> bool: - if envKey is not None and envKey in os.environ: - return bool(os.environ[envKey]) + if not isinstance(value, str): + return None - value = self._raw.get(configKey) + if value == "": + return None - if not isinstance(value, bool): - raise InvalidNodeConfiguration(f"Invalid f{configKey} type \"{type(value)}\", expected: \"bool\".") + path = Path(value).expanduser().absolute() + if not path.exists(): + return None - return value + return path diff --git a/coretex/configuration/user.py b/coretex/configuration/user.py index 7e977870..b6005dea 100644 --- a/coretex/configuration/user.py +++ b/coretex/configuration/user.py @@ -1,43 +1,39 @@ from typing import Dict, Any, Optional from datetime import datetime, timezone +from dataclasses import dataclass import os -import json +from .base import BaseConfiguration, CONFIG_DIR from ..utils import decodeDate -from ..configuration import CONFIG_DIR USER_CONFIG_PATH = CONFIG_DIR / "user_config.json" +USER_DEFAULT_CONFIG = { + "username": os.environ.get("CTX_USERNAME"), + "password": os.environ.get("CTX_PASSWORD"), + "token": None, + "refreshToken": None, + "tokenExpirationDate": None, + "refreshTokenExpirationDate": None, + "serverUrl": os.environ.get("CTX_API_URL", "https://api.coretex.ai/"), + "projectId": os.environ.get("CTX_PROJECT_ID") +} class InvalidUserConfiguration(Exception): pass -def initializeConfig() -> Dict[str, Any]: - config = { - "username": os.environ.get("CTX_USERNAME"), - "password": os.environ.get("CTX_PASSWORD"), - "token": None, - "refreshToken": None, - "tokenExpirationDate": None, - "refreshTokenExpirationDate": None, - "serverUrl": os.environ.get("CTX_API_URL", "https://api.coretex.ai/"), - "projectId": os.environ.get("CTX_PROJECT_ID") - } +@dataclass +class LoginInfo: - if not USER_CONFIG_PATH.exists(): - with USER_CONFIG_PATH.open("w") as configFile: - json.dump(config, configFile, indent = 4) - else: - with open(USER_CONFIG_PATH, "r") as file: - config = json.load(file) - - if not isinstance(config, dict): - raise InvalidUserConfiguration(f"Invalid config type \"{type(config)}\", expected: \"dict\".") - - return config + username: str + password: str + token: str + tokenExpirationDate: str + refreshToken: str + refreshTokenExpirationDate: str def hasExpired(tokenExpirationDate: Optional[str]) -> bool: @@ -48,54 +44,60 @@ def hasExpired(tokenExpirationDate: Optional[str]) -> bool: return currentDate >= decodeDate(tokenExpirationDate) -class UserConfiguration: +class UserConfiguration(BaseConfiguration): def __init__(self) -> None: - self._raw = initializeConfig() + super().__init__(USER_CONFIG_PATH) + if not "CTX_API_URL" in os.environ: + os.environ["CTX_API_URL"] = self.serverUrl + + @classmethod + def getDefaultConfig(cls) -> Dict[str, Any]: + return USER_DEFAULT_CONFIG @property def username(self) -> str: - return self.getStrValue("username", "CTX_USERNAME") + return self.getValue("username", str, "CTX_USERNAME") @username.setter - def username(self, value: Optional[str]) -> None: + def username(self, value: str) -> None: self._raw["username"] = value @property def password(self) -> str: - return self.getStrValue("password", "CTX_PASSWORD") + return self.getValue("password", str, "CTX_PASSWORD") @password.setter - def password(self, value: Optional[str]) -> None: + def password(self, value: str) -> None: self._raw["password"] = value @property - def token(self) -> str: - return self.getStrValue("token") + def token(self) -> Optional[str]: + return self.getOptValue("token", str) @token.setter def token(self, value: Optional[str]) -> None: self._raw["token"] = value @property - def refreshToken(self) -> str: - return self.getStrValue("refreshToken") + def refreshToken(self) -> Optional[str]: + return self.getOptValue("refreshToken", str) @refreshToken.setter def refreshToken(self, value: Optional[str]) -> None: self._raw["refreshToken"] = value @property - def tokenExpirationDate(self) -> str: - return self.getStrValue("tokenExpirationDate") + def tokenExpirationDate(self) -> Optional[str]: + return self.getOptValue("tokenExpirationDate", str) @tokenExpirationDate.setter def tokenExpirationDate(self, value: Optional[str]) -> None: self._raw["tokenExpirationDate"] = value @property - def refreshTokenExpirationDate(self) -> str: - return self.getStrValue("refreshTokenExpirationDate") + def refreshTokenExpirationDate(self) -> Optional[str]: + return self.getOptValue("refreshTokenExpirationDate", str) @refreshTokenExpirationDate.setter def refreshTokenExpirationDate(self, value: Optional[str]) -> None: @@ -103,15 +105,15 @@ def refreshTokenExpirationDate(self, value: Optional[str]) -> None: @property def serverUrl(self) -> str: - return self.getStrValue("serverUrl", "CTX_API_URL") + return self.getValue("serverUrl", str, "CTX_API_URL") @serverUrl.setter - def serverUrl(self, value: Optional[str]) -> None: + def serverUrl(self, value: str) -> None: self._raw["serverUrl"] = value @property - def projectId(self) -> int: - return self.getIntValue("projectId", "CTX_PROJECT_ID") + def projectId(self) -> Optional[int]: + return self.getOptValue("projectId", int, "CTX_PROJECT_ID") @projectId.setter def projectId(self, value: Optional[int]) -> None: @@ -123,48 +125,37 @@ def isValid(self) -> bool: @property def hasTokenExpired(self) -> bool: - if self._raw.get("token") is None: + if self.token is None: return True - tokenExpirationDate = self._raw.get("tokenExpirationDate") - if tokenExpirationDate is None: + if self.tokenExpirationDate is None: return True - return hasExpired(tokenExpirationDate) + return hasExpired(self.tokenExpirationDate) @property def hasRefreshTokenExpired(self) -> bool: - if self._raw.get("refreshToken") is None: + if self.refreshToken is None: return True - refreshTokenExpirationDate = self._raw.get("refreshTokenExpirationDate") - if refreshTokenExpirationDate is None: + if self.refreshTokenExpirationDate is None: return True - return hasExpired(refreshTokenExpirationDate) - - def save(self) -> None: - with USER_CONFIG_PATH.open("w") as configFile: - json.dump(self.__dict__, configFile, indent = 4) - - def getStrValue(self, configKey: str, envKey: Optional[str] = None) -> str: - if envKey is not None and envKey in os.environ: - return os.environ[envKey] - - value = self._raw.get(configKey) - - if not isinstance(value, str): - raise InvalidUserConfiguration(f"Invalid f{configKey} type \"{type(value)}\", expected: \"str\".") - - return value + return hasExpired(self.refreshTokenExpirationDate) - def getIntValue(self, configKey: str, envKey: Optional[str] = None) -> int: - if envKey is not None and envKey in os.environ: - return int(os.environ[envKey]) + def isUserConfigured(self) -> bool: + # ovako?? + if not isinstance(self._raw.get("username"), str): + return False - value = self._raw.get(configKey) + return True - if not isinstance(value, int): - raise InvalidUserConfiguration(f"Invalid f{configKey} type \"{type(value)}\", expected: \"str\".") + def saveLoginData(self, loginInfo: LoginInfo) -> None: + self.username = loginInfo.username + self.password = loginInfo.password + self.token = loginInfo.token + self.tokenExpirationDate = loginInfo.tokenExpirationDate + self.refreshToken = loginInfo.refreshToken + self.refreshTokenExpirationDate = loginInfo.refreshTokenExpirationDate - return value + self.save() diff --git a/coretex/old_configuration.py b/coretex/old_configuration.py new file mode 100644 index 00000000..fe81000d --- /dev/null +++ b/coretex/old_configuration.py @@ -0,0 +1,137 @@ +# # Copyright (C) 2023 Coretex LLC + +# # This file is part of Coretex.ai + +# # This program is free software: you can redistribute it and/or modify +# # it under the terms of the GNU Affero General Public License as +# # published by the Free Software Foundation, either version 3 of the +# # License, or (at your option) any later version. + +# # This program is distributed in the hope that it will be useful, +# # but WITHOUT ANY WARRANTY; without even the implied warranty of +# # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# # GNU Affero General Public License for more details. + +# # You should have received a copy of the GNU Affero General Public License +# # along with this program. If not, see . + +# from typing import Dict, Any, Optional +# from pathlib import Path + +# import os +# import json +# import sys + + +# def isCliRuntime() -> bool: +# executablePath = sys.argv[0] +# return ( +# executablePath.endswith("/bin/coretex") and +# os.access(executablePath, os.X_OK) +# ) + + +# def getEnvVar(key: str, default: str) -> str: +# if os.environ.get(key) is None: +# return default + +# return os.environ[key] + + +# CONFIG_DIR = Path.home().joinpath(".config", "coretex") +# DEFAULT_CONFIG_PATH = CONFIG_DIR / "config.json" + + +# DEFAULT_CONFIG: Dict[str, Any] = { +# # os.environ used directly here since we don't wanna +# # set those variables to any value if they don't exist +# # in the os.environ the same way we do for properties which +# # call genEnvVar +# "username": os.environ.get("CTX_USERNAME"), +# "password": os.environ.get("CTX_PASSWORD"), +# "token": None, +# "refreshToken": None, +# "serverUrl": getEnvVar("CTX_API_URL", "https://api.coretex.ai/"), +# "storagePath": getEnvVar("CTX_STORAGE_PATH", "~/.coretex") +# } + + +# def loadConfig() -> Dict[str, Any]: +# with DEFAULT_CONFIG_PATH.open("r") as configFile: +# try: +# config: Dict[str, Any] = json.load(configFile) +# except json.JSONDecodeError: +# config = {} + +# for key, value in DEFAULT_CONFIG.items(): +# if not key in config: +# config[key] = value + +# return config + + +# def _syncConfigWithEnv() -> None: +# # If configuration does not exist create default one +# if not DEFAULT_CONFIG_PATH.exists(): +# DEFAULT_CONFIG_PATH.parent.mkdir(parents = True, exist_ok = True) +# config = DEFAULT_CONFIG.copy() +# else: +# config = loadConfig() + +# saveConfig(config) + +# if not "CTX_API_URL" in os.environ: +# os.environ["CTX_API_URL"] = config["serverUrl"] + +# secretsKey = config.get("secretsKey") +# if isinstance(secretsKey, str) and secretsKey != "": +# os.environ["CTX_SECRETS_KEY"] = secretsKey + +# if not isCliRuntime(): +# os.environ["CTX_STORAGE_PATH"] = config["storagePath"] +# else: +# os.environ["CTX_STORAGE_PATH"] = f"{CONFIG_DIR}/data" + + +# def saveConfig(config: Dict[str, Any]) -> None: +# configPath = DEFAULT_CONFIG_PATH.expanduser() +# with configPath.open("w+") as configFile: +# json.dump(config, configFile, indent = 4) + + +# def isUserConfigured(config: Dict[str, Any]) -> bool: +# return ( +# config.get("username") is not None and +# config.get("password") is not None and +# config.get("storagePath") is not None +# ) + + +# def isNodeConfigured(config: Dict[str, Any]) -> bool: +# return ( +# config.get("nodeName") is not None and +# config.get("storagePath") is not None and +# config.get("image") is not None and +# config.get("serverUrl") is not None and +# config.get("nodeAccessToken") is not None and +# config.get("nodeRam") is not None and +# config.get("nodeSwap") is not None and +# config.get("nodeSharedMemory") is not None and +# config.get("nodeMode") is not None +# ) + + +# def getInitScript(config: Dict[str, Any]) -> Optional[Path]: +# value = config.get("initScript") + +# if not isinstance(value, str): +# return None + +# if value == "": +# return None + +# path = Path(value).expanduser().absolute() +# if not path.exists(): +# return None + +# return path diff --git a/coretex/utils/misc.py b/coretex/utils/misc.py new file mode 100644 index 00000000..9df00d44 --- /dev/null +++ b/coretex/utils/misc.py @@ -0,0 +1,27 @@ +# Copyright (C) 2023 Coretex LLC + +# This file is part of Coretex.ai + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +import sys + + +def isCliRuntime() -> bool: + executablePath = sys.argv[0] + return ( + executablePath.endswith("/bin/coretex") and + os.access(executablePath, os.X_OK) + ) diff --git a/coretex/utils/process.py b/coretex/utils/process.py index 9747e306..e3515ea6 100644 --- a/coretex/utils/process.py +++ b/coretex/utils/process.py @@ -21,10 +21,8 @@ import logging import subprocess -from ..entities import LogSeverity - -def logProcessOutput(output: bytes, severity: LogSeverity) -> None: +def logProcessOutput(output: bytes, severity: int) -> None: decoded = output.decode("UTF-8") for line in decoded.split("\n"): @@ -33,7 +31,7 @@ def logProcessOutput(output: bytes, severity: LogSeverity) -> None: continue # ignoring type for now, has to be fixed in coretexpylib - logging.getLogger("coretexpylib").log(severity.stdSeverity, line) + logging.getLogger("coretexpylib").log(severity, line) class CommandException(Exception): @@ -73,7 +71,7 @@ def command( line = stdout.readline() stdOutStr = "\n".join([stdOutStr, line.decode("utf-8")]) if not ignoreStdout: - logProcessOutput(line, LogSeverity.info) + logProcessOutput(line, logging.INFO) if stderr is None and not ignoreStderr: commandArgs = " ".join(args) @@ -83,7 +81,7 @@ def command( lines = stderr.readlines() stdErrStr = b"".join(lines).decode("utf-8") if not ignoreStderr: - logProcessOutput(b"".join(lines), LogSeverity.warning if returnCode == 0 else LogSeverity.fatal) + logProcessOutput(b"".join(lines), logging.WARNING if returnCode == 0 else logging.FATAL) if returnCode != 0 and check: commandArgs = " ".join(args) From 2092ecf7c60ea4c74ba2e1f8925e11deb0cd1e3b Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Fri, 29 Mar 2024 15:18:17 +0100 Subject: [PATCH 04/28] CTX-5430: Refactored rest of the commands. Saving changes. --- coretex/cli/commands/model.py | 54 ++++++++++++++-------------- coretex/cli/commands/project.py | 8 ++--- coretex/cli/main.py | 4 +-- coretex/cli/modules/project_utils.py | 8 ++--- coretex/cli/modules/user.py | 2 +- coretex/configuration/base.py | 17 +++++++++ coretex/configuration/node.py | 45 +++++++++++++++++++++-- coretex/configuration/user.py | 23 ++++++++++-- 8 files changed, 118 insertions(+), 43 deletions(-) diff --git a/coretex/cli/commands/model.py b/coretex/cli/commands/model.py index 7962ed8b..0b7a257e 100644 --- a/coretex/cli/commands/model.py +++ b/coretex/cli/commands/model.py @@ -1,39 +1,39 @@ -# from typing import Optional +from typing import Optional -# import click +import click -# from ..modules import project_utils, user, utils, ui -# from ...entities import Model -# from ...configuration import loadConfig +from ..modules import project_utils, user, utils, ui +from ...entities import Model +from ...configuration import UserConfiguration -# @click.command() -# @click.argument("path", type = click.Path(exists = True, file_okay = False, dir_okay = True)) -# @click.option("-n", "--name", type = str, required = True) -# @click.option("-p", "--project", type = str, required = False, default = None) -# @click.option("-a", "--accuracy", type = click.FloatRange(0, 1), required = False, default = 1) -# def create(name: str, path: str, project: Optional[str], accuracy: float) -> None: -# config = loadConfig() +@click.command() +@click.argument("path", type = click.Path(exists = True, file_okay = False, dir_okay = True)) +@click.option("-n", "--name", type = str, required = True) +@click.option("-p", "--project", type = str, required = False, default = None) +@click.option("-a", "--accuracy", type = click.FloatRange(0, 1), required = False, default = 1) +def create(name: str, path: str, project: Optional[str], accuracy: float) -> None: + userConfig = UserConfiguration() -# # If project was provided used that, otherwise get the one from config -# # If project that was provided does not exist prompt user to create a new -# # one with that name -# ctxProject = project_utils.getProject(project, config) -# if ctxProject is None: -# return + # If project was provided used that, otherwise get the one from config + # If project that was provided does not exist prompt user to create a new + # one with that name + ctxProject = project_utils.getProject(project, userConfig) + if ctxProject is None: + return -# ui.progressEcho("Creating the model...") + ui.progressEcho("Creating the model...") -# model = Model.createProjectModel(name, ctxProject.id, accuracy) -# model.upload(path) + model = Model.createProjectModel(name, ctxProject.id, accuracy) + model.upload(path) -# ui.successEcho(f"Model \"{model.name}\" created successfully") + ui.successEcho(f"Model \"{model.name}\" created successfully") -# @click.group() -# @utils.onBeforeCommandExecute(user.initializeUserSession) -# def model() -> None: -# pass +@click.group() +@utils.onBeforeCommandExecute(user.initializeUserSession) +def model() -> None: + pass -# model.add_command(create) +model.add_command(create) diff --git a/coretex/cli/commands/project.py b/coretex/cli/commands/project.py index d4ef0692..d081a23f 100644 --- a/coretex/cli/commands/project.py +++ b/coretex/cli/commands/project.py @@ -22,13 +22,13 @@ from ..modules import ui, project_utils, utils, user from ...entities import Project from ...networking import NetworkRequestError -from ...configuration import loadConfig, saveConfig +from ...configuration import UserConfiguration def selectProject(projectId: int) -> None: - config = loadConfig() - config["projectId"] = projectId - saveConfig(config) + nodeConfig = UserConfiguration() + nodeConfig.projectId = projectId + nodeConfig.save() @click.command() diff --git a/coretex/cli/main.py b/coretex/cli/main.py index 0b2ad40c..9e807dcb 100644 --- a/coretex/cli/main.py +++ b/coretex/cli/main.py @@ -20,7 +20,7 @@ from .commands.login import login # from .commands.model import model from .commands.node import node -# from .commands.project import project +from .commands.project import project from .modules.intercept import ClickExceptionInterceptor @@ -31,5 +31,5 @@ def cli() -> None: cli.add_command(login) # cli.add_command(model) -# cli.add_command(project) +cli.add_command(project) cli.add_command(node) diff --git a/coretex/cli/modules/project_utils.py b/coretex/cli/modules/project_utils.py index 075cb48f..38ebbdde 100644 --- a/coretex/cli/modules/project_utils.py +++ b/coretex/cli/modules/project_utils.py @@ -5,6 +5,7 @@ from . import ui from ...entities import Project, ProjectType from ...networking import EntityNotCreated +from ...configuration import UserConfiguration def selectProjectType() -> ProjectType: @@ -38,16 +39,15 @@ def promptProjectCreate(message: str, name: str) -> Optional[Project]: raise click.ClickException(f"Failed to create project \"{name}\".") -def getProject(name: Optional[str], config: Dict[str, Any]) -> Optional[Project]: +def getProject(name: Optional[str], config: UserConfiguration) -> Optional[Project]: if name is not None: try: return Project.fetchOne(name = name) except: return promptProjectCreate("Project not found. Do you want to create a new Project with that name?", name) - projectId = config.get("projectId") - if projectId is not None: - return Project.fetchById(projectId) + if config.projectId is not None: + return Project.fetchById(config.projectId) # Generic message on how to specify the Project raise click.ClickException( diff --git a/coretex/cli/modules/user.py b/coretex/cli/modules/user.py index c0dec9f4..77551f2f 100644 --- a/coretex/cli/modules/user.py +++ b/coretex/cli/modules/user.py @@ -65,7 +65,7 @@ def authenticate(retryCount: int = 0) -> LoginInfo: def initializeUserSession() -> None: config = UserConfiguration() - if config.username is None or config.password is None: + if config.isUserConfigured(): errorEcho("User configuration not found. Please authenticate with your credentials.") loginInfo = authenticate() config.saveLoginData(loginInfo) diff --git a/coretex/configuration/base.py b/coretex/configuration/base.py index 7e515254..36f0107b 100644 --- a/coretex/configuration/base.py +++ b/coretex/configuration/base.py @@ -1,3 +1,20 @@ +# Copyright (C) 2023 Coretex LLC + +# This file is part of Coretex.ai + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from typing import Dict, Any, Optional, Type, TypeVar from abc import abstractmethod from pathlib import Path diff --git a/coretex/configuration/node.py b/coretex/configuration/node.py index d017c91b..f786cbe1 100644 --- a/coretex/configuration/node.py +++ b/coretex/configuration/node.py @@ -1,4 +1,21 @@ -from typing import Dict, Any, Optional, TypeVar +# Copyright (C) 2023 Coretex LLC + +# This file is part of Coretex.ai + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from typing import Dict, Any, Optional from pathlib import Path import os @@ -7,8 +24,6 @@ from .base import BaseConfiguration, CONFIG_DIR - - def getEnvVar(key: str, default: str) -> str: if os.environ.get(key) is None: return default @@ -176,6 +191,30 @@ def modelId(self, value: Optional[int]) -> None: self._raw["modelId"] = value def isNodeConfigured(self) -> bool: + if self._raw.get("nodeName") is not None and isinstance("nodeName", str): + return True + + if self._raw.get("password") is not None and isinstance("password", str): + return True + + if self._raw.get("image") is not None and isinstance("image", str): + return True + + if self._raw.get("nodeAccessToken") is not None and isinstance("nodeAccessToken", str): + return True + + if self._raw.get("nodeRam") is not None and isinstance("nodeRam", int): + return True + + if self._raw.get("nodeSwap") is not None and isinstance("nodeSwap", int): + return True + + if self._raw.get("nodeSharedMemory") is not None and isinstance("nodeSharedMemory", int): + return True + + if self._raw.get("nodeMode") is not None and isinstance("nodeMode", int): + return True + return False def getInitScriptPath(self) -> Optional[Path]: diff --git a/coretex/configuration/user.py b/coretex/configuration/user.py index b6005dea..31c4c2d7 100644 --- a/coretex/configuration/user.py +++ b/coretex/configuration/user.py @@ -1,3 +1,20 @@ +# Copyright (C) 2023 Coretex LLC + +# This file is part of Coretex.ai + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from typing import Dict, Any, Optional from datetime import datetime, timezone from dataclasses import dataclass @@ -144,8 +161,10 @@ def hasRefreshTokenExpired(self) -> bool: return hasExpired(self.refreshTokenExpirationDate) def isUserConfigured(self) -> bool: - # ovako?? - if not isinstance(self._raw.get("username"), str): + if self._raw.get("username") is None or not isinstance(self._raw.get("username"), str): + return False + + if self._raw.get("password") is None or not isinstance(self._raw.get("password"), str): return False return True From d885bc228649a74a9f31081e4669df99c6b224ca Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 17 Apr 2024 09:19:10 +0200 Subject: [PATCH 05/28] CTX-5430: Finished merge of develop into branch. This should fix all the linter errors. --- coretex/cli/modules/node.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index 8f1a5aa2..330ff026 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -287,7 +287,7 @@ def configureNode(config: NodeConfiguration, verbose: bool) -> None: config.cpuCount = config_defaults.DEFAULT_CPU_COUNT if config_defaults.DEFAULT_CPU_COUNT is not None else 0 # is this fine? config.nodeMode = config_defaults.DEFAULT_NODE_MODE config.allowDocker = config_defaults.DEFAULT_ALLOW_DOCKER - config.secretsKey = config_defaults.DEFAULT_SECRETS_KEY + config.nodeSecret = config_defaults.DEFAULT_NODE_SECRET config.initScript = config_defaults.DEFAULT_INIT_SCRIPT publicKey: Optional[bytes] = None @@ -299,7 +299,7 @@ def configureNode(config: NodeConfiguration, verbose: bool) -> None: config.nodeSharedMemory = clickPrompt("Node POSIX shared memory limit in GB (press enter to use default)", config_defaults.DEFAULT_SHARED_MEMORY, type = int) config.cpuCount = clickPrompt("Enter the number of CPUs the container will use (press enter to use default)", config_defaults.DEFAULT_CPU_COUNT, type = int) config.allowDocker = clickPrompt("Allow Node to access system docker? This is a security risk! (Y/n)", config_defaults.DEFAULT_ALLOW_DOCKER, type = bool) - config.secretsKey = clickPrompt("Enter a key used for decrypting your Coretex Secrets", config_defaults.DEFAULT_SECRETS_KEY, type = str, hide_input = True) + config.nodeSecret = clickPrompt("Enter a key used for decrypting your Coretex Secrets", config_defaults.DEFAULT_NODE_SECRET, type = str, hide_input = True) config.initScript = _configureInitScript() nodeMode, modelId = selectNodeMode(config.storagePath) @@ -308,7 +308,7 @@ def configureNode(config: NodeConfiguration, verbose: bool) -> None: config.modelId = modelId nodeSecret: str = clickPrompt("Enter a secret which will be used to generate RSA key-pair for Node", config_defaults.DEFAULT_NODE_SECRET, type = str, hide_input = True) - config["nodeSecret"] = nodeSecret + config.nodeSecret = nodeSecret if nodeSecret != config_defaults.DEFAULT_NODE_SECRET: progressEcho("Generating RSA key-pair (2048 bits long) using provided node secret...") @@ -317,7 +317,7 @@ def configureNode(config: NodeConfiguration, verbose: bool) -> None: else: stdEcho("To configure node manually run coretex node config with --verbose flag.") - config["nodeAccessToken"] = registerNode(config["nodeName"], publicKey) + config.nodeAccessToken = registerNode(config.nodeName, publicKey) def initializeNodeConfiguration() -> None: From c81cd6a303f0343296fcdadd4ddb5846ebe24e5d Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 17 Apr 2024 10:08:14 +0200 Subject: [PATCH 06/28] CTX-5430: Cleanup changes saving. --- coretex/__init__.py | 3 +- coretex/cli/commands/login.py | 11 ++-- coretex/cli/commands/node.py | 54 +++++++++---------- coretex/cli/modules/intercept.py | 2 - coretex/cli/modules/node.py | 90 ++++++++++++++++++------------- coretex/cli/modules/ui.py | 3 +- coretex/cli/modules/update.py | 21 -------- coretex/cli/modules/user.py | 14 ++--- coretex/configuration/__init__.py | 9 ++-- coretex/configuration/node.py | 10 +--- coretex/utils/__init__.py | 1 + 11 files changed, 102 insertions(+), 116 deletions(-) diff --git a/coretex/__init__.py b/coretex/__init__.py index 4f711967..e323e2c9 100644 --- a/coretex/__init__.py +++ b/coretex/__init__.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -print('sync1') # Internal - not for outside use from .configuration import _syncConfigWithEnv _syncConfigWithEnv() @@ -23,7 +22,7 @@ # Internal - not for outside use from ._logger import _initializeDefaultLogger, _initializeCLILogger -from .utils.misc import isCliRuntime +from .utils import isCliRuntime if isCliRuntime(): diff --git a/coretex/cli/commands/login.py b/coretex/cli/commands/login.py index 3f6e10de..d713921c 100644 --- a/coretex/cli/commands/login.py +++ b/coretex/cli/commands/login.py @@ -17,8 +17,7 @@ import click -from ..modules.user import authenticate -from ..modules.ui import clickPrompt, stdEcho, successEcho +from ..modules import user, ui from ...configuration import UserConfiguration @@ -26,7 +25,7 @@ def login() -> None: config = UserConfiguration() if config.isUserConfigured(): - if not clickPrompt( + if not ui.clickPrompt( f"User already logged in with username {config.username}.\nWould you like to log in with a different user (Y/n)?", type = bool, default = True, @@ -34,8 +33,8 @@ def login() -> None: ): return - stdEcho("Please enter your credentials:") - loginInfo = authenticate() + ui.stdEcho("Please enter your credentials:") + loginInfo = user.authenticate() config.saveLoginData(loginInfo) - successEcho(f"User {config.username} successfully logged in.") + ui.successEcho(f"User {config.username} successfully logged in.") diff --git a/coretex/cli/commands/node.py b/coretex/cli/commands/node.py index e72062d5..e978815a 100644 --- a/coretex/cli/commands/node.py +++ b/coretex/cli/commands/node.py @@ -19,21 +19,19 @@ import click +from ...utils import docker from ..modules import node as node_module -from ..modules.ui import clickPrompt, successEcho, errorEcho, previewConfig -from ..modules.update import NodeStatus, getNodeStatus, activateAutoUpdate, dumpScript, UPDATE_SCRIPT_NAME -from ..modules.utils import onBeforeCommandExecute -from ..modules.user import initializeUserSession +from ..modules import update as update_module +from ..modules import ui, user, utils from ...configuration import UserConfiguration, NodeConfiguration, CONFIG_DIR -from ...utils import docker @click.command() @click.option("--image", type = str, help = "Docker image url") -@onBeforeCommandExecute(node_module.initializeNodeConfiguration) +@utils.onBeforeCommandExecute(node_module.initializeNodeConfiguration) def start(image: Optional[str]) -> None: if node_module.isRunning(): - if not clickPrompt( + if not ui.clickPrompt( "Node is already running. Do you wish to restart the Node? (Y/n)", type = bool, default = True, @@ -60,36 +58,36 @@ def start(image: Optional[str]) -> None: node_module.start(dockerImage, userConfig, nodeConfig) - activateAutoUpdate(CONFIG_DIR, userConfig, nodeConfig) + update_module.activateAutoUpdate(CONFIG_DIR, userConfig, nodeConfig) @click.command() def stop() -> None: if not node_module.isRunning(): - errorEcho("Node is already offline.") + ui.errorEcho("Node is already offline.") return node_module.stop() @click.command() -@onBeforeCommandExecute(node_module.initializeNodeConfiguration) +@utils.onBeforeCommandExecute(node_module.initializeNodeConfiguration) def update() -> None: userConfig = UserConfiguration() nodeConfig = NodeConfiguration() - nodeStatus = getNodeStatus() + nodeStatus = node_module.getNodeStatus() - if nodeStatus == NodeStatus.inactive: - errorEcho("Node is not running. To update Node you need to start it first.") + if nodeStatus == node_module.NodeStatus.inactive: + ui.errorEcho("Node is not running. To update Node you need to start it first.") return - if nodeStatus == NodeStatus.reconnecting: - errorEcho("Node is reconnecting. Cannot update now.") + if nodeStatus == node_module.NodeStatus.reconnecting: + ui.errorEcho("Node is reconnecting. Cannot update now.") return - if nodeStatus == NodeStatus.busy: - if not clickPrompt( + if nodeStatus == node_module.NodeStatus.busy: + if not ui.clickPrompt( "Node is busy, do you wish to terminate the current execution to perform the update? (Y/n)", type = bool, default = True, @@ -100,13 +98,13 @@ def update() -> None: node_module.stop() if not node_module.shouldUpdate(nodeConfig.image): - successEcho("Node is already up to date.") + ui.successEcho("Node is already up to date.") return node_module.pull(nodeConfig.image) - if getNodeStatus() == NodeStatus.busy: - if not clickPrompt( + if node_module.getNodeStatus() == node_module.NodeStatus.busy: + if not ui.clickPrompt( "Node is busy, do you wish to terminate the current execution to perform the update? (Y/n)", type = bool, default = True, @@ -123,13 +121,13 @@ def update() -> None: @click.option("--verbose", is_flag = True, help = "Configure node settings manually.") def config(verbose: bool) -> None: if node_module.isRunning(): - if not clickPrompt( + if not ui.clickPrompt( "Node is already running. Do you wish to stop the Node? (Y/n)", type = bool, default = True, show_default = False ): - errorEcho("If you wish to reconfigure your node, use coretex node stop commands first.") + ui.errorEcho("If you wish to reconfigure your node, use coretex node stop commands first.") return node_module.stop() @@ -138,7 +136,7 @@ def config(verbose: bool) -> None: nodeConfig = NodeConfiguration() if nodeConfig.isNodeConfigured(): - if not clickPrompt( + if not ui.clickPrompt( "Node configuration already exists. Would you like to update? (Y/n)", type = bool, default = True, @@ -148,17 +146,17 @@ def config(verbose: bool) -> None: node_module.configureNode(nodeConfig, verbose) nodeConfig.save() - previewConfig(userConfig, nodeConfig) + ui.previewConfig(userConfig, nodeConfig) # Updating auto-update script since node configuration is changed - dumpScript(CONFIG_DIR / UPDATE_SCRIPT_NAME, userConfig, nodeConfig) + update_module.dumpScript(CONFIG_DIR / update_module.UPDATE_SCRIPT_NAME, userConfig, nodeConfig) - successEcho("Node successfully configured.") + ui.successEcho("Node successfully configured.") @click.group() -@onBeforeCommandExecute(docker.isDockerAvailable) -@onBeforeCommandExecute(initializeUserSession) +@utils.onBeforeCommandExecute(docker.isDockerAvailable) +@utils.onBeforeCommandExecute(user.initializeUserSession) def node() -> None: pass diff --git a/coretex/cli/modules/intercept.py b/coretex/cli/modules/intercept.py index 5c24f621..020fc42d 100644 --- a/coretex/cli/modules/intercept.py +++ b/coretex/cli/modules/intercept.py @@ -21,8 +21,6 @@ import click -from .ui import errorEcho - class ClickExceptionInterceptor(click.Group): diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index 330ff026..f8faaf67 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -15,16 +15,15 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Any, Dict, Tuple, Optional -from enum import Enum +from typing import Tuple, Optional +from enum import Enum, IntEnum from pathlib import Path from base64 import b64encode import logging +import requests -from . import config_defaults -from .utils import isGPUAvailable -from .ui import clickPrompt, arrowPrompt, highlightEcho, errorEcho, progressEcho, successEcho, stdEcho +from . import config_defaults, utils, ui from .node_mode import NodeMode from ...cryptography import rsa from ...networking import networkManager, NetworkRequestError @@ -32,10 +31,20 @@ from ...entities.model import Model from ...configuration import UserConfiguration, NodeConfiguration + class NodeException(Exception): pass +class NodeStatus(IntEnum): + + inactive = 1 + active = 2 + busy = 3 + deleted = 4 + reconnecting = 5 + + class ImageType(Enum): official = "official" @@ -44,9 +53,9 @@ class ImageType(Enum): def pull(image: str) -> None: try: - progressEcho(f"Fetching image {image}...") + ui.progressEcho(f"Fetching image {image}...") docker.imagePull(image) - successEcho(f"Image {image} successfully fetched.") + ui.successEcho(f"Image {image} successfully fetched.") except BaseException as ex: logging.getLogger("cli").debug(ex, exc_info = ex) raise NodeException("Failed to fetch latest node version.") @@ -62,7 +71,7 @@ def exists() -> bool: def start(dockerImage: str, userConfig: UserConfiguration, nodeConfig: NodeConfiguration) -> None: try: - progressEcho("Starting Coretex Node...") + ui.progressEcho("Starting Coretex Node...") docker.createNetwork(config_defaults.DOCKER_CONTAINER_NETWORK) environ = { @@ -102,7 +111,7 @@ def start(dockerImage: str, userConfig: UserConfiguration, nodeConfig: NodeConfi volumes ) - successEcho("Successfully started Coretex Node.") + ui.successEcho("Successfully started Coretex Node.") except BaseException as ex: logging.getLogger("cli").debug(ex, exc_info = ex) raise NodeException("Failed to start Coretex Node.") @@ -119,15 +128,24 @@ def clean() -> None: def stop() -> None: try: - progressEcho("Stopping Coretex Node...") + ui.progressEcho("Stopping Coretex Node...") docker.stopContainer(config_defaults.DOCKER_CONTAINER_NAME) clean() - successEcho("Successfully stopped Coretex Node....") + ui.successEcho("Successfully stopped Coretex Node....") except BaseException as ex: logging.getLogger("cli").debug(ex, exc_info = ex) raise NodeException("Failed to stop Coretex Node.") +def getNodeStatus() -> NodeStatus: + try: + response = requests.get(f"http://localhost:21000/status", timeout = 1) + status = response.json()["status"] + return NodeStatus(status) + except: + return NodeStatus.inactive + + def getRepoFromImageUrl(image: str) -> str: imageName = image.split("/")[-1] if not ":" in imageName: @@ -200,7 +218,7 @@ def selectImageType() -> ImageType: } choices = list(availableImages.keys()) - selectedImage = arrowPrompt(choices, "Please select image that you want to use (use arrow keys to select an option):") + selectedImage = ui.arrowPrompt(choices, "Please select image that you want to use (use arrow keys to select an option):") return availableImages[selectedImage] @@ -209,12 +227,12 @@ def selectModelId(storagePath: str, retryCount: int = 0) -> int: if retryCount >= 3: raise RuntimeError("Failed to fetch Coretex Model. Terminating...") - modelId: int = clickPrompt("Specify Coretex Model ID that you want to use:", type = int) + modelId: int = ui.clickPrompt("Specify Coretex Model ID that you want to use:", type = int) try: model = Model.fetchById(modelId) except: - errorEcho(f"Failed to fetch model with id {modelId}.") + ui.errorEcho(f"Failed to fetch model with id {modelId}.") return selectModelId(storagePath, retryCount + 1) modelDir = Path(storagePath) / "models" @@ -232,7 +250,7 @@ def selectNodeMode(storagePath: str) -> Tuple[int, Optional[int]]: } choices = list(availableNodeModes.keys()) - selectedMode = arrowPrompt(choices, "Please select Coretex Node mode (use arrow keys to select an option):") + selectedMode = ui.arrowPrompt(choices, "Please select Coretex Node mode (use arrow keys to select an option):") if availableNodeModes[selectedMode] == NodeMode.functionExclusive: modelId = selectModelId(storagePath) @@ -242,7 +260,7 @@ def selectNodeMode(storagePath: str) -> Tuple[int, Optional[int]]: def _configureInitScript() -> str: - initScript = clickPrompt("Enter a path to sh script which will be executed before Node starts", config_defaults.DEFAULT_INIT_SCRIPT, type = str) + initScript = ui.clickPrompt("Enter a path to sh script which will be executed before Node starts", config_defaults.DEFAULT_INIT_SCRIPT, type = str) if initScript == config_defaults.DEFAULT_INIT_SCRIPT: return config_defaults.DEFAULT_INIT_SCRIPT @@ -250,29 +268,29 @@ def _configureInitScript() -> str: path = Path(initScript).expanduser().absolute() if path.is_dir(): - errorEcho("Provided path is pointing to a directory, file expected!") + ui.errorEcho("Provided path is pointing to a directory, file expected!") return _configureInitScript() if not path.exists(): - errorEcho("Provided file does not exist!") + ui.errorEcho("Provided file does not exist!") return _configureInitScript() return str(path) def configureNode(config: NodeConfiguration, verbose: bool) -> None: - highlightEcho("[Node Configuration]") - config.nodeName = clickPrompt("Node name", type = str) + ui.highlightEcho("[Node Configuration]") + config.nodeName = ui.clickPrompt("Node name", type = str) config.nodeAccessToken = registerNode(config.nodeName) imageType = selectImageType() if imageType == ImageType.custom: - config.image = clickPrompt("Specify URL of docker image that you want to use:", type = str) + config.image = ui.clickPrompt("Specify URL of docker image that you want to use:", type = str) else: config.image = "coretexai/coretex-node" - if isGPUAvailable(): - config.allowGpu = clickPrompt("Do you want to allow the Node to access your GPU? (Y/n)", type = bool, default = True) + if utils.isGPUAvailable(): + config.allowGpu = ui.clickPrompt("Do you want to allow the Node to access your GPU? (Y/n)", type = bool, default = True) else: config.allowGpu = False @@ -293,13 +311,13 @@ def configureNode(config: NodeConfiguration, verbose: bool) -> None: publicKey: Optional[bytes] = None if verbose: - config.storagePath = clickPrompt("Storage path (press enter to use default)", config_defaults.DEFAULT_STORAGE_PATH, type = str) - config.nodeRam = clickPrompt("Node RAM memory limit in GB (press enter to use default)", config_defaults.DEFAULT_RAM_MEMORY, type = int) - config.nodeSwap = clickPrompt("Node swap memory limit in GB, make sure it is larger than mem limit (press enter to use default)", config_defaults.DEFAULT_SWAP_MEMORY, type = int) - config.nodeSharedMemory = clickPrompt("Node POSIX shared memory limit in GB (press enter to use default)", config_defaults.DEFAULT_SHARED_MEMORY, type = int) - config.cpuCount = clickPrompt("Enter the number of CPUs the container will use (press enter to use default)", config_defaults.DEFAULT_CPU_COUNT, type = int) - config.allowDocker = clickPrompt("Allow Node to access system docker? This is a security risk! (Y/n)", config_defaults.DEFAULT_ALLOW_DOCKER, type = bool) - config.nodeSecret = clickPrompt("Enter a key used for decrypting your Coretex Secrets", config_defaults.DEFAULT_NODE_SECRET, type = str, hide_input = True) + config.storagePath = ui.clickPrompt("Storage path (press enter to use default)", config_defaults.DEFAULT_STORAGE_PATH, type = str) + config.nodeRam = ui.clickPrompt("Node RAM memory limit in GB (press enter to use default)", config_defaults.DEFAULT_RAM_MEMORY, type = int) + config.nodeSwap = ui.clickPrompt("Node swap memory limit in GB, make sure it is larger than mem limit (press enter to use default)", config_defaults.DEFAULT_SWAP_MEMORY, type = int) + config.nodeSharedMemory = ui.clickPrompt("Node POSIX shared memory limit in GB (press enter to use default)", config_defaults.DEFAULT_SHARED_MEMORY, type = int) + config.cpuCount = ui.clickPrompt("Enter the number of CPUs the container will use (press enter to use default)", config_defaults.DEFAULT_CPU_COUNT, type = int) + config.allowDocker = ui.clickPrompt("Allow Node to access system docker? This is a security risk! (Y/n)", config_defaults.DEFAULT_ALLOW_DOCKER, type = bool) + config.nodeSecret = ui.clickPrompt("Enter a key used for decrypting your Coretex Secrets", config_defaults.DEFAULT_NODE_SECRET, type = str, hide_input = True) config.initScript = _configureInitScript() nodeMode, modelId = selectNodeMode(config.storagePath) @@ -307,15 +325,15 @@ def configureNode(config: NodeConfiguration, verbose: bool) -> None: if modelId is not None: config.modelId = modelId - nodeSecret: str = clickPrompt("Enter a secret which will be used to generate RSA key-pair for Node", config_defaults.DEFAULT_NODE_SECRET, type = str, hide_input = True) + nodeSecret: str = ui.clickPrompt("Enter a secret which will be used to generate RSA key-pair for Node", config_defaults.DEFAULT_NODE_SECRET, type = str, hide_input = True) config.nodeSecret = nodeSecret if nodeSecret != config_defaults.DEFAULT_NODE_SECRET: - progressEcho("Generating RSA key-pair (2048 bits long) using provided node secret...") + ui.progressEcho("Generating RSA key-pair (2048 bits long) using provided node secret...") rsaKey = rsa.generateKey(2048, nodeSecret.encode("utf-8")) publicKey = rsa.getPublicKeyBytes(rsaKey.public_key()) else: - stdEcho("To configure node manually run coretex node config with --verbose flag.") + ui.stdEcho("To configure node manually run coretex node config with --verbose flag.") config.nodeAccessToken = registerNode(config.nodeName, publicKey) @@ -326,9 +344,9 @@ def initializeNodeConfiguration() -> None: if config.isNodeConfigured(): return - errorEcho("Node configuration not found.") + ui.errorEcho("Node configuration not found.") if isRunning(): - stopNode = clickPrompt( + stopNode = ui.clickPrompt( "Node is already running. Do you wish to stop the Node? (Y/n)", type = bool, default = True, @@ -336,7 +354,7 @@ def initializeNodeConfiguration() -> None: ) if not stopNode: - errorEcho("If you wish to reconfigure your node, use \"coretex node stop\" command first.") + ui.errorEcho("If you wish to reconfigure your node, use \"coretex node stop\" command first.") return stop() diff --git a/coretex/cli/modules/ui.py b/coretex/cli/modules/ui.py index f69bc783..611c260b 100644 --- a/coretex/cli/modules/ui.py +++ b/coretex/cli/modules/ui.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Any, List, Dict, Optional, Union +from typing import Any, List, Optional, Union from tabulate import tabulate @@ -23,7 +23,6 @@ import inquirer from .node_mode import NodeMode -from .config_defaults import DEFAULT_CPU_COUNT from ...configuration import UserConfiguration, NodeConfiguration diff --git a/coretex/cli/modules/update.py b/coretex/cli/modules/update.py index 3ad29ae3..c4f1ea95 100644 --- a/coretex/cli/modules/update.py +++ b/coretex/cli/modules/update.py @@ -15,11 +15,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from enum import IntEnum from pathlib import Path -import requests - from . import config_defaults from .cron import jobExists, scheduleJob from .node import getRepoFromImageUrl, getTagFromImageUrl @@ -31,15 +28,6 @@ UPDATE_SCRIPT_NAME = "ctx_node_update.sh" -class NodeStatus(IntEnum): - - inactive = 1 - active = 2 - busy = 3 - deleted = 4 - reconnecting = 5 - - def generateUpdateScript(userConfig: UserConfiguration, nodeConfig: NodeConfiguration) -> str: _, dockerPath, _ = command(["which", "docker"], ignoreStdout = True, ignoreStderr = True) bashScriptTemplatePath = RESOURCES_DIR / "update_script_template.sh" @@ -85,12 +73,3 @@ def activateAutoUpdate(configDir: Path, userConfig: UserConfiguration, nodeConfi if not jobExists(UPDATE_SCRIPT_NAME): scheduleJob(configDir, UPDATE_SCRIPT_NAME) - - -def getNodeStatus() -> NodeStatus: - try: - response = requests.get(f"http://localhost:21000/status", timeout = 1) - status = response.json()["status"] - return NodeStatus(status) - except: - return NodeStatus.inactive diff --git a/coretex/cli/modules/user.py b/coretex/cli/modules/user.py index f5ce80f2..ab96ef9e 100644 --- a/coretex/cli/modules/user.py +++ b/coretex/cli/modules/user.py @@ -17,10 +17,10 @@ from datetime import datetime, timezone -from .ui import clickPrompt, errorEcho, progressEcho +from . import ui from ...utils import decodeDate from ...networking import networkManager, NetworkResponse, NetworkRequestError -from ...configuration import LoginInfo, UserConfiguration +from ...configuration import UserConfiguration, LoginInfo def authenticateUser(username: str, password: str) -> NetworkResponse: @@ -40,14 +40,14 @@ def authenticate(retryCount: int = 0) -> LoginInfo: if retryCount >= 3: raise RuntimeError("Failed to authenticate. Terminating...") - username = clickPrompt("Email", type = str) - password = clickPrompt("Password", type = str, hide_input = True) + username = ui.clickPrompt("Email", type = str) + password = ui.clickPrompt("Password", type = str, hide_input = True) - progressEcho("Authenticating...") + ui.progressEcho("Authenticating...") response = networkManager.authenticate(username, password, False) if response.hasFailed(): - errorEcho("Failed to authenticate. Please try again...") + ui.errorEcho("Failed to authenticate. Please try again...") return authenticate(retryCount + 1) jsonResponse = response.getJson(dict) @@ -66,7 +66,7 @@ def initializeUserSession() -> None: config = UserConfiguration() if not config.isUserConfigured(): - errorEcho("User configuration not found. Please authenticate with your credentials.") + ui.errorEcho("User configuration not found. Please authenticate with your credentials.") loginInfo = authenticate() config.saveLoginData(loginInfo) else: diff --git a/coretex/configuration/__init__.py b/coretex/configuration/__init__.py index 2d985d90..b5b480f8 100644 --- a/coretex/configuration/__init__.py +++ b/coretex/configuration/__init__.py @@ -1,10 +1,13 @@ -from .base import CONFIG_DIR from .user import UserConfiguration, LoginInfo from .node import NodeConfiguration +from .base import CONFIG_DIR def _syncConfigWithEnv() -> None: - # TODO: explain this ! - print("sync") + # If configuration doesn't exist default one will be created + # Initialization of User and Node Configuration classes will do + # the necessary sync between config properties and corresponding + # environment variables (e.g. storagePath -> CTX_STORAGE_PATH) + UserConfiguration() NodeConfiguration() diff --git a/coretex/configuration/node.py b/coretex/configuration/node.py index a777285d..609f2c77 100644 --- a/coretex/configuration/node.py +++ b/coretex/configuration/node.py @@ -19,9 +19,9 @@ from pathlib import Path import os -import sys from .base import BaseConfiguration, CONFIG_DIR +from ..utils import isCliRuntime def getEnvVar(key: str, default: str) -> str: @@ -49,14 +49,6 @@ def getEnvVar(key: str, default: str) -> str: } -def isCliRuntime() -> bool: - executablePath = sys.argv[0] - return ( - executablePath.endswith("/bin/coretex") and - os.access(executablePath, os.X_OK) - ) - - class InvalidNodeConfiguration(Exception): pass diff --git a/coretex/utils/__init__.py b/coretex/utils/__init__.py index 64b2ecc2..8d9a3af6 100644 --- a/coretex/utils/__init__.py +++ b/coretex/utils/__init__.py @@ -22,3 +22,4 @@ from .image import resizeWithPadding, cropToWidth from .process import logProcessOutput, command, CommandException from .logs import createFileHandler +from .misc import isCliRuntime From 521d7d64df76feb1a0001b1cdd04c1d408aaa3c6 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 17 Apr 2024 11:41:58 +0200 Subject: [PATCH 07/28] CTX-5430: Saving changes. --- coretex/configuration/node.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/coretex/configuration/node.py b/coretex/configuration/node.py index 609f2c77..4e9e2f65 100644 --- a/coretex/configuration/node.py +++ b/coretex/configuration/node.py @@ -183,31 +183,32 @@ def modelId(self, value: Optional[int]) -> None: self._raw["modelId"] = value def isNodeConfigured(self) -> bool: - if self._raw.get("nodeName") is not None and isinstance("nodeName", str): - return True + isConfigured = True + if self._raw.get("nodeName") is None or isinstance("nodeName", str): + isConfigured = False - if self._raw.get("password") is not None and isinstance("password", str): - return True + if self._raw.get("password") is None or isinstance("password", str): + isConfigured = False - if self._raw.get("image") is not None and isinstance("image", str): - return True + if self._raw.get("image") is None or isinstance("image", str): + isConfigured = False - if self._raw.get("nodeAccessToken") is not None and isinstance("nodeAccessToken", str): - return True + if self._raw.get("nodeAccessToken") is None or isinstance("nodeAccessToken", str): + isConfigured = False - if self._raw.get("nodeRam") is not None and isinstance("nodeRam", int): - return True + if self._raw.get("nodeRam") is None or isinstance("nodeRam", int): + isConfigured = False - if self._raw.get("nodeSwap") is not None and isinstance("nodeSwap", int): - return True + if self._raw.get("nodeSwap") is None or isinstance("nodeSwap", int): + isConfigured = False - if self._raw.get("nodeSharedMemory") is not None and isinstance("nodeSharedMemory", int): - return True + if self._raw.get("nodeSharedMemory") is None or isinstance("nodeSharedMemory", int): + isConfigured = False - if self._raw.get("nodeMode") is not None and isinstance("nodeMode", int): - return True + if self._raw.get("nodeMode") is None or isinstance("nodeMode", int): + isConfigured = False - return False + return isConfigured def getInitScriptPath(self) -> Optional[Path]: value = self._raw.get("initScript") From 0c731ff6e22c744a373feea22241adbd103539ad Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 17 Apr 2024 14:06:32 +0200 Subject: [PATCH 08/28] CTX-5430: Saving changes, cleaning code (have issue with imports and sync not being done first).... --- coretex/cli/modules/node.py | 69 ++++++------------- coretex/cli/modules/ui.py | 2 +- coretex/cli/modules/update.py | 3 +- coretex/configuration/__init__.py | 20 +++++- coretex/configuration/config_defaults.py | 21 +++++- coretex/configuration/node.py | 29 ++++---- coretex/entities/__init__.py | 1 + coretex/entities/node/__init__.py | 19 +++++ .../modules => entities/node}/node_mode.py | 0 coretex/entities/node/node_status.py | 27 ++++++++ 10 files changed, 121 insertions(+), 70 deletions(-) create mode 100644 coretex/entities/node/__init__.py rename coretex/{cli/modules => entities/node}/node_mode.py (100%) create mode 100644 coretex/entities/node/node_status.py diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index 444187a3..59da9c76 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Any, Dict, Tuple, Optional +from typing import Tuple, Optional from enum import Enum, IntEnum from pathlib import Path from base64 import b64encode @@ -23,13 +23,12 @@ import logging import requests -from . import config_defaults, utils, ui -from .node_mode import NodeMode +from . import utils, ui from ...cryptography import rsa from ...networking import networkManager, NetworkRequestError from ...utils import CommandException, docker -from ...entities.model import Model -from ...configuration import UserConfiguration, NodeConfiguration +from ...entities import Model, NodeMode +from ...configuration import config_defaults, UserConfiguration, NodeConfiguration class NodeException(Exception): @@ -259,26 +258,26 @@ def selectNodeMode(storagePath: str) -> Tuple[int, Optional[int]]: return availableNodeModes[selectedMode], None -def promptCpu(config: Dict[str, Any], cpuLimit: int) -> int: +def promptCpu(cpuLimit: int) -> int: cpuCount: int = ui.clickPrompt(f"Enter the number of CPUs the container will use (Maximum: {cpuLimit}) (press enter to use default)", cpuLimit, type = int) if cpuCount > cpuLimit: ui.errorEcho(f"ERROR: CPU limit in Docker Desktop ({cpuLimit}) is lower than the specified value ({cpuCount})") - return promptCpu(config, cpuLimit) + return promptCpu(cpuLimit) return cpuCount -def promptRam(config: Dict[str, Any], ramLimit: int) -> int: +def promptRam(ramLimit: int) -> int: nodeRam: int = ui.clickPrompt(f"Node RAM memory limit in GB (Minimum: {config_defaults.MINIMUM_RAM_MEMORY}GB, Maximum: {ramLimit}GB) (press enter to use default)", ramLimit, type = int) if nodeRam > ramLimit: - ui.errorEcho(f"ERROR: RAM limit in Docker Desktop ({ramLimit}GB) is lower than the configured value ({config['ramLimit']}GB). Please adjust resource limitations in Docker Desktop settings.") - return promptRam(config, ramLimit) + ui.errorEcho(f"ERROR: RAM limit in Docker Desktop ({ramLimit}GB) is lower than the configured value ({nodeRam}GB). Please adjust resource limitations in Docker Desktop settings.") + return promptRam(ramLimit) if nodeRam < config_defaults.MINIMUM_RAM_MEMORY: - ui.errorEcho(f"ERROR: Configured RAM ({config['ramLimit']}GB) is lower than the minimum Node RAM requirement ({config_defaults.MINIMUM_RAM_MEMORY}GB).") - return promptRam(config, ramLimit) + ui.errorEcho(f"ERROR: Configured RAM ({nodeRam}GB) is lower than the minimum Node RAM requirement ({config_defaults.MINIMUM_RAM_MEMORY}GB).") + return promptRam(ramLimit) return nodeRam @@ -309,38 +308,6 @@ def checkResourceLimitations() -> None: raise RuntimeError(f"Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM_MEMORY}GB) is higher than your current Docker desktop RAM limit ({ramLimit}GB). Please adjust resource limitations in Docker Desktop settings to match Node requirements.") -def isConfigurationValid(config: Dict[str, Any]) -> bool: - isValid = True - cpuLimit, ramLimit = docker.getResourceLimits() - - if not isinstance(config["nodeRam"], int): - ui.errorEcho(f"Invalid config \"nodeRam\" field type \"{type(config['nodeRam'])}\". Expected: \"int\"") - isValid = False - - if not isinstance(config["cpuCount"], int): - ui.errorEcho(f"Invalid config \"cpuCount\" field type \"{type(config['cpuCount'])}\". Expected: \"int\"") - isValid = False - - if config["cpuCount"] > cpuLimit: - ui.errorEcho(f"Configuration not valid. CPU limit in Docker Desktop ({cpuLimit}) is lower than the configured value ({config['cpuCount']})") - isValid = False - - if ramLimit < config_defaults.MINIMUM_RAM_MEMORY: - ui.errorEcho(f"Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM_MEMORY}GB) is higher than your current Docker desktop RAM limit ({ramLimit}GB). Please adjust resource limitations in Docker Desktop settings to match Node requirements.") - isValid = False - - if config["nodeRam"] > ramLimit: - ui.errorEcho(f"Configuration not valid. RAM limit in Docker Desktop ({ramLimit}GB) is lower than the configured value ({config['nodeRam']}GB)") - isValid = False - - - if config["nodeRam"] < config_defaults.MINIMUM_RAM_MEMORY: - ui.errorEcho(f"Configuration not valid. Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM_MEMORY}GB) is higher than the configured value ({config['nodeRam']}GB)") - isValid = False - - return isValid - - def configureNode(config: NodeConfiguration, verbose: bool) -> None: ui.highlightEcho("[Node Configuration]") @@ -367,7 +334,7 @@ def configureNode(config: NodeConfiguration, verbose: bool) -> None: config.nodeRam = min(ramLimit, config_defaults.DEFAULT_RAM_MEMORY) config.nodeSwap = config_defaults.DEFAULT_SWAP_MEMORY config.nodeSharedMemory = config_defaults.DEFAULT_SHARED_MEMORY - config.cpuCount = config_defaults.DEFAULT_CPU_COUNT + config.cpuCount = config_defaults.DEFAULT_CPU_COUNT if config_defaults.DEFAULT_CPU_COUNT is not None else 0 config.nodeMode = config_defaults.DEFAULT_NODE_MODE config.allowDocker = config_defaults.DEFAULT_ALLOW_DOCKER config.nodeSecret = config_defaults.DEFAULT_NODE_SECRET @@ -378,9 +345,9 @@ def configureNode(config: NodeConfiguration, verbose: bool) -> None: if verbose: configstoragePath = ui.clickPrompt("Storage path (press enter to use default)", config_defaults.DEFAULT_STORAGE_PATH, type = str) - config.cpuCount = promptCpu(config, cpuLimit) + config.cpuCount = promptCpu(cpuLimit) - config.nodeRam = promptRam(config, ramLimit) + config.nodeRam = promptRam(ramLimit) config.nodeSwap = ui.clickPrompt("Node swap memory limit in GB, make sure it is larger than mem limit (press enter to use default)", config_defaults.DEFAULT_SWAP_MEMORY, type = int) config.nodeSharedMemory = ui.clickPrompt("Node POSIX shared memory limit in GB (press enter to use default)", config_defaults.DEFAULT_SHARED_MEMORY, type = int) @@ -408,8 +375,12 @@ def configureNode(config: NodeConfiguration, verbose: bool) -> None: def initializeNodeConfiguration() -> None: config = NodeConfiguration() - if config.isNodeConfigured() and not isConfigurationValid(config): - raise RuntimeError(f"Invalid configuration. Please run \"coretex node config\" command to configure Node.") + if config.isNodeConfigured(): + isConfigValid, errors = config.isConfigurationValid() + if not isConfigValid: + for error in errors: + ui.errorEcho(error) + raise RuntimeError(f"Invalid configuration. Please run \"coretex node config\" command to configure Node.") if not config.isNodeConfigured(): ui.errorEcho("Node configuration not found.") diff --git a/coretex/cli/modules/ui.py b/coretex/cli/modules/ui.py index 611c260b..f6b2b7fa 100644 --- a/coretex/cli/modules/ui.py +++ b/coretex/cli/modules/ui.py @@ -22,7 +22,7 @@ import click import inquirer -from .node_mode import NodeMode +from ...entities import NodeMode from ...configuration import UserConfiguration, NodeConfiguration diff --git a/coretex/cli/modules/update.py b/coretex/cli/modules/update.py index c4f1ea95..e847c336 100644 --- a/coretex/cli/modules/update.py +++ b/coretex/cli/modules/update.py @@ -17,12 +17,11 @@ from pathlib import Path -from . import config_defaults from .cron import jobExists, scheduleJob from .node import getRepoFromImageUrl, getTagFromImageUrl from ..resources import RESOURCES_DIR from ...utils import command -from ...configuration import CONFIG_DIR, UserConfiguration, NodeConfiguration +from ...configuration import config_defaults, CONFIG_DIR, UserConfiguration, NodeConfiguration UPDATE_SCRIPT_NAME = "ctx_node_update.sh" diff --git a/coretex/configuration/__init__.py b/coretex/configuration/__init__.py index b5b480f8..ec7059aa 100644 --- a/coretex/configuration/__init__.py +++ b/coretex/configuration/__init__.py @@ -1,13 +1,31 @@ +# Copyright (C) 2023 Coretex LLC + +# This file is part of Coretex.ai + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from .user import UserConfiguration, LoginInfo from .node import NodeConfiguration from .base import CONFIG_DIR - def _syncConfigWithEnv() -> None: # If configuration doesn't exist default one will be created # Initialization of User and Node Configuration classes will do # the necessary sync between config properties and corresponding # environment variables (e.g. storagePath -> CTX_STORAGE_PATH) + print('sync') + UserConfiguration() NodeConfiguration() diff --git a/coretex/configuration/config_defaults.py b/coretex/configuration/config_defaults.py index 52a1f954..d362545e 100644 --- a/coretex/configuration/config_defaults.py +++ b/coretex/configuration/config_defaults.py @@ -1,9 +1,26 @@ +# Copyright (C) 2023 Coretex LLC + +# This file is part of Coretex.ai + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from pathlib import Path import os -from .node_mode import NodeMode -from ...statistics import getAvailableRamMemory +from ..entities import NodeMode +from ..statistics import getAvailableRamMemory DOCKER_CONTAINER_NAME = "coretex_node" diff --git a/coretex/configuration/node.py b/coretex/configuration/node.py index b2a4f188..e2595476 100644 --- a/coretex/configuration/node.py +++ b/coretex/configuration/node.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Dict, Any, Optional +from typing import Dict, Any, List, Optional, Tuple from pathlib import Path import os @@ -197,37 +197,36 @@ def isNodeConfigured(self) -> bool: not isinstance(self._raw.get("nodeMode"), int) ) - def isConfigurationValid(self) -> bool: + def isConfigurationValid(self) -> Tuple[bool, List[str]]: isValid = True errorMessages = [] cpuLimit, ramLimit = docker.getResourceLimits() - if not isinstance(config["nodeRam"], int): - ui.errorEcho(f"Invalid config \"nodeRam\" field type \"{type(config['nodeRam'])}\". Expected: \"int\"") + if not isinstance(self._raw.get("nodeRam"), int): + errorMessages.append(f"Invalid config \"nodeRam\" field type \"{type(self._raw.get('nodeRam'))}\". Expected: \"int\"") isValid = False - if not isinstance(config["cpuCount"], int): - ui.errorEcho(f"Invalid config \"cpuCount\" field type \"{type(config['cpuCount'])}\". Expected: \"int\"") + if not isinstance(self._raw.get("cpuCount"), int): + errorMessages.append(f"Invalid config \"cpuCount\" field type \"{type(self._raw.get('cpuCount'))}\". Expected: \"int\"") isValid = False - if config["cpuCount"] > cpuLimit: - ui.errorEcho(f"Configuration not valid. CPU limit in Docker Desktop ({cpuLimit}) is lower than the configured value ({config['cpuCount']})") + if self.cpuCount > cpuLimit: + errorMessages.append(f"Configuration not valid. CPU limit in Docker Desktop ({cpuLimit}) is lower than the configured value ({self._raw.get('cpuCount')})") isValid = False if ramLimit < config_defaults.MINIMUM_RAM_MEMORY: - ui.errorEcho(f"Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM_MEMORY}GB) is higher than your current Docker desktop RAM limit ({ramLimit}GB). Please adjust resource limitations in Docker Desktop settings to match Node requirements.") + errorMessages.append(f"Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM_MEMORY}GB) is higher than your current Docker desktop RAM limit ({ramLimit}GB). Please adjust resource limitations in Docker Desktop settings to match Node requirements.") isValid = False - if config["nodeRam"] > ramLimit: - ui.errorEcho(f"Configuration not valid. RAM limit in Docker Desktop ({ramLimit}GB) is lower than the configured value ({config['nodeRam']}GB)") + if self.nodeRam > ramLimit: + errorMessages.append(f"Configuration not valid. RAM limit in Docker Desktop ({ramLimit}GB) is lower than the configured value ({self._raw.get('nodeRam')}GB)") isValid = False - - if config["nodeRam"] < config_defaults.MINIMUM_RAM_MEMORY: - ui.errorEcho(f"Configuration not valid. Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM_MEMORY}GB) is higher than the configured value ({config['nodeRam']}GB)") + if self.nodeRam < config_defaults.MINIMUM_RAM_MEMORY: + errorMessages.append(f"Configuration not valid. Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM_MEMORY}GB) is higher than the configured value ({self._raw.get('nodeRam')}GB)") isValid = False - return isValid + return isValid, errorMessages def getInitScriptPath(self) -> Optional[Path]: diff --git a/coretex/entities/__init__.py b/coretex/entities/__init__.py index 6e17246e..90f3738a 100644 --- a/coretex/entities/__init__.py +++ b/coretex/entities/__init__.py @@ -22,3 +22,4 @@ from .sample import * from .secret import * from .task_run import * +from .node import * diff --git a/coretex/entities/node/__init__.py b/coretex/entities/node/__init__.py new file mode 100644 index 00000000..001c3771 --- /dev/null +++ b/coretex/entities/node/__init__.py @@ -0,0 +1,19 @@ +# Copyright (C) 2023 Coretex LLC + +# This file is part of Coretex.ai + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from .node_mode import NodeMode +from .node_status import NodeStatus diff --git a/coretex/cli/modules/node_mode.py b/coretex/entities/node/node_mode.py similarity index 100% rename from coretex/cli/modules/node_mode.py rename to coretex/entities/node/node_mode.py diff --git a/coretex/entities/node/node_status.py b/coretex/entities/node/node_status.py new file mode 100644 index 00000000..1598cfb3 --- /dev/null +++ b/coretex/entities/node/node_status.py @@ -0,0 +1,27 @@ +# Copyright (C) 2023 Coretex LLC + +# This file is part of Coretex.ai + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from enum import IntEnum + + +class NodeStatus(IntEnum): + + inactive = 1 + active = 2 + busy = 3 + deleted = 4 + reconnecting = 5 From 35b94bcf66aa4a517976163b0998ea67e4f63f53 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 17 Apr 2024 16:11:35 +0200 Subject: [PATCH 09/28] CTX-5430: Saving last changes. --- coretex/cli/commands/node.py | 2 +- coretex/cli/modules/node.py | 7 ++++--- coretex/cli/modules/ui.py | 2 +- coretex/configuration/__init__.py | 2 -- coretex/configuration/config_defaults.py | 2 +- coretex/entities/__init__.py | 1 - coretex/{entities => }/node/__init__.py | 0 coretex/{entities => }/node/node_mode.py | 0 coretex/{entities => }/node/node_status.py | 0 9 files changed, 7 insertions(+), 9 deletions(-) rename coretex/{entities => }/node/__init__.py (100%) rename coretex/{entities => }/node/node_mode.py (100%) rename coretex/{entities => }/node/node_status.py (100%) diff --git a/coretex/cli/commands/node.py b/coretex/cli/commands/node.py index afea4d63..07e2a0c9 100644 --- a/coretex/cli/commands/node.py +++ b/coretex/cli/commands/node.py @@ -155,9 +155,9 @@ def config(verbose: bool) -> None: @click.group() -@utils.onBeforeCommandExecute(node_module.checkResourceLimitations) @utils.onBeforeCommandExecute(docker.isDockerAvailable) @utils.onBeforeCommandExecute(user.initializeUserSession) +@utils.onBeforeCommandExecute(node_module.checkResourceLimitations) def node() -> None: pass diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index 59da9c76..cf610e85 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -27,7 +27,8 @@ from ...cryptography import rsa from ...networking import networkManager, NetworkRequestError from ...utils import CommandException, docker -from ...entities import Model, NodeMode +from ...entities import Model +from ...node import NodeMode from ...configuration import config_defaults, UserConfiguration, NodeConfiguration @@ -331,7 +332,7 @@ def configureNode(config: NodeConfiguration, verbose: bool) -> None: config.image += f":latest-{tag}" config.storagePath = config_defaults.DEFAULT_STORAGE_PATH - config.nodeRam = min(ramLimit, config_defaults.DEFAULT_RAM_MEMORY) + config.nodeRam = int(min(ramLimit, config_defaults.DEFAULT_RAM_MEMORY)) config.nodeSwap = config_defaults.DEFAULT_SWAP_MEMORY config.nodeSharedMemory = config_defaults.DEFAULT_SHARED_MEMORY config.cpuCount = config_defaults.DEFAULT_CPU_COUNT if config_defaults.DEFAULT_CPU_COUNT is not None else 0 @@ -343,7 +344,7 @@ def configureNode(config: NodeConfiguration, verbose: bool) -> None: publicKey: Optional[bytes] = None if verbose: - configstoragePath = ui.clickPrompt("Storage path (press enter to use default)", config_defaults.DEFAULT_STORAGE_PATH, type = str) + config.storagePath = ui.clickPrompt("Storage path (press enter to use default)", config_defaults.DEFAULT_STORAGE_PATH, type = str) config.cpuCount = promptCpu(cpuLimit) diff --git a/coretex/cli/modules/ui.py b/coretex/cli/modules/ui.py index f6b2b7fa..1be9bcde 100644 --- a/coretex/cli/modules/ui.py +++ b/coretex/cli/modules/ui.py @@ -22,7 +22,7 @@ import click import inquirer -from ...entities import NodeMode +from ...node import NodeMode from ...configuration import UserConfiguration, NodeConfiguration diff --git a/coretex/configuration/__init__.py b/coretex/configuration/__init__.py index ec7059aa..1e3d531a 100644 --- a/coretex/configuration/__init__.py +++ b/coretex/configuration/__init__.py @@ -25,7 +25,5 @@ def _syncConfigWithEnv() -> None: # the necessary sync between config properties and corresponding # environment variables (e.g. storagePath -> CTX_STORAGE_PATH) - print('sync') - UserConfiguration() NodeConfiguration() diff --git a/coretex/configuration/config_defaults.py b/coretex/configuration/config_defaults.py index d362545e..05ee9d9c 100644 --- a/coretex/configuration/config_defaults.py +++ b/coretex/configuration/config_defaults.py @@ -19,7 +19,7 @@ import os -from ..entities import NodeMode +from ..node import NodeMode from ..statistics import getAvailableRamMemory diff --git a/coretex/entities/__init__.py b/coretex/entities/__init__.py index 90f3738a..6e17246e 100644 --- a/coretex/entities/__init__.py +++ b/coretex/entities/__init__.py @@ -22,4 +22,3 @@ from .sample import * from .secret import * from .task_run import * -from .node import * diff --git a/coretex/entities/node/__init__.py b/coretex/node/__init__.py similarity index 100% rename from coretex/entities/node/__init__.py rename to coretex/node/__init__.py diff --git a/coretex/entities/node/node_mode.py b/coretex/node/node_mode.py similarity index 100% rename from coretex/entities/node/node_mode.py rename to coretex/node/node_mode.py diff --git a/coretex/entities/node/node_status.py b/coretex/node/node_status.py similarity index 100% rename from coretex/entities/node/node_status.py rename to coretex/node/node_status.py From 04668015388f29456718990f1aadfd9dbd414470 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 17 Apr 2024 16:15:27 +0200 Subject: [PATCH 10/28] CTX-5430: Saving changes ramLimit must be int not float. --- coretex/utils/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coretex/utils/docker.py b/coretex/utils/docker.py index 4364b88a..ef10f0c6 100644 --- a/coretex/utils/docker.py +++ b/coretex/utils/docker.py @@ -129,4 +129,4 @@ def getResourceLimits() -> Tuple[int, int]: _, output, _ = command(["docker", "info", "--format", "{{json .}}"], ignoreStdout = True, ignoreStderr = True) jsonOutput = json.loads(output) - return jsonOutput["NCPU"], round(jsonOutput["MemTotal"] / (1024 ** 3), 0) + return jsonOutput["NCPU"], round(jsonOutput["MemTotal"] / (1024 ** 3), None) From 4a80f55b36802e3e3c83542295bb5157a5aeee77 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 17 Apr 2024 16:58:06 +0200 Subject: [PATCH 11/28] CTX-5430: More code cleanup. --- coretex/cli/main.py | 4 +- coretex/cli/modules/node.py | 60 +++++++++++++++++------- coretex/configuration/config_defaults.py | 4 +- coretex/utils/docker.py | 2 +- 4 files changed, 48 insertions(+), 22 deletions(-) diff --git a/coretex/cli/main.py b/coretex/cli/main.py index 9e807dcb..a3d09166 100644 --- a/coretex/cli/main.py +++ b/coretex/cli/main.py @@ -18,7 +18,7 @@ import click from .commands.login import login -# from .commands.model import model +from .commands.model import model from .commands.node import node from .commands.project import project @@ -30,6 +30,6 @@ def cli() -> None: pass cli.add_command(login) -# cli.add_command(model) +cli.add_command(model) cli.add_command(project) cli.add_command(node) diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index cf610e85..8195490c 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -335,7 +335,7 @@ def configureNode(config: NodeConfiguration, verbose: bool) -> None: config.nodeRam = int(min(ramLimit, config_defaults.DEFAULT_RAM_MEMORY)) config.nodeSwap = config_defaults.DEFAULT_SWAP_MEMORY config.nodeSharedMemory = config_defaults.DEFAULT_SHARED_MEMORY - config.cpuCount = config_defaults.DEFAULT_CPU_COUNT if config_defaults.DEFAULT_CPU_COUNT is not None else 0 + config.cpuCount = int(min(cpuLimit, config_defaults.DEFAULT_CPU_COUNT)) config.nodeMode = config_defaults.DEFAULT_NODE_MODE config.allowDocker = config_defaults.DEFAULT_ALLOW_DOCKER config.nodeSecret = config_defaults.DEFAULT_NODE_SECRET @@ -373,31 +373,55 @@ def configureNode(config: NodeConfiguration, verbose: bool) -> None: config.nodeAccessToken = registerNode(config.nodeName, publicKey) +def promptNodeConfiguration() -> bool: + if isRunning(): + stopNode = ui.clickPrompt( + "Node is already running. Do you wish to stop the Node? (Y/n)", + type = bool, + default = True, + show_default = False + ) + + if not stopNode: + ui.errorEcho("If you wish to reconfigure your node, use \"coretex node stop\" command first.") + return False + + stop() + + return True + + def initializeNodeConfiguration() -> None: config = NodeConfiguration() if config.isNodeConfigured(): isConfigValid, errors = config.isConfigurationValid() - if not isConfigValid: - for error in errors: - ui.errorEcho(error) - raise RuntimeError(f"Invalid configuration. Please run \"coretex node config\" command to configure Node.") + + if isConfigValid: + return + + for error in errors: + ui.errorEcho(error) + + if not ui.clickPrompt( + f"Existing node configuration is invalid. Would you like to update? (Y/n)", + type = bool, + default = True, + show_default = False + ): + return + + if not promptNodeConfiguration(): + return + + configureNode(config, verbose = False) + config.save() if not config.isNodeConfigured(): ui.errorEcho("Node configuration not found.") - if isRunning(): - stopNode = ui.clickPrompt( - "Node is already running. Do you wish to stop the Node? (Y/n)", - type = bool, - default = True, - show_default = False - ) - - if not stopNode: - ui.errorEcho("If you wish to reconfigure your node, use \"coretex node stop\" command first.") - return - - stop() + + if not promptNodeConfiguration(): + return configureNode(config, verbose = False) config.save() diff --git a/coretex/configuration/config_defaults.py b/coretex/configuration/config_defaults.py index 05ee9d9c..54a5ad16 100644 --- a/coretex/configuration/config_defaults.py +++ b/coretex/configuration/config_defaults.py @@ -23,6 +23,8 @@ from ..statistics import getAvailableRamMemory +cpuCount = os.cpu_count() + DOCKER_CONTAINER_NAME = "coretex_node" DOCKER_CONTAINER_NETWORK = "coretex_node" DEFAULT_STORAGE_PATH = str(Path.home() / ".coretex") @@ -30,7 +32,7 @@ MINIMUM_RAM_MEMORY = 6 DEFAULT_SWAP_MEMORY = DEFAULT_RAM_MEMORY * 2 DEFAULT_SHARED_MEMORY = 2 -DEFAULT_CPU_COUNT = os.cpu_count() +DEFAULT_CPU_COUNT = cpuCount if cpuCount is not None else 0 DEFAULT_NODE_MODE = NodeMode.execution DEFAULT_ALLOW_DOCKER = False DEFAULT_NODE_SECRET = "" diff --git a/coretex/utils/docker.py b/coretex/utils/docker.py index ef10f0c6..6115d1b7 100644 --- a/coretex/utils/docker.py +++ b/coretex/utils/docker.py @@ -129,4 +129,4 @@ def getResourceLimits() -> Tuple[int, int]: _, output, _ = command(["docker", "info", "--format", "{{json .}}"], ignoreStdout = True, ignoreStderr = True) jsonOutput = json.loads(output) - return jsonOutput["NCPU"], round(jsonOutput["MemTotal"] / (1024 ** 3), None) + return jsonOutput["NCPU"], round(jsonOutput["MemTotal"] / (1024 ** 3)) From f3bb4b7d42e331cdafbb912aad0e0e85a9496a20 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 15 May 2024 15:32:12 +0200 Subject: [PATCH 12/28] CTX-5430: import fix. --- coretex/cli/commands/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coretex/cli/commands/task.py b/coretex/cli/commands/task.py index 4434dfb2..12270905 100644 --- a/coretex/cli/commands/task.py +++ b/coretex/cli/commands/task.py @@ -20,7 +20,7 @@ import click from ... import folder_manager -from modules import user, utils, project_utils +from ..modules import user, utils, project_utils from ..._task import TaskRunWorker, LoggerUploadWorker, executeProcess, readTaskConfig from ...configuration import UserConfiguration from ...entities import TaskRun, TaskRunStatus From fa9a0f126ee36b6abd743271f84d19967941d641 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Thu, 16 May 2024 09:51:58 +0200 Subject: [PATCH 13/28] CTX-5430: Unit tests fix. --- coretex/configuration/base.py | 2 ++ coretex/networking/user_data.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/coretex/configuration/base.py b/coretex/configuration/base.py index 36f0107b..799d0a88 100644 --- a/coretex/configuration/base.py +++ b/coretex/configuration/base.py @@ -69,5 +69,7 @@ def getOptValue(self, configKey: str, valueType: Type[T], envKey: Optional[str] return value def save(self) -> None: + if not self._path.parent.exists(): + self._path.parent.mkdir(parents = True, exist_ok = True) with self._path.open("w") as configFile: json.dump(self._raw, configFile, indent = 4) diff --git a/coretex/networking/user_data.py b/coretex/networking/user_data.py index a0ee200c..cabb01e9 100644 --- a/coretex/networking/user_data.py +++ b/coretex/networking/user_data.py @@ -19,6 +19,7 @@ from pathlib import Path import json +from ..configuration.user import USER_CONFIG_PATH CONFIG_PATH = Path("~/.config/coretex/config.json").expanduser() @@ -28,7 +29,7 @@ class UserData: def __init__(self) -> None: - with open(CONFIG_PATH, "r") as configFile: + with open(USER_CONFIG_PATH, "r") as configFile: self.__values: Dict[str, Any] = json.load(configFile) def __getOptionalStr(self, key: str) -> Optional[str]: From 4ca0a8ef13257f16c949c96c4ee326ec2259eecb Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Tue, 23 Jul 2024 17:08:07 +0200 Subject: [PATCH 14/28] CTX-5430: Mypy errors fixed, fixing bugs while testing. Saving changes. --- coretex/cli/commands/login.py | 12 ++++++------ coretex/cli/commands/node.py | 11 ++++++----- coretex/cli/modules/node.py | 2 +- coretex/cli/modules/project_utils.py | 2 +- coretex/cli/modules/utils.py | 2 +- coretex/configuration/node.py | 17 ++++++++--------- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/coretex/cli/commands/login.py b/coretex/cli/commands/login.py index 37e6b77c..753bf773 100644 --- a/coretex/cli/commands/login.py +++ b/coretex/cli/commands/login.py @@ -23,10 +23,10 @@ @click.command() def login() -> None: - config = UserConfiguration() - if config.isUserConfigured(): + userConfig = UserConfiguration() + if userConfig.isUserConfigured(): if not ui.clickPrompt( - f"User already logged in with username {config.username}.\nWould you like to log in with a different user (Y/n)?", + f"User already logged in with username {userConfig.username}.\nWould you like to log in with a different user (Y/n)?", type = bool, default = True, show_default = False @@ -34,6 +34,6 @@ def login() -> None: return ui.stdEcho("Please enter your credentials:") - user.configUser() - config.save() - ui.successEcho(f"User {config.username} successfully logged in.") + user.configUser(userConfig) + userConfig.save() + ui.successEcho(f"User {userConfig.username} successfully logged in.") diff --git a/coretex/cli/commands/node.py b/coretex/cli/commands/node.py index 4411bdc2..ff121378 100644 --- a/coretex/cli/commands/node.py +++ b/coretex/cli/commands/node.py @@ -23,7 +23,8 @@ from ..modules import user, ui, utils from ..modules import node as node_module from ..modules import update as update_module -from ...configuration import UserConfiguration, NodeConfiguration, CONFIG_DIR +from ..modules.node import NodeStatus +from ...configuration import UserConfiguration, NodeConfiguration @click.command() @@ -84,15 +85,15 @@ def update(autoAccept: bool, autoDecline: bool) -> None: nodeStatus = node_module.getNodeStatus() - if nodeStatus == node_module.NodeStatus.inactive: + if nodeStatus == NodeStatus.inactive: ui.errorEcho("Node is not running. To update Node you need to start it first.") return - if nodeStatus == node_module.NodeStatus.reconnecting: + if nodeStatus == NodeStatus.reconnecting: ui.errorEcho("Node is reconnecting. Cannot update now.") return - if nodeStatus == update_module.NodeStatus.busy and not autoAccept: + if nodeStatus == NodeStatus.busy and not autoAccept: if autoDecline: return @@ -128,7 +129,7 @@ def update(autoAccept: bool, autoDecline: bool) -> None: node_module.stop() ui.stdEcho("Updating node...") - node_module.start(nodeConfig.image, config) + node_module.start(nodeConfig.image, userConfig, nodeConfig) docker.removeDanglingImages( node_module.getRepoFromImageUrl(nodeConfig.image), diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index 1f61b8d6..f9b5bbb1 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -416,7 +416,7 @@ def configureNode(nodeConfig: NodeConfiguration, verbose: bool) -> None: nodeConfig.storagePath = config_defaults.DEFAULT_STORAGE_PATH nodeConfig.nodeRam = int(min(max(config_defaults.MINIMUM_RAM, ramLimit), config_defaults.DEFAULT_RAM)) - nodeConfig.nodeSwap = config_defaults.DEFAULT_SWAP_MEMORY + nodeConfig.nodeSwap = min(swapLimit, int(max(config_defaults.DEFAULT_SWAP_MEMORY, swapLimit))) nodeConfig.nodeSharedMemory = config_defaults.DEFAULT_SHARED_MEMORY nodeConfig.cpuCount = int(min(cpuLimit, config_defaults.DEFAULT_CPU_COUNT)) nodeConfig.nodeMode = config_defaults.DEFAULT_NODE_MODE diff --git a/coretex/cli/modules/project_utils.py b/coretex/cli/modules/project_utils.py index eca5837c..246be85f 100644 --- a/coretex/cli/modules/project_utils.py +++ b/coretex/cli/modules/project_utils.py @@ -68,7 +68,7 @@ def promptProjectSelect() -> Optional[Project]: ui.progressEcho("Validating project...") try: project = Project.fetchOne(name = name) - ui.successEcho(f"Project \"{project}\" selected successfully!") + ui.successEcho(f"Project \"{name}\" selected successfully!") selectProject(project.id) except ValueError: ui.errorEcho(f"Project \"{name}\" not found.") diff --git a/coretex/cli/modules/utils.py b/coretex/cli/modules/utils.py index 2187eb89..89830264 100644 --- a/coretex/cli/modules/utils.py +++ b/coretex/cli/modules/utils.py @@ -63,7 +63,7 @@ def createEnvironment(venvPython: Path) -> None: ctxSource = fetchCtxSource() if ctxSource is not None: - command([str(venvPython), "-m", "pip", "install", ctxSource], ignoreStdout = True) + command([str(venvPython), "-m", "pip", "install", ctxSource], ignoreStdout = True, ignoreStderr = True) def checkEnvironment() -> None: diff --git a/coretex/configuration/node.py b/coretex/configuration/node.py index 4e65baae..01fd357a 100644 --- a/coretex/configuration/node.py +++ b/coretex/configuration/node.py @@ -227,15 +227,14 @@ def endpointInvocationPrice(self, value: Optional[float]) -> None: def isNodeConfigured(self) -> bool: return ( - not isinstance(self._raw.get("nodeName"), str) or - not isinstance(self._raw.get("password"), str) or - not isinstance(self._raw.get("image"), str) or - not isinstance(self._raw.get("nodeAccessToken"), str) or - not isinstance(self._raw.get("cpuCount"), int) or - not isinstance(self._raw.get("nodeRam"), int) or - not isinstance(self._raw.get("nodeSwap"), int) or - not isinstance(self._raw.get("nodeSharedMemory"), int) or - not isinstance(self._raw.get("nodeMode"), int) + isinstance(self._raw.get("nodeName"), str) and + isinstance(self._raw.get("image"), str) and + isinstance(self._raw.get("nodeAccessToken"), str) and + isinstance(self._raw.get("cpuCount"), int) and + isinstance(self._raw.get("nodeRam"), int) and + isinstance(self._raw.get("nodeSwap"), int) and + isinstance(self._raw.get("nodeSharedMemory"), int) and + isinstance(self._raw.get("nodeMode"), int) ) def isConfigurationValid(self) -> Tuple[bool, List[str]]: From 9b28f5e842a3e3a0b3e6979fdb0e6555ada1388f Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 24 Jul 2024 10:07:43 +0200 Subject: [PATCH 15/28] CTX-5430: Rest of cleanup and possible bug fixes. Tests were failing with CTX_STORAGE_PATH env var being None. --- coretex/cli/modules/node.py | 53 +++---------------------------------- coretex/folder_manager.py | 2 +- 2 files changed, 4 insertions(+), 51 deletions(-) diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index f9b5bbb1..7430576d 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -16,7 +16,7 @@ # along with this program. If not, see . from typing import Any, Dict, Optional -from enum import Enum, IntEnum +from enum import Enum from pathlib import Path from base64 import b64encode @@ -25,13 +25,11 @@ import click -from . import utils, ui -from . import config_defaults +from . import utils, ui, config_defaults from ...cryptography import rsa from ...networking import networkManager, NetworkRequestError from ...utils import CommandException, docker -from ...entities import Model -from ...node import NodeMode +from ...node import NodeMode, NodeStatus from ...configuration import UserConfiguration, NodeConfiguration @@ -39,15 +37,6 @@ class NodeException(Exception): pass -class NodeStatus(IntEnum): - - inactive = 1 - active = 2 - busy = 3 - deleted = 4 - reconnecting = 5 - - class ImageType(Enum): official = "official" @@ -355,42 +344,6 @@ def checkResourceLimitations() -> None: raise RuntimeError(f"Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM}GB) is higher than your current Docker desktop RAM limit ({ramLimit}GB). Please adjust resource limitations in Docker Desktop settings to match Node requirements.") -# def isConfigurationValid(config: Dict[str, Any]) -> bool: -# isValid = True -# cpuLimit, ramLimit = docker.getResourceLimits() -# swapLimit = docker.getDockerSwapLimit() - -# if not isinstance(config["nodeRam"], int): -# errorEcho(f"Invalid config \"nodeRam\" field type \"{type(config['nodeRam'])}\". Expected: \"int\"") -# isValid = False - -# if not isinstance(config["cpuCount"], int): -# errorEcho(f"Invalid config \"cpuCount\" field type \"{type(config['cpuCount'])}\". Expected: \"int\"") -# isValid = False - -# if config["cpuCount"] > cpuLimit: -# errorEcho(f"Configuration not valid. CPU limit in Docker Desktop ({cpuLimit}) is lower than the configured value ({config['cpuCount']})") -# isValid = False - -# if ramLimit < config_defaults.MINIMUM_RAM: -# errorEcho(f"Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM}GB) is higher than your current Docker desktop RAM limit ({ramLimit}GB). Please adjust resource limitations in Docker Desktop settings to match Node requirements.") -# isValid = False - -# if config["nodeRam"] > ramLimit: -# errorEcho(f"Configuration not valid. RAM limit in Docker Desktop ({ramLimit}GB) is lower than the configured value ({config['nodeRam']}GB)") -# isValid = False - -# if config["nodeSwap"] > swapLimit: -# errorEcho(f"Configuration not valid. RAM limit in Docker Desktop ({swapLimit}GB) is lower than the configured value ({config['nodeSwap']}GB)") -# isValid = False - -# if config["nodeRam"] < config_defaults.MINIMUM_RAM: -# errorEcho(f"Configuration not valid. Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM}GB) is higher than the configured value ({config['nodeRam']}GB)") -# isValid = False - -# return isValid - - def configureNode(nodeConfig: NodeConfiguration, verbose: bool) -> None: ui.highlightEcho("[Node Configuration]") diff --git a/coretex/folder_manager.py b/coretex/folder_manager.py index 002a4c9a..4bc342c6 100644 --- a/coretex/folder_manager.py +++ b/coretex/folder_manager.py @@ -60,7 +60,7 @@ def _createFolder(name: str) -> Path: return path -_root = Path(os.environ["CTX_STORAGE_PATH"]).expanduser() +_root = Path(os.environ.get("CTX_STORAGE_PATH", "~/.coretex")).expanduser() samplesFolder = _createFolder("samples") modelsFolder = _createFolder("models") From e51959e02b7c521264f1184de36d833226fa3bfd Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 24 Jul 2024 13:42:38 +0200 Subject: [PATCH 16/28] CTX-5430: Code cleanup, removed userdata. --- coretex/cli/commands/model.py | 9 ++- coretex/cli/commands/node.py | 26 ++++---- coretex/cli/commands/project.py | 6 +- coretex/cli/commands/task.py | 9 +-- coretex/cli/main.py | 4 +- coretex/cli/modules/node.py | 5 +- coretex/cli/modules/update.py | 2 + coretex/cli/modules/utils.py | 3 +- coretex/networking/network_manager.py | 22 +++---- coretex/networking/user_data.py | 88 --------------------------- 10 files changed, 46 insertions(+), 128 deletions(-) delete mode 100644 coretex/networking/user_data.py diff --git a/coretex/cli/commands/model.py b/coretex/cli/commands/model.py index 3d335ff1..10f93f73 100644 --- a/coretex/cli/commands/model.py +++ b/coretex/cli/commands/model.py @@ -2,7 +2,10 @@ import click -from ..modules import project_utils, user, utils, ui +from ..modules import ui +from ..modules.project_utils import getProject +from ..modules.user import initializeUserSession +from ..modules.utils import onBeforeCommandExecute from ...entities import Model from ...configuration import UserConfiguration @@ -18,7 +21,7 @@ def create(name: str, path: str, project: Optional[str], accuracy: float) -> Non # If project was provided used that, otherwise get the one from config # If project that was provided does not exist prompt user to create a new # one with that name - ctxProject = project_utils.getProject(project, userConfig) + ctxProject = getProject(project, userConfig) if ctxProject is None: return @@ -32,7 +35,7 @@ def create(name: str, path: str, project: Optional[str], accuracy: float) -> Non @click.group() -@utils.onBeforeCommandExecute(user.initializeUserSession) +@onBeforeCommandExecute(initializeUserSession) def model() -> None: pass diff --git a/coretex/cli/commands/node.py b/coretex/cli/commands/node.py index ff121378..c58a2968 100644 --- a/coretex/cli/commands/node.py +++ b/coretex/cli/commands/node.py @@ -20,16 +20,18 @@ import click from ...utils import docker -from ..modules import user, ui, utils +from ..modules import ui from ..modules import node as node_module -from ..modules import update as update_module from ..modules.node import NodeStatus +from ..modules.user import initializeUserSession +from ..modules.utils import onBeforeCommandExecute, checkEnvironment +from ..modules.update import activateAutoUpdate, getNodeStatus from ...configuration import UserConfiguration, NodeConfiguration @click.command() @click.option("--image", type = str, help = "Docker image url") -@utils.onBeforeCommandExecute(node_module.initializeNodeConfiguration) +@onBeforeCommandExecute(node_module.initializeNodeConfiguration) def start(image: Optional[str]) -> None: if node_module.isRunning(): if not ui.clickPrompt( @@ -59,7 +61,7 @@ def start(image: Optional[str]) -> None: node_module.start(dockerImage, userConfig, nodeConfig) docker.removeDanglingImages(node_module.getRepoFromImageUrl(dockerImage), node_module.getTagFromImageUrl(dockerImage)) - update_module.activateAutoUpdate() + activateAutoUpdate() @click.command() @@ -74,7 +76,7 @@ def stop() -> None: @click.command() @click.option("-y", "autoAccept", is_flag = True, help = "Accepts all prompts.") @click.option("-n", "autoDecline", is_flag = True, help = "Declines all prompts.") -@utils.onBeforeCommandExecute(node_module.initializeNodeConfiguration) +@onBeforeCommandExecute(node_module.initializeNodeConfiguration) def update(autoAccept: bool, autoDecline: bool) -> None: if autoAccept and autoDecline: ui.errorEcho("Only one of the flags (\"-y\" or \"-n\") can be used at the same time.") @@ -114,7 +116,7 @@ def update(autoAccept: bool, autoDecline: bool) -> None: ui.stdEcho("Updating node...") node_module.pull(nodeConfig.image) - if update_module.getNodeStatus() == update_module.NodeStatus.busy and not autoAccept: + if getNodeStatus() == NodeStatus.busy and not autoAccept: if autoDecline: return @@ -136,8 +138,6 @@ def update(autoAccept: bool, autoDecline: bool) -> None: node_module.getTagFromImageUrl(nodeConfig.image) ) - update_module.activateAutoUpdate() - @click.command() @click.option("--verbose", is_flag = True, help = "Configure node settings manually.") @@ -171,14 +171,14 @@ def config(verbose: bool) -> None: ui.previewConfig(userConfig, nodeConfig) ui.successEcho("Node successfully configured.") - update_module.activateAutoUpdate() + activateAutoUpdate() @click.group() -@utils.onBeforeCommandExecute(docker.isDockerAvailable) -@utils.onBeforeCommandExecute(user.initializeUserSession) -@utils.onBeforeCommandExecute(node_module.checkResourceLimitations) -@utils.onBeforeCommandExecute(utils.checkEnvironment) +@onBeforeCommandExecute(docker.isDockerAvailable) +@onBeforeCommandExecute(initializeUserSession) +@onBeforeCommandExecute(node_module.checkResourceLimitations) +@onBeforeCommandExecute(checkEnvironment) def node() -> None: pass diff --git a/coretex/cli/commands/project.py b/coretex/cli/commands/project.py index aaccbb82..16bb3e56 100644 --- a/coretex/cli/commands/project.py +++ b/coretex/cli/commands/project.py @@ -19,7 +19,9 @@ import click -from ..modules import ui, project_utils, utils, user +from ..modules import ui, project_utils +from ..modules.user import initializeUserSession +from ..modules.utils import onBeforeCommandExecute from ...entities import Project, ProjectVisibility from ...networking import RequestFailedError from ...configuration import UserConfiguration @@ -90,7 +92,7 @@ def select(name: str) -> None: @click.group() -@utils.onBeforeCommandExecute(user.initializeUserSession) +@onBeforeCommandExecute(initializeUserSession) def project() -> None: pass diff --git a/coretex/cli/commands/task.py b/coretex/cli/commands/task.py index 62f9bbed..f12fb526 100644 --- a/coretex/cli/commands/task.py +++ b/coretex/cli/commands/task.py @@ -19,8 +19,9 @@ import click -from ... import folder_manager -from ..modules import user, utils, project_utils +from ..modules.project_utils import getProject +from ..modules.user import initializeUserSession +from ..modules.utils import onBeforeCommandExecute from ... import folder_manager from ...configuration import UserConfiguration from ...entities import TaskRun, TaskRunStatus @@ -33,7 +34,7 @@ class RunException(Exception): @click.command() -@utils.onBeforeCommandExecute(user.initializeUserSession) +@onBeforeCommandExecute(initializeUserSession) @click.argument("path", type = click.Path(exists = True, dir_okay = False)) @click.option("--name", type = str, default = None) @click.option("--description", type = str, default = None) @@ -49,7 +50,7 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo folder_manager.clearTempFiles() - selectedProject = project_utils.getProject(project, userConfig) + selectedProject = getProject(project, userConfig) # clearing temporary files in case that node was manually killed before if selectedProject is None: diff --git a/coretex/cli/main.py b/coretex/cli/main.py index 92151481..96da0e0b 100644 --- a/coretex/cli/main.py +++ b/coretex/cli/main.py @@ -25,10 +25,8 @@ from .commands.task import run from .commands.project import project -from .modules import ui +from .modules import ui, utils from .modules.intercept import ClickExceptionInterceptor -from .modules import utils - from ..utils.process import CommandException diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index 7430576d..e47819dc 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -25,7 +25,8 @@ import click -from . import utils, ui, config_defaults +from . import ui, config_defaults +from .utils import isGPUAvailable from ...cryptography import rsa from ...networking import networkManager, NetworkRequestError from ...utils import CommandException, docker @@ -358,7 +359,7 @@ def configureNode(nodeConfig: NodeConfiguration, verbose: bool) -> None: else: nodeConfig.image = "coretexai/coretex-node" - if utils.isGPUAvailable(): + if isGPUAvailable(): nodeConfig.allowGpu = ui.clickPrompt("Do you want to allow the Node to access your GPU? (Y/n)", type = bool, default = True) else: nodeConfig.allowGpu = False diff --git a/coretex/cli/modules/update.py b/coretex/cli/modules/update.py index d531dffd..85931e98 100644 --- a/coretex/cli/modules/update.py +++ b/coretex/cli/modules/update.py @@ -18,6 +18,8 @@ from enum import IntEnum from pathlib import Path +import logging + import requests from .utils import getExecPath diff --git a/coretex/cli/modules/utils.py b/coretex/cli/modules/utils.py index 89830264..cde07880 100644 --- a/coretex/cli/modules/utils.py +++ b/coretex/cli/modules/utils.py @@ -24,6 +24,7 @@ import venv import shutil import logging +import platform from py3nvml import py3nvml @@ -31,8 +32,6 @@ import requests from . import ui -import platform - from ...configuration import DEFAULT_VENV_PATH from ...utils.process import command diff --git a/coretex/networking/network_manager.py b/coretex/networking/network_manager.py index 2fb869e0..3985a5d0 100644 --- a/coretex/networking/network_manager.py +++ b/coretex/networking/network_manager.py @@ -19,7 +19,7 @@ from .network_response import NetworkResponse from .network_manager_base import NetworkManagerBase -from .user_data import UserData +from ..configuration import UserConfiguration class NetworkManager(NetworkManagerBase): @@ -33,27 +33,27 @@ class NetworkManager(NetworkManagerBase): def __init__(self) -> None: super().__init__() - self.__userData = UserData() + self.__userConfig = UserConfiguration() @property def _apiToken(self) -> Optional[str]: - return self.__userData.apiToken + return self.__userConfig.token @_apiToken.setter def _apiToken(self, value: Optional[str]) -> None: - self.__userData.apiToken = value + self.__userConfig.token = value @property def _refreshToken(self) -> Optional[str]: - return self.__userData.refreshToken + return self.__userConfig.refreshToken @_refreshToken.setter def _refreshToken(self, value: Optional[str]) -> None: - self.__userData.refreshToken = value + self.__userConfig.refreshToken = value @property def hasStoredCredentials(self) -> bool: - return self.__userData.hasStoredCredentials + return self.__userConfig.isUserConfigured() def authenticate(self, username: str, password: str, storeCredentials: bool = True) -> NetworkResponse: """ @@ -82,8 +82,8 @@ def authenticate(self, username: str, password: str, storeCredentials: bool = Tr """ if storeCredentials: - self.__userData.username = username - self.__userData.password = password + self.__userConfig.username = username + self.__userConfig.password = password # authenticate using credentials stored in requests.Session.auth return super().authenticate(username, password, storeCredentials) @@ -101,10 +101,10 @@ def authenticateWithStoredCredentials(self) -> NetworkResponse: ValueError -> if credentials are not found """ - if self.__userData.username is None or self.__userData.password is None: + if self.__userConfig.username is None or self.__userConfig.password is None: raise ValueError(">> [Coretex] Credentials not stored") - return self.authenticate(self.__userData.username, self.__userData.password) + return self.authenticate(self.__userConfig.username, self.__userConfig.password) networkManager: NetworkManagerBase = NetworkManager() diff --git a/coretex/networking/user_data.py b/coretex/networking/user_data.py deleted file mode 100644 index cabb01e9..00000000 --- a/coretex/networking/user_data.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (C) 2023 Coretex LLC - -# This file is part of Coretex.ai - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from typing import Any, Optional, Dict -from pathlib import Path - -import json -from ..configuration.user import USER_CONFIG_PATH - - -CONFIG_PATH = Path("~/.config/coretex/config.json").expanduser() - - -# Only used by NetworkManager, should not be used anywhere else -class UserData: - - def __init__(self) -> None: - with open(USER_CONFIG_PATH, "r") as configFile: - self.__values: Dict[str, Any] = json.load(configFile) - - def __getOptionalStr(self, key: str) -> Optional[str]: - value = self.__values[key] - if value is None: - return None - - if isinstance(value, str): - return value - - raise ValueError(f">> [Coretex] {key} is not of type optional str") - - def __setValue(self, key: str, value: Any) -> None: - if not key in self.__values: - raise KeyError(f">> [Coretex] {key} not found") - - self.__values[key] = value - - with open(CONFIG_PATH, "w") as configFile: - json.dump(self.__values, configFile, indent = 4) - - @property - def hasStoredCredentials(self) -> bool: - return self.username is not None and self.password is not None - - @property - def username(self) -> Optional[str]: - return self.__getOptionalStr("username") - - @username.setter - def username(self, value: Optional[str]) -> None: - self.__setValue("username", value) - - @property - def password(self) -> Optional[str]: - return self.__getOptionalStr("password") - - @password.setter - def password(self, value: Optional[str]) -> None: - self.__setValue("password", value) - - @property - def apiToken(self) -> Optional[str]: - return self.__getOptionalStr("token") - - @apiToken.setter - def apiToken(self, value: Optional[str]) -> None: - self.__setValue("token", value) - - @property - def refreshToken(self) -> Optional[str]: - return self.__getOptionalStr("refreshToken") - - @refreshToken.setter - def refreshToken(self, value: Optional[str]) -> None: - self.__setValue("refreshToken", value) From f1e2b7c146391fdf2349406db35f58fb16319718 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 24 Jul 2024 13:47:47 +0200 Subject: [PATCH 17/28] CTX-5430n: testing cli on machines --- coretex/cli/commands/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coretex/cli/commands/node.py b/coretex/cli/commands/node.py index c58a2968..dcbcdcab 100644 --- a/coretex/cli/commands/node.py +++ b/coretex/cli/commands/node.py @@ -56,8 +56,8 @@ def start(image: Optional[str]) -> None: dockerImage = nodeConfig.image - if node_module.shouldUpdate(dockerImage): - node_module.pull(dockerImage) + # if node_module.shouldUpdate(dockerImage): + # node_module.pull(dockerImage) node_module.start(dockerImage, userConfig, nodeConfig) docker.removeDanglingImages(node_module.getRepoFromImageUrl(dockerImage), node_module.getTagFromImageUrl(dockerImage)) From 82edb68d4d3dbffa8c9edd4056a51b03763d8ddc Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 24 Jul 2024 14:08:14 +0200 Subject: [PATCH 18/28] CTX-5430: Code cleanup testing finished. --- coretex/cli/commands/node.py | 4 +- coretex/configuration/__init__.py | 2 +- coretex/configuration/user.py | 54 ------------ coretex/old_configuration.py | 142 ------------------------------ 4 files changed, 3 insertions(+), 199 deletions(-) delete mode 100644 coretex/old_configuration.py diff --git a/coretex/cli/commands/node.py b/coretex/cli/commands/node.py index dcbcdcab..c58a2968 100644 --- a/coretex/cli/commands/node.py +++ b/coretex/cli/commands/node.py @@ -56,8 +56,8 @@ def start(image: Optional[str]) -> None: dockerImage = nodeConfig.image - # if node_module.shouldUpdate(dockerImage): - # node_module.pull(dockerImage) + if node_module.shouldUpdate(dockerImage): + node_module.pull(dockerImage) node_module.start(dockerImage, userConfig, nodeConfig) docker.removeDanglingImages(node_module.getRepoFromImageUrl(dockerImage), node_module.getTagFromImageUrl(dockerImage)) diff --git a/coretex/configuration/__init__.py b/coretex/configuration/__init__.py index 57fe3794..7247699c 100644 --- a/coretex/configuration/__init__.py +++ b/coretex/configuration/__init__.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from .user import UserConfiguration, LoginInfo +from .user import UserConfiguration from .node import NodeConfiguration from .base import CONFIG_DIR, DEFAULT_VENV_PATH diff --git a/coretex/configuration/user.py b/coretex/configuration/user.py index e9c82944..9903fef8 100644 --- a/coretex/configuration/user.py +++ b/coretex/configuration/user.py @@ -17,7 +17,6 @@ from typing import Dict, Any, Optional from datetime import datetime, timezone -from dataclasses import dataclass import os @@ -38,29 +37,6 @@ } -class InvalidUserConfiguration(Exception): - pass - - -@dataclass -class LoginInfo: - - username: str - password: str - token: str - tokenExpirationDate: str - refreshToken: str - refreshTokenExpirationDate: str - - -def hasExpired(tokenExpirationDate: Optional[str]) -> bool: - if tokenExpirationDate is None: - return False - - currentDate = datetime.utcnow().replace(tzinfo = timezone.utc) - return currentDate >= decodeDate(tokenExpirationDate) - - class UserConfiguration(BaseConfiguration): def __init__(self) -> None: @@ -140,26 +116,6 @@ def projectId(self, value: Optional[int]) -> None: def isValid(self) -> bool: return self._raw.get("username") is not None and self._raw.get("password") is not None - @property - def hasTokenExpired(self) -> bool: - if self.token is None: - return True - - if self.tokenExpirationDate is None: - return True - - return hasExpired(self.tokenExpirationDate) - - @property - def hasRefreshTokenExpired(self) -> bool: - if self.refreshToken is None: - return True - - if self.refreshTokenExpirationDate is None: - return True - - return hasExpired(self.refreshTokenExpirationDate) - def isUserConfigured(self) -> bool: if self._raw.get("username") is None or not isinstance(self._raw.get("username"), str): return False @@ -169,16 +125,6 @@ def isUserConfigured(self) -> bool: return True - def saveLoginData(self, loginInfo: LoginInfo) -> None: - self.username = loginInfo.username - self.password = loginInfo.password - self.token = loginInfo.token - self.tokenExpirationDate = loginInfo.tokenExpirationDate - self.refreshToken = loginInfo.refreshToken - self.refreshTokenExpirationDate = loginInfo.refreshTokenExpirationDate - - self.save() - def isTokenValid(self, tokenName: str) -> bool: tokenValue = self._raw.get(tokenName) if not isinstance(tokenValue, str) or len(tokenValue) == 0: diff --git a/coretex/old_configuration.py b/coretex/old_configuration.py deleted file mode 100644 index a1e4cd40..00000000 --- a/coretex/old_configuration.py +++ /dev/null @@ -1,142 +0,0 @@ -# # Copyright (C) 2023 Coretex LLC - -# # This file is part of Coretex.ai - -# # This program is free software: you can redistribute it and/or modify -# # it under the terms of the GNU Affero General Public License as -# # published by the Free Software Foundation, either version 3 of the -# # License, or (at your option) any later version. - -# # This program is distributed in the hope that it will be useful, -# # but WITHOUT ANY WARRANTY; without even the implied warranty of -# # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# # GNU Affero General Public License for more details. - -# # You should have received a copy of the GNU Affero General Public License -# # along with this program. If not, see . - -# from typing import Dict, Any, Optional -# from pathlib import Path - -# import os -# import json -# import sys -# import logging - - -# def isCliRuntime() -> bool: -# executablePath = sys.argv[0] -# return ( -# executablePath.endswith("/bin/coretex") and -# os.access(executablePath, os.X_OK) -# ) - - -# def getEnvVar(key: str, default: str) -> str: -# if os.environ.get(key) is None: -# return default - -# return os.environ[key] - - -# CONFIG_DIR = Path.home().joinpath(".config", "coretex") -# DEFAULT_CONFIG_PATH = CONFIG_DIR / "config.json" -# DEFAULT_VENV_PATH = CONFIG_DIR / "venv" - - -# DEFAULT_CONFIG: Dict[str, Any] = { -# # os.environ used directly here since we don't wanna -# # set those variables to any value if they don't exist -# # in the os.environ the same way we do for properties which -# # call genEnvVar -# "username": os.environ.get("CTX_USERNAME"), -# "password": os.environ.get("CTX_PASSWORD"), -# "token": None, -# "refreshToken": None, -# "serverUrl": getEnvVar("CTX_API_URL", "https://api.coretex.ai/"), -# "storagePath": getEnvVar("CTX_STORAGE_PATH", "~/.coretex") -# } - - -# def loadConfig() -> Dict[str, Any]: -# with DEFAULT_CONFIG_PATH.open("r") as configFile: -# content = configFile.read() -# try: -# config: Dict[str, Any] = json.loads(content) -# except json.JSONDecodeError as exc: -# logging.getLogger("cli").debug( -# f"Failed to load corrupted config file. Content: {content}. Exception: {exc}", exc_info = exc -# ) -# config = {} - -# for key, value in DEFAULT_CONFIG.items(): -# if not key in config: -# config[key] = value - -# return config - - -# def _syncConfigWithEnv() -> None: -# # If configuration does not exist create default one -# if not DEFAULT_CONFIG_PATH.exists(): -# DEFAULT_CONFIG_PATH.parent.mkdir(parents = True, exist_ok = True) -# config = DEFAULT_CONFIG.copy() -# else: -# config = loadConfig() - -# saveConfig(config) - -# if not "CTX_API_URL" in os.environ: -# os.environ["CTX_API_URL"] = config["serverUrl"] - -# secretsKey = config.get("secretsKey") -# if isinstance(secretsKey, str) and secretsKey != "": -# os.environ["CTX_SECRETS_KEY"] = secretsKey - -# if not isCliRuntime(): -# os.environ["CTX_STORAGE_PATH"] = config["storagePath"] -# else: -# os.environ["CTX_STORAGE_PATH"] = f"{CONFIG_DIR}/data" - - -# def saveConfig(config: Dict[str, Any]) -> None: -# configPath = DEFAULT_CONFIG_PATH.expanduser() -# with configPath.open("w+") as configFile: -# json.dump(config, configFile, indent = 4) - - -# def isUserConfigured(config: Dict[str, Any]) -> bool: -# return ( -# config.get("username") is not None and -# config.get("password") is not None -# ) - - -# def isNodeConfigured(config: Dict[str, Any]) -> bool: -# return ( -# config.get("nodeName") is not None and -# config.get("storagePath") is not None and -# config.get("image") is not None and -# config.get("serverUrl") is not None and -# config.get("nodeAccessToken") is not None and -# config.get("nodeRam") is not None and -# config.get("nodeSwap") is not None and -# config.get("nodeSharedMemory") is not None and -# config.get("nodeMode") is not None -# ) - - -# def getInitScript(config: Dict[str, Any]) -> Optional[Path]: -# value = config.get("initScript") - -# if not isinstance(value, str): -# return None - -# if value == "": -# return None - -# path = Path(value).expanduser().absolute() -# if not path.exists(): -# return None - -# return path From 0c258ad4c2bd2ce1a98bd8aa151cec5bdbf30cef Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 24 Jul 2024 14:16:11 +0200 Subject: [PATCH 19/28] CTX-5430: More cleaning, large string formatting. --- coretex/cli/modules/node.py | 11 ++++++++-- coretex/configuration/node.py | 38 +++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index e47819dc..abb90c9e 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -278,7 +278,10 @@ def promptRam(ramLimit: int) -> int: return promptRam(ramLimit) if nodeRam < config_defaults.MINIMUM_RAM: - ui.errorEcho(f"ERROR: Configured RAM ({nodeRam}GB) is lower than the minimum Node RAM requirement ({config_defaults.MINIMUM_RAM}GB).") + ui.errorEcho( + f"ERROR: Configured RAM ({nodeRam}GB) is lower than " + "the minimum Node RAM requirement ({config_defaults.MINIMUM_RAM}GB)." + ) return promptRam(ramLimit) return nodeRam @@ -342,7 +345,11 @@ def checkResourceLimitations() -> None: _, ramLimit = docker.getResourceLimits() if ramLimit < config_defaults.MINIMUM_RAM: - raise RuntimeError(f"Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM}GB) is higher than your current Docker desktop RAM limit ({ramLimit}GB). Please adjust resource limitations in Docker Desktop settings to match Node requirements.") + raise RuntimeError( + f"Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM}GB) " + "is higher than your current Docker desktop RAM limit ({ramLimit}GB). " + "Please adjust resource limitations in Docker Desktop settings to match Node requirements." + ) def configureNode(nodeConfig: NodeConfiguration, verbose: bool) -> None: diff --git a/coretex/configuration/node.py b/coretex/configuration/node.py index 01fd357a..0100348c 100644 --- a/coretex/configuration/node.py +++ b/coretex/configuration/node.py @@ -244,35 +244,57 @@ def isConfigurationValid(self) -> Tuple[bool, List[str]]: swapLimit = docker.getDockerSwapLimit() if not isinstance(self._raw.get("nodeRam"), int): - errorMessages.append(f"Invalid config \"nodeRam\" field type \"{type(self._raw.get('nodeRam'))}\". Expected: \"int\"") + errorMessages.append( + f"Invalid config \"nodeRam\" field type \"{type(self._raw.get('nodeRam'))}\". Expected: \"int\"" + ) isValid = False if not isinstance(self._raw.get("cpuCount"), int): - errorMessages.append(f"Invalid config \"cpuCount\" field type \"{type(self._raw.get('cpuCount'))}\". Expected: \"int\"") + errorMessages.append( + f"Invalid config \"cpuCount\" field type \"{type(self._raw.get('cpuCount'))}\". Expected: \"int\"" + ) isValid = False if not isinstance(self._raw.get("nodeSwap"), int): - errorMessages.append(f"Invalid config \"nodeSwap\" field type \"{type(self._raw.get('nodeSwap'))}\". Expected: \"int\"") + errorMessages.append( + f"Invalid config \"nodeSwap\" field type \"{type(self._raw.get('nodeSwap'))}\". Expected: \"int\"" + ) isValid = False if self.cpuCount > cpuLimit: - errorMessages.append(f"Configuration not valid. CPU limit in Docker Desktop ({cpuLimit}) is lower than the configured value ({self._raw.get('cpuCount')})") + errorMessages.append( + f"Configuration not valid. CPU limit in Docker Desktop ({cpuLimit}) " + "is lower than the configured value ({self._raw.get('cpuCount')})" + ) isValid = False if ramLimit < config_defaults.MINIMUM_RAM_MEMORY: - errorMessages.append(f"Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM_MEMORY}GB) is higher than your current Docker desktop RAM limit ({ramLimit}GB). Please adjust resource limitations in Docker Desktop settings to match Node requirements.") + errorMessages.append( + f"Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM_MEMORY}GB) " + "is higher than your current Docker desktop RAM limit ({ramLimit}GB). " + "Please adjust resource limitations in Docker Desktop settings to match Node requirements." + ) isValid = False if self.nodeRam > ramLimit: - errorMessages.append(f"Configuration not valid. RAM limit in Docker Desktop ({ramLimit}GB) is lower than the configured value ({self._raw.get('nodeRam')}GB)") + errorMessages.append( + f"Configuration not valid. RAM limit in Docker Desktop ({ramLimit}GB) " + "is lower than the configured value ({self._raw.get('nodeRam')}GB)" + ) isValid = False if self.nodeRam < config_defaults.MINIMUM_RAM_MEMORY: - errorMessages.append(f"Configuration not valid. Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM_MEMORY}GB) is higher than the configured value ({self._raw.get('nodeRam')}GB)") + errorMessages.append( + f"Configuration not valid. Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM_MEMORY}GB) " + "is higher than the configured value ({self._raw.get('nodeRam')}GB)" + ) isValid = False if self.nodeSwap > swapLimit: - errorMessages.append(f"Configuration not valid. SWAP limit in Docker Desktop ({swapLimit}GB) is lower than the configured value ({self.nodeSwap}GB)") + errorMessages.append( + f"Configuration not valid. SWAP limit in Docker Desktop ({swapLimit}GB) " + "is lower than the configured value ({self.nodeSwap}GB)" + ) isValid = False return isValid, errorMessages From 981eacc3f981b52ba3a3c67443239182192c1042 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Fri, 26 Jul 2024 15:42:42 +0200 Subject: [PATCH 20/28] CTX-5430: Code cleanup based on comments from PR. --- coretex/cli/commands/node.py | 2 +- coretex/configuration/base.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/coretex/cli/commands/node.py b/coretex/cli/commands/node.py index c58a2968..79c953ae 100644 --- a/coretex/cli/commands/node.py +++ b/coretex/cli/commands/node.py @@ -19,13 +19,13 @@ import click -from ...utils import docker from ..modules import ui from ..modules import node as node_module from ..modules.node import NodeStatus from ..modules.user import initializeUserSession from ..modules.utils import onBeforeCommandExecute, checkEnvironment from ..modules.update import activateAutoUpdate, getNodeStatus +from ...utils import docker from ...configuration import UserConfiguration, NodeConfiguration diff --git a/coretex/configuration/base.py b/coretex/configuration/base.py index e0336b18..595ebebb 100644 --- a/coretex/configuration/base.py +++ b/coretex/configuration/base.py @@ -22,6 +22,7 @@ import os import json + T = TypeVar("T", int, float, str, bool) CONFIG_DIR = Path.home().joinpath(".config", "coretex") @@ -32,6 +33,7 @@ class InvalidConfiguration(Exception): pass class BaseConfiguration: + def __init__(self, path: Path) -> None: self._path = path @@ -72,5 +74,6 @@ def getOptValue(self, configKey: str, valueType: Type[T], envKey: Optional[str] def save(self) -> None: if not self._path.parent.exists(): self._path.parent.mkdir(parents = True, exist_ok = True) + with self._path.open("w") as configFile: json.dump(self._raw, configFile, indent = 4) From d7595f0356794441a9ebbbefb6504317fae6df66 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Tue, 6 Aug 2024 10:07:37 +0200 Subject: [PATCH 21/28] CTX-5430: Save changes, need to test further. --- coretex/cli/commands/login.py | 11 ++-- coretex/cli/commands/model.py | 2 +- coretex/cli/commands/node.py | 21 +++---- coretex/cli/commands/project.py | 2 +- coretex/cli/commands/task.py | 2 +- coretex/cli/modules/node.py | 33 ++++++----- coretex/cli/modules/project_utils.py | 4 +- coretex/cli/modules/user.py | 32 ++++++----- coretex/configuration/__init__.py | 79 ++++++++++++++++++++++++++- coretex/configuration/base.py | 71 ++++++++++++++++++------ coretex/configuration/node.py | 47 +++------------- coretex/configuration/user.py | 41 +++++++------- coretex/networking/network_manager.py | 32 +++++++---- 13 files changed, 238 insertions(+), 139 deletions(-) diff --git a/coretex/cli/commands/login.py b/coretex/cli/commands/login.py index 753bf773..e46bd3f7 100644 --- a/coretex/cli/commands/login.py +++ b/coretex/cli/commands/login.py @@ -18,13 +18,13 @@ import click from ..modules import user, ui -from ...configuration import UserConfiguration +from ...configuration import UserConfiguration, InvalidConfiguration, ConfigurationNotFound @click.command() def login() -> None: - userConfig = UserConfiguration() - if userConfig.isUserConfigured(): + try: + userConfig = UserConfiguration.load() if not ui.clickPrompt( f"User already logged in with username {userConfig.username}.\nWould you like to log in with a different user (Y/n)?", type = bool, @@ -33,7 +33,10 @@ def login() -> None: ): return + except (ConfigurationNotFound, InvalidConfiguration): + pass + ui.stdEcho("Please enter your credentials:") - user.configUser(userConfig) + userConfig = user.configUser() userConfig.save() ui.successEcho(f"User {userConfig.username} successfully logged in.") diff --git a/coretex/cli/commands/model.py b/coretex/cli/commands/model.py index 10f93f73..6d2d4bb0 100644 --- a/coretex/cli/commands/model.py +++ b/coretex/cli/commands/model.py @@ -16,7 +16,7 @@ @click.option("-p", "--project", type = str, required = False, default = None) @click.option("-a", "--accuracy", type = click.FloatRange(0, 1), required = False, default = 1) def create(name: str, path: str, project: Optional[str], accuracy: float) -> None: - userConfig = UserConfiguration() + userConfig = UserConfiguration.load() # If project was provided used that, otherwise get the one from config # If project that was provided does not exist prompt user to create a new diff --git a/coretex/cli/commands/node.py b/coretex/cli/commands/node.py index 79c953ae..212dbfea 100644 --- a/coretex/cli/commands/node.py +++ b/coretex/cli/commands/node.py @@ -26,7 +26,7 @@ from ..modules.utils import onBeforeCommandExecute, checkEnvironment from ..modules.update import activateAutoUpdate, getNodeStatus from ...utils import docker -from ...configuration import UserConfiguration, NodeConfiguration +from ...configuration import UserConfiguration, NodeConfiguration, InvalidConfiguration, ConfigurationNotFound @click.command() @@ -47,8 +47,8 @@ def start(image: Optional[str]) -> None: if node_module.exists(): node_module.clean() - nodeConfig = NodeConfiguration() - userConfig = UserConfiguration() + nodeConfig = NodeConfiguration.load() + userConfig = UserConfiguration.load() if image is not None: nodeConfig.image = image # store forced image (flagged) so we can run autoupdate afterwards @@ -82,8 +82,8 @@ def update(autoAccept: bool, autoDecline: bool) -> None: ui.errorEcho("Only one of the flags (\"-y\" or \"-n\") can be used at the same time.") return - userConfig = UserConfiguration() - nodeConfig = NodeConfiguration() + userConfig = UserConfiguration.load() + nodeConfig = NodeConfiguration.load() nodeStatus = node_module.getNodeStatus() @@ -154,10 +154,9 @@ def config(verbose: bool) -> None: node_module.stop() - userConfig = UserConfiguration() - nodeConfig = NodeConfiguration() - - if nodeConfig.isNodeConfigured(): + userConfig = UserConfiguration.load() + try: + nodeConfig = NodeConfiguration.load() if not ui.clickPrompt( "Node configuration already exists. Would you like to update? (Y/n)", type = bool, @@ -165,8 +164,10 @@ def config(verbose: bool) -> None: show_default = False ): return + except (ConfigurationNotFound, InvalidConfiguration): + pass - node_module.configureNode(nodeConfig, verbose) + nodeConfig = node_module.configureNode(verbose) nodeConfig.save() ui.previewConfig(userConfig, nodeConfig) diff --git a/coretex/cli/commands/project.py b/coretex/cli/commands/project.py index 16bb3e56..12d582ea 100644 --- a/coretex/cli/commands/project.py +++ b/coretex/cli/commands/project.py @@ -45,7 +45,7 @@ def create(name: Optional[str], projectType: Optional[int], description: Optiona @click.option("--name", "-n", type = str, help = "New Project name") @click.option("--description", "-d", type = str, help = "New Project description") def edit(project: Optional[str], name: Optional[str], description: Optional[str]) -> None: - userConfiguration = UserConfiguration() + userConfiguration = UserConfiguration.load() defaultProjectId = userConfiguration.projectId if defaultProjectId is None and project is None: ui.errorEcho(f"To use edit command you need to specifiy project name using \"--project\" or \"-p\" flag, or you can select default project using \"coretex project select\" command.") diff --git a/coretex/cli/commands/task.py b/coretex/cli/commands/task.py index f12fb526..68a3ac33 100644 --- a/coretex/cli/commands/task.py +++ b/coretex/cli/commands/task.py @@ -41,7 +41,7 @@ class RunException(Exception): @click.option("--snapshot", type = bool, default = False) @click.option("--project", "-p", type = str) def run(path: str, name: Optional[str], description: Optional[str], snapshot: bool, project: Optional[str]) -> None: - userConfig = UserConfiguration() + userConfig = UserConfiguration.load() if userConfig.refreshToken is None: raise RunException(f"Failed to execute \"coretex run {path}\" command. Authenticate again using \"coretex login\" command and try again.") diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index abb90c9e..0adaa725 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -31,7 +31,7 @@ from ...networking import networkManager, NetworkRequestError from ...utils import CommandException, docker from ...node import NodeMode, NodeStatus -from ...configuration import UserConfiguration, NodeConfiguration +from ...configuration import UserConfiguration, NodeConfiguration, InvalidConfiguration, ConfigurationNotFound class NodeException(Exception): @@ -352,8 +352,9 @@ def checkResourceLimitations() -> None: ) -def configureNode(nodeConfig: NodeConfiguration, verbose: bool) -> None: +def configureNode(verbose: bool) -> NodeConfiguration: ui.highlightEcho("[Node Configuration]") + nodeConfig = NodeConfiguration({}) # create new empty node config cpuLimit, ramLimit = docker.getResourceLimits() swapLimit = docker.getDockerSwapLimit() @@ -446,25 +447,23 @@ def configureNode(nodeConfig: NodeConfiguration, verbose: bool) -> None: nodeConfig.nodeAccessToken = registerNode(nodeConfig.nodeName, nodeMode, publicKey, nearWalletId, endpointInvocationPrice) nodeConfig.nodeMode = nodeMode + return nodeConfig -def initializeNodeConfiguration() -> None: - nodeConfig = NodeConfiguration() - isConfigured = nodeConfig.isNodeConfigured() - isConfigValid, errors = nodeConfig.isConfigurationValid() - - if isConfigured: - if isConfigValid: - return - ui.errorEcho("Invalid node configuration found.") - for error in errors: +def initializeNodeConfiguration() -> None: + try: + nodeConfig = NodeConfiguration.load() + return + except ConfigurationNotFound: + ui.errorEcho("Node configuration not found.") + if not click.confirm("Would you like to configure the node?", default = True): + raise + except InvalidConfiguration as ex: + for error in ex.errors: ui.errorEcho(error) if not click.confirm("Would you like to update the configuration?", default = True): - raise RuntimeError("Invalid configuration. Please use \"coretex node config\" to update a Node configuration.") - - if not isConfigured: - ui.errorEcho("Node configuration not found.") + raise if isRunning(): if not click.confirm("Node is already running. Do you wish to stop the Node?", default = True): @@ -473,5 +472,5 @@ def initializeNodeConfiguration() -> None: stop() - configureNode(nodeConfig, verbose = False) + nodeConfig = configureNode(verbose = False) nodeConfig.save() diff --git a/coretex/cli/modules/project_utils.py b/coretex/cli/modules/project_utils.py index 246be85f..91c5fef6 100644 --- a/coretex/cli/modules/project_utils.py +++ b/coretex/cli/modules/project_utils.py @@ -11,7 +11,7 @@ def selectProject(projectId: int) -> None: - userConfiguration = UserConfiguration() + userConfiguration = UserConfiguration.load() userConfiguration.projectId = projectId userConfiguration.save() @@ -128,7 +128,7 @@ def getProject(name: Optional[str], userConfig: UserConfiguration) -> Optional[P def isProjectSelected() -> bool: - userConfig = UserConfiguration() + userConfig = UserConfiguration.load() if userConfig.projectId is None: return False diff --git a/coretex/cli/modules/user.py b/coretex/cli/modules/user.py index 30f3072c..da2051dc 100644 --- a/coretex/cli/modules/user.py +++ b/coretex/cli/modules/user.py @@ -17,13 +17,15 @@ from . import ui from ...networking import networkManager, NetworkRequestError -from ...configuration import UserConfiguration +from ...configuration import UserConfiguration, InvalidConfiguration, ConfigurationNotFound -def configUser(userConfig: UserConfiguration, retryCount: int = 0) -> None: +def configUser(retryCount: int = 0) -> UserConfiguration: if retryCount >= 3: raise RuntimeError("Failed to authenticate. Terminating...") + userConfig = UserConfiguration({}) # create new empty user config + username = ui.clickPrompt("Email", type = str) password = ui.clickPrompt("Password", type = str, hide_input = True) @@ -32,7 +34,7 @@ def configUser(userConfig: UserConfiguration, retryCount: int = 0) -> None: if response.hasFailed(): ui.errorEcho("Failed to authenticate. Please try again...") - return configUser(userConfig, retryCount + 1) + return configUser(retryCount + 1) jsonResponse = response.getJson(dict) userConfig.username = username @@ -42,6 +44,8 @@ def configUser(userConfig: UserConfiguration, retryCount: int = 0) -> None: userConfig.refreshToken = jsonResponse.get("refresh_token") userConfig.refreshTokenExpirationDate = jsonResponse.get("refresh_expires_on") + return userConfig + def authenticateUser(userConfig: UserConfiguration) -> None: response = networkManager.authenticate(userConfig.username, userConfig.password) @@ -49,7 +53,7 @@ def authenticateUser(userConfig: UserConfiguration) -> None: if response.statusCode >= 500: raise NetworkRequestError(response, "Something went wrong, please try again later.") elif response.statusCode >= 400: - configUser(userConfig) + userConfig.update(configUser()) else: jsonResponse = response.getJson(dict) userConfig.token = jsonResponse["token"] @@ -75,14 +79,16 @@ def authenticateWithRefreshToken(userConfig: UserConfiguration) -> None: def initializeUserSession() -> None: - userConfig = UserConfiguration() - - if not userConfig.isUserConfigured(): - ui.errorEcho("User configuration not found. Please authenticate with your credentials.") - configUser(userConfig) - elif not userConfig.isTokenValid("token") and userConfig.isTokenValid("refreshToken"): - authenticateWithRefreshToken(userConfig) - else: - authenticateUser(userConfig) + try: + userConfig = UserConfiguration.load() + if userConfig.isTokenValid("token"): + return + + if not userConfig.isTokenValid("token") and userConfig.isTokenValid("refreshToken"): + authenticateWithRefreshToken(userConfig) + else: + authenticateUser(userConfig) + except (ConfigurationNotFound, InvalidConfiguration): + userConfig = configUser() userConfig.save() diff --git a/coretex/configuration/__init__.py b/coretex/configuration/__init__.py index 7247699c..5335f56f 100644 --- a/coretex/configuration/__init__.py +++ b/coretex/configuration/__init__.py @@ -15,15 +15,88 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from pathlib import Path + +import os +import json +import logging + from .user import UserConfiguration from .node import NodeConfiguration -from .base import CONFIG_DIR, DEFAULT_VENV_PATH +from .base import CONFIG_DIR, DEFAULT_VENV_PATH, InvalidConfiguration, ConfigurationNotFound +from ..utils import isCliRuntime + + +def configMigration(configPath: Path) -> None: + with configPath.open("r") as file: + oldConfig = json.load(file) + + userRaw = { + "username": oldConfig.get("username"), + "password": oldConfig.get("password"), + "token": oldConfig.get("token"), + "refreshToken": oldConfig.get("refreshToken"), + "tokenExpirationDate": oldConfig.get("tokenExpirationDate"), + "refreshTokenExpirationDate": oldConfig.get("refreshTokenExpirationDate"), + "serverUrl": oldConfig.get("serverUrl"), + "projectId": oldConfig.get("projectId"), + } + + nodeRaw = { + "nodeName": oldConfig.get("nodeName"), + "nodeAccessToken": oldConfig.get("nodeAccessToken"), + "storagePath": oldConfig.get("storagePath"), + "image": oldConfig.get("image"), + "allowGpu": oldConfig.get("allowGpu"), + "nodeRam": oldConfig.get("nodeRam"), + "nodeSharedMemory": oldConfig.get("nodeSharedMemory"), + "cpuCount": oldConfig.get("cpuCount"), + "nodeMode": oldConfig.get("nodeMode"), + "allowDocker": oldConfig.get("allowDocker"), + "nodeSecret": oldConfig.get("nodeSecret"), + "initScript": oldConfig.get("initScript"), + "modelId": oldConfig.get("modelId"), + } + + userConfig = UserConfiguration(userRaw) + nodeConfig = NodeConfiguration(nodeRaw) + userConfig.save() + nodeConfig.save() + configPath.unlink() + def _syncConfigWithEnv() -> None: + print("i am syncing") # If configuration doesn't exist default one will be created # Initialization of User and Node Configuration classes will do # the necessary sync between config properties and corresponding # environment variables (e.g. storagePath -> CTX_STORAGE_PATH) - UserConfiguration() - NodeConfiguration() + # old configuration exists, fill out new config files with old configuration + oldConfigPath = CONFIG_DIR / "config.json" + if oldConfigPath.exists(): + configMigration(oldConfigPath) + + try: + userConfig = UserConfiguration.load() + if not "CTX_API_URL" in os.environ: + os.environ["CTX_API_URL"] = userConfig.serverUrl + except (ConfigurationNotFound, InvalidConfiguration) as ex: + if not isCliRuntime(): + logging.error(f">> [Coretex] Error loading configuration. Reason: {ex}") + logging.info("\tIf this message from Coretex Node you can safely ignore it.") + + try: + nodeConfig = NodeConfiguration.load() + + if isinstance(nodeConfig.nodeSecret, str) and nodeConfig.nodeSecret != "": + os.environ["CTX_SECRETS_KEY"] = nodeConfig.nodeSecret + + if not isCliRuntime(): + os.environ["CTX_STORAGE_PATH"] = nodeConfig.storagePath + else: + os.environ["CTX_STORAGE_PATH"] = f"{CONFIG_DIR}/data" + except (ConfigurationNotFound, InvalidConfiguration) as ex: + if not isCliRuntime(): + logging.error(f">> [Coretex] Error loading configuration. Reason: {ex}") + logging.info("\tIf this message from Coretex Node you can safely ignore it.") diff --git a/coretex/configuration/base.py b/coretex/configuration/base.py index 595ebebb..7eeb9601 100644 --- a/coretex/configuration/base.py +++ b/coretex/configuration/base.py @@ -15,7 +15,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Dict, Any, Optional, Type, TypeVar +from typing import Dict, Any, Optional, Type, TypeVar, List, Tuple +from typing_extensions import Self from abc import abstractmethod from pathlib import Path @@ -30,36 +31,69 @@ class InvalidConfiguration(Exception): + + def __init__(self, message: str,errors: List[str]) -> None: + super().__init__(message) + self.errors = errors + + +class ConfigurationNotFound(Exception): pass -class BaseConfiguration: - def __init__(self, path: Path) -> None: - self._path = path +class BaseConfiguration: - if not path.exists(): - self._raw = self.getDefaultConfig() - self.save() - else: - with path.open("r") as file: - self._raw = json.load(file) + def __init__(self, raw: Dict[str, Any]) -> None: + self._raw = raw @classmethod @abstractmethod - def getDefaultConfig(cls) -> Dict[str, Any]: + def getConfigPath(cls) -> Path: + pass + + @abstractmethod + def _isConfigured(self) -> bool: + pass + + @abstractmethod + def _isConfigValid(self) -> Tuple[bool, List[str]]: pass + @classmethod + def load(cls) -> Self: + configPath = cls.getConfigPath() + if not configPath.exists(): + raise ConfigurationNotFound(f"Configuration not found at path: {configPath}") + + with configPath.open("r") as file: + raw = json.load(file) + + config = cls(raw) + + isValid, errors = config._isConfigValid() + + if not config._isConfigured(): + raise ConfigurationNotFound("Configuration not found.") + + if not isValid: + raise InvalidConfiguration("Invalid configuration found.", errors) + + return config + def _value(self, configKey: str, valueType: Type[T], envKey: Optional[str] = None) -> Optional[T]: if envKey is not None and envKey in os.environ: return valueType(os.environ[envKey]) return self._raw.get(configKey) - def getValue(self, configKey: str, valueType: Type[T], envKey: Optional[str] = None) -> T: + def getValue(self, configKey: str, valueType: Type[T], envKey: Optional[str] = None, default: Optional[T] = None) -> T: value = self._value(configKey, valueType, envKey) + if value is None: + value = default + if not isinstance(value, valueType): - raise InvalidConfiguration(f"Invalid {configKey} type \"{type(value)}\", expected: \"{valueType.__name__}\".") + raise TypeError(f"Invalid {configKey} type \"{type(value)}\", expected: \"{valueType.__name__}\".") return value @@ -72,8 +106,13 @@ def getOptValue(self, configKey: str, valueType: Type[T], envKey: Optional[str] return value def save(self) -> None: - if not self._path.parent.exists(): - self._path.parent.mkdir(parents = True, exist_ok = True) + configPath = self.getConfigPath() - with self._path.open("w") as configFile: + if not configPath.parent.exists(): + configPath.parent.mkdir(parents = True, exist_ok = True) + + with configPath.open("w") as configFile: json.dump(self._raw, configFile, indent = 4) + + def update(self, config: Self) -> None: + self._raw = config._raw diff --git a/coretex/configuration/node.py b/coretex/configuration/node.py index 0100348c..6594657f 100644 --- a/coretex/configuration/node.py +++ b/coretex/configuration/node.py @@ -22,7 +22,7 @@ from . import config_defaults from .base import BaseConfiguration, CONFIG_DIR -from ..utils import docker, isCliRuntime +from ..utils import docker from ..node import NodeMode @@ -33,44 +33,14 @@ def getEnvVar(key: str, default: str) -> str: return os.environ[key] -NODE_CONFIG_PATH = CONFIG_DIR / "node_config.json" -NODE_DEFAULT_CONFIG = { - "nodeName": os.environ.get("CTX_NODE_NAME"), - "nodeAccessToken": None, - "storagePath": getEnvVar("CTX_STORAGE_PATH", "~/.coretex"), - "image": "coretexai/coretex-node:latest-cpu", - "allowGpu": False, - "nodeRam": None, - "nodeSharedMemory": None, - "cpuCount": None, - "nodeMode": None, - "allowDocker": False, - "nodeSecret": None, - "initScript": None, - "modelId": None, -} - - -class InvalidNodeConfiguration(Exception): - pass - - class NodeConfiguration(BaseConfiguration): - def __init__(self) -> None: - super().__init__(NODE_CONFIG_PATH) - nodeSecret = self.nodeSecret - if isinstance(nodeSecret, str) and nodeSecret != "": - os.environ["CTX_SECRETS_KEY"] = nodeSecret - - if not isCliRuntime(): - os.environ["CTX_STORAGE_PATH"] = self.storagePath - else: - os.environ["CTX_STORAGE_PATH"] = f"{CONFIG_DIR}/data" + def __init__(self, raw: Dict[str, Any]) -> None: + super().__init__(raw) @classmethod - def getDefaultConfig(cls) -> Dict[str, Any]: - return NODE_DEFAULT_CONFIG + def getConfigPath(cls) -> Path: + return CONFIG_DIR / "node_config.json" @property def nodeName(self) -> str: @@ -169,7 +139,7 @@ def nodeMode(self) -> int: nodeMode = self.getOptValue("nodeMode", int) if nodeMode is None: - nodeMode = NodeMode.execution + nodeMode = NodeMode.any return nodeMode @@ -225,7 +195,7 @@ def endpointInvocationPrice(self) -> Optional[float]: def endpointInvocationPrice(self, value: Optional[float]) -> None: self._raw["endpointInvocationPrice"] = value - def isNodeConfigured(self) -> bool: + def _isConfigured(self) -> bool: return ( isinstance(self._raw.get("nodeName"), str) and isinstance(self._raw.get("image"), str) and @@ -237,7 +207,7 @@ def isNodeConfigured(self) -> bool: isinstance(self._raw.get("nodeMode"), int) ) - def isConfigurationValid(self) -> Tuple[bool, List[str]]: + def _isConfigValid(self) -> Tuple[bool, List[str]]: isValid = True errorMessages = [] cpuLimit, ramLimit = docker.getResourceLimits() @@ -299,7 +269,6 @@ def isConfigurationValid(self) -> Tuple[bool, List[str]]: return isValid, errorMessages - def getInitScriptPath(self) -> Optional[Path]: value = self._raw.get("initScript") diff --git a/coretex/configuration/user.py b/coretex/configuration/user.py index 9903fef8..a0f3145c 100644 --- a/coretex/configuration/user.py +++ b/coretex/configuration/user.py @@ -15,7 +15,9 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Dict, Any, Optional +from pathlib import Path +from typing import Dict, Any, List, Optional, Tuple +from typing_extensions import Self from datetime import datetime, timezone import os @@ -25,28 +27,19 @@ USER_CONFIG_PATH = CONFIG_DIR / "user_config.json" -USER_DEFAULT_CONFIG = { - "username": os.environ.get("CTX_USERNAME"), - "password": os.environ.get("CTX_PASSWORD"), - "token": None, - "refreshToken": None, - "tokenExpirationDate": None, - "refreshTokenExpirationDate": None, - "serverUrl": os.environ.get("CTX_API_URL", "https://api.coretex.ai/"), - "projectId": os.environ.get("CTX_PROJECT_ID") -} class UserConfiguration(BaseConfiguration): - def __init__(self) -> None: - super().__init__(USER_CONFIG_PATH) + def __init__(self, raw: Dict[str, Any]) -> None: + super().__init__(raw) + if not "CTX_API_URL" in os.environ: os.environ["CTX_API_URL"] = self.serverUrl @classmethod - def getDefaultConfig(cls) -> Dict[str, Any]: - return USER_DEFAULT_CONFIG + def getConfigPath(cls) -> Path: + return CONFIG_DIR / "user_config.json" @property def username(self) -> str: @@ -98,7 +91,7 @@ def refreshTokenExpirationDate(self, value: Optional[str]) -> None: @property def serverUrl(self) -> str: - return self.getValue("serverUrl", str, "CTX_API_URL") + return self.getValue("serverUrl", str, "CTX_API_URL", "https://api.coretex.ai/") @serverUrl.setter def serverUrl(self, value: str) -> None: @@ -112,18 +105,22 @@ def projectId(self) -> Optional[int]: def projectId(self, value: Optional[int]) -> None: self._raw["projectId"] = value - @property - def isValid(self) -> bool: + def _isConfigured(self) -> bool: return self._raw.get("username") is not None and self._raw.get("password") is not None - def isUserConfigured(self) -> bool: + def _isConfigValid(self) -> Tuple[bool, List[str]]: + isValid = True + errorMessages = [] + if self._raw.get("username") is None or not isinstance(self._raw.get("username"), str): - return False + errorMessages.append("Field \"username\" not found in configuration.") + isValid = False if self._raw.get("password") is None or not isinstance(self._raw.get("password"), str): - return False + errorMessages.append("Field \"password\" not found in configuration.") + isValid = False - return True + return isValid, errorMessages def isTokenValid(self, tokenName: str) -> bool: tokenValue = self._raw.get(tokenName) diff --git a/coretex/networking/network_manager.py b/coretex/networking/network_manager.py index 3985a5d0..432f8561 100644 --- a/coretex/networking/network_manager.py +++ b/coretex/networking/network_manager.py @@ -33,27 +33,39 @@ class NetworkManager(NetworkManagerBase): def __init__(self) -> None: super().__init__() - self.__userConfig = UserConfiguration() + self._userApiToken: Optional[str] = None + self._userRefreshToken: Optional[str] = None + self._username: Optional[str] = None + self._password: Optional[str] = None + + try: + userConfig = UserConfiguration.load() + self._userApiToken = userConfig.token + self._userRefreshToken = userConfig.refreshToken + self._username = userConfig.username + self._password = userConfig.password + except: + pass @property def _apiToken(self) -> Optional[str]: - return self.__userConfig.token + return self._userApiToken @_apiToken.setter def _apiToken(self, value: Optional[str]) -> None: - self.__userConfig.token = value + self._userApiToken = value @property def _refreshToken(self) -> Optional[str]: - return self.__userConfig.refreshToken + return self._userRefreshToken @_refreshToken.setter def _refreshToken(self, value: Optional[str]) -> None: - self.__userConfig.refreshToken = value + self._userRefreshToken = value @property def hasStoredCredentials(self) -> bool: - return self.__userConfig.isUserConfigured() + return self._username is not None and self._password is not None def authenticate(self, username: str, password: str, storeCredentials: bool = True) -> NetworkResponse: """ @@ -82,8 +94,8 @@ def authenticate(self, username: str, password: str, storeCredentials: bool = Tr """ if storeCredentials: - self.__userConfig.username = username - self.__userConfig.password = password + self._username = username + self._password = password # authenticate using credentials stored in requests.Session.auth return super().authenticate(username, password, storeCredentials) @@ -101,10 +113,10 @@ def authenticateWithStoredCredentials(self) -> NetworkResponse: ValueError -> if credentials are not found """ - if self.__userConfig.username is None or self.__userConfig.password is None: + if self._username is None or self._password is None: raise ValueError(">> [Coretex] Credentials not stored") - return self.authenticate(self.__userConfig.username, self.__userConfig.password) + return self.authenticate(self._username, self._password) networkManager: NetworkManagerBase = NetworkManager() From a0594c3037d0e53bd52c1003eb98dd615dec2e16 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Tue, 6 Aug 2024 14:41:26 +0200 Subject: [PATCH 22/28] CTX-5430: Removed isConfigured() method since it was just causing troubles, ive added isConfigured checks in isConfigValid. More improvements, migration fix. --- coretex/cli/modules/user.py | 15 ++- coretex/configuration/__init__.py | 10 +- coretex/configuration/base.py | 8 -- coretex/configuration/config_defaults.py | 6 +- coretex/configuration/node.py | 152 ++++++++++++++++------- coretex/configuration/user.py | 13 +- 6 files changed, 132 insertions(+), 72 deletions(-) diff --git a/coretex/cli/modules/user.py b/coretex/cli/modules/user.py index da2051dc..92e3da73 100644 --- a/coretex/cli/modules/user.py +++ b/coretex/cli/modules/user.py @@ -88,7 +88,20 @@ def initializeUserSession() -> None: authenticateWithRefreshToken(userConfig) else: authenticateUser(userConfig) - except (ConfigurationNotFound, InvalidConfiguration): + except ConfigurationNotFound: + ui.errorEcho("User configuration not found.") + if not ui.clickPrompt("Would you like to configure the user? (Y/n)", type = bool, default = True, show_default = False): + raise + + userConfig = configUser() + except InvalidConfiguration as ex: + ui.errorEcho("Invalid user configuration found.") + for error in ex.errors: + ui.errorEcho(f"{error}") + + if not ui.clickPrompt("Would you like to reconfigure the user? (Y/n)", type = bool, default = True, show_default = False): + raise + userConfig = configUser() userConfig.save() diff --git a/coretex/configuration/__init__.py b/coretex/configuration/__init__.py index 5335f56f..0bd9e032 100644 --- a/coretex/configuration/__init__.py +++ b/coretex/configuration/__init__.py @@ -30,6 +30,7 @@ def configMigration(configPath: Path) -> None: with configPath.open("r") as file: oldConfig = json.load(file) + logging.warning(f"[Coretex] >> WARNING: Old configuration:\n{oldConfig}") userRaw = { "username": oldConfig.get("username"), @@ -66,7 +67,6 @@ def configMigration(configPath: Path) -> None: def _syncConfigWithEnv() -> None: - print("i am syncing") # If configuration doesn't exist default one will be created # Initialization of User and Node Configuration classes will do # the necessary sync between config properties and corresponding @@ -75,6 +75,10 @@ def _syncConfigWithEnv() -> None: # old configuration exists, fill out new config files with old configuration oldConfigPath = CONFIG_DIR / "config.json" if oldConfigPath.exists(): + logging.warning( + f"[Coretex] >> WARNING: Old configuration found at path: {oldConfigPath}. Migrating to new configuration." + f"\nFields with invalid values might be overrided in this process." + ) configMigration(oldConfigPath) try: @@ -84,7 +88,7 @@ def _syncConfigWithEnv() -> None: except (ConfigurationNotFound, InvalidConfiguration) as ex: if not isCliRuntime(): logging.error(f">> [Coretex] Error loading configuration. Reason: {ex}") - logging.info("\tIf this message from Coretex Node you can safely ignore it.") + logging.info("\tIf this message came from Coretex Node you can safely ignore it.") try: nodeConfig = NodeConfiguration.load() @@ -99,4 +103,4 @@ def _syncConfigWithEnv() -> None: except (ConfigurationNotFound, InvalidConfiguration) as ex: if not isCliRuntime(): logging.error(f">> [Coretex] Error loading configuration. Reason: {ex}") - logging.info("\tIf this message from Coretex Node you can safely ignore it.") + logging.info("\tIf this message came from Coretex Node you can safely ignore it.") diff --git a/coretex/configuration/base.py b/coretex/configuration/base.py index 7eeb9601..0ac56d68 100644 --- a/coretex/configuration/base.py +++ b/coretex/configuration/base.py @@ -51,10 +51,6 @@ def __init__(self, raw: Dict[str, Any]) -> None: def getConfigPath(cls) -> Path: pass - @abstractmethod - def _isConfigured(self) -> bool: - pass - @abstractmethod def _isConfigValid(self) -> Tuple[bool, List[str]]: pass @@ -71,10 +67,6 @@ def load(cls) -> Self: config = cls(raw) isValid, errors = config._isConfigValid() - - if not config._isConfigured(): - raise ConfigurationNotFound("Configuration not found.") - if not isValid: raise InvalidConfiguration("Invalid configuration found.", errors) diff --git a/coretex/configuration/config_defaults.py b/coretex/configuration/config_defaults.py index ee1cf3f2..57feaffc 100644 --- a/coretex/configuration/config_defaults.py +++ b/coretex/configuration/config_defaults.py @@ -28,9 +28,9 @@ DOCKER_CONTAINER_NAME = "coretex_node" DOCKER_CONTAINER_NETWORK = "coretex_node" DEFAULT_STORAGE_PATH = str(Path.home() / ".coretex") -DEFAULT_RAM_MEMORY = getAvailableRam() -MINIMUM_RAM_MEMORY = 6 -DEFAULT_SWAP_MEMORY = DEFAULT_RAM_MEMORY * 2 +DEFAULT_RAM = getAvailableRam() +MINIMUM_RAM = 6 +DEFAULT_SWAP_MEMORY = DEFAULT_RAM * 2 DEFAULT_SHARED_MEMORY = 2 DEFAULT_CPU_COUNT = cpuCount if cpuCount is not None else 0 DEFAULT_NODE_MODE = NodeMode.execution diff --git a/coretex/configuration/node.py b/coretex/configuration/node.py index 6594657f..bc46f478 100644 --- a/coretex/configuration/node.py +++ b/coretex/configuration/node.py @@ -33,6 +33,10 @@ def getEnvVar(key: str, default: str) -> str: return os.environ[key] +class DockerConfigurationException(Exception): + pass + + class NodeConfiguration(BaseConfiguration): def __init__(self, raw: Dict[str, Any]) -> None: @@ -87,7 +91,7 @@ def nodeRam(self) -> int: nodeRam = self.getOptValue("nodeRam", int) if nodeRam is None: - nodeRam = config_defaults.DEFAULT_RAM_MEMORY + nodeRam = config_defaults.DEFAULT_RAM return nodeRam @@ -195,78 +199,134 @@ def endpointInvocationPrice(self) -> Optional[float]: def endpointInvocationPrice(self, value: Optional[float]) -> None: self._raw["endpointInvocationPrice"] = value - def _isConfigured(self) -> bool: - return ( - isinstance(self._raw.get("nodeName"), str) and - isinstance(self._raw.get("image"), str) and - isinstance(self._raw.get("nodeAccessToken"), str) and - isinstance(self._raw.get("cpuCount"), int) and - isinstance(self._raw.get("nodeRam"), int) and - isinstance(self._raw.get("nodeSwap"), int) and - isinstance(self._raw.get("nodeSharedMemory"), int) and - isinstance(self._raw.get("nodeMode"), int) - ) - - def _isConfigValid(self) -> Tuple[bool, List[str]]: + def validateRamField(self, ramLimit: int) -> Tuple[bool, int, str]: isValid = True - errorMessages = [] - cpuLimit, ramLimit = docker.getResourceLimits() - swapLimit = docker.getDockerSwapLimit() + message = "" + + if ramLimit < config_defaults.MINIMUM_RAM: + isValid = False + raise DockerConfigurationException( + f"Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM}GB) " + f"is higher than your current Docker desktop RAM limit ({ramLimit}GB). " + "Please adjust resource limitations in Docker Desktop settings to match Node requirements." + ) + + defaultRamValue = int(min(max(config_defaults.MINIMUM_RAM, ramLimit), config_defaults.DEFAULT_RAM)) if not isinstance(self._raw.get("nodeRam"), int): - errorMessages.append( + isValid = False + message = ( f"Invalid config \"nodeRam\" field type \"{type(self._raw.get('nodeRam'))}\". Expected: \"int\"" + f"Using default value of {defaultRamValue} GB" ) + + if self.nodeRam < config_defaults.MINIMUM_RAM: isValid = False + message = ( + f"WARNING: Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM}GB) " + f"is higher than the configured value ({self._raw.get('nodeRam')}GB)" + f"Overriding \"nodeRam\" field to match node RAM requirements." + ) + + if self.nodeRam > ramLimit: + isValid = False + message = ( + f"WARNING: RAM limit in Docker Desktop ({ramLimit}GB) " + f"is lower than the configured value ({self._raw.get('nodeRam')}GB)" + f"Overriding \"nodeRam\" field to limit in Docker Desktop." + ) + + return isValid, defaultRamValue, message + + def validateCPUCount(self, cpuLimit: int) -> Tuple[bool, int, str]: + isValid = True + message = "" + defaultCPUCount = config_defaults.DEFAULT_CPU_COUNT if config_defaults.DEFAULT_CPU_COUNT <= cpuLimit else cpuLimit if not isinstance(self._raw.get("cpuCount"), int): - errorMessages.append( + isValid = False + message = ( f"Invalid config \"cpuCount\" field type \"{type(self._raw.get('cpuCount'))}\". Expected: \"int\"" + f"Using default value of {defaultCPUCount} cores" ) + + if self.cpuCount > cpuLimit: isValid = False + message = ( + f"WARNING: CPU limit in Docker Desktop ({cpuLimit}) " + f"is lower than the configured value ({self._raw.get('cpuCount')})" + f"Overriding \"cpuCount\" field to limit in Docker Desktop." + ) + + return isValid, cpuLimit, message + + def validateSWAPMemory(self, swapLimit: int) -> Tuple[bool, int, str]: + isValid = True + message = "" + defaultSWAPMemory = config_defaults.DEFAULT_SWAP_MEMORY if config_defaults.DEFAULT_SWAP_MEMORY <= swapLimit else swapLimit if not isinstance(self._raw.get("nodeSwap"), int): - errorMessages.append( + isValid = False + message = ( f"Invalid config \"nodeSwap\" field type \"{type(self._raw.get('nodeSwap'))}\". Expected: \"int\"" + f"Using default value of {defaultSWAPMemory} GB" ) - isValid = False - if self.cpuCount > cpuLimit: - errorMessages.append( - f"Configuration not valid. CPU limit in Docker Desktop ({cpuLimit}) " - "is lower than the configured value ({self._raw.get('cpuCount')})" + if self.nodeSwap > swapLimit: + isValid = False + message = ( + f"WARNING: SWAP limit in Docker Desktop ({swapLimit}GB) " + f"is lower than the configured value ({self.nodeSwap}GB)" + f"Overriding \"nodeSwap\" field to limit in Docker Desktop." ) + + return isValid, defaultSWAPMemory, message + + def _isConfigValid(self) -> Tuple[bool, List[str]]: + isValid = True + errorMessages = [] + cpuLimit, ramLimit = docker.getResourceLimits() + swapLimit = docker.getDockerSwapLimit() + + if not isinstance(self._raw.get("nodeName"), str): isValid = False + errorMessages.append("Required field \"nodeName\" missing") - if ramLimit < config_defaults.MINIMUM_RAM_MEMORY: - errorMessages.append( - f"Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM_MEMORY}GB) " - "is higher than your current Docker desktop RAM limit ({ramLimit}GB). " - "Please adjust resource limitations in Docker Desktop settings to match Node requirements." - ) + if not isinstance(self._raw.get("image"), str): isValid = False + errorMessages.append("Required field \"image\" missing") - if self.nodeRam > ramLimit: - errorMessages.append( - f"Configuration not valid. RAM limit in Docker Desktop ({ramLimit}GB) " - "is lower than the configured value ({self._raw.get('nodeRam')}GB)" - ) + if not isinstance(self._raw.get("nodeAccessToken"), str): isValid = False + errorMessages.append("Required field \"nodeAccessToken\" missing") - if self.nodeRam < config_defaults.MINIMUM_RAM_MEMORY: - errorMessages.append( - f"Configuration not valid. Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM_MEMORY}GB) " - "is higher than the configured value ({self._raw.get('nodeRam')}GB)" - ) + if not isinstance(self._raw.get("nodeName"), str): + errorMessages.append(f"Invalid configuration. Missing required field \"nodeName\".") isValid = False - if self.nodeSwap > swapLimit: - errorMessages.append( - f"Configuration not valid. SWAP limit in Docker Desktop ({swapLimit}GB) " - "is lower than the configured value ({self.nodeSwap}GB)" - ) + if not isinstance(self._raw.get("image"), str): + errorMessages.append(f"Invalid configuration. Missing required field \"image\".") + isValid = False + + if not isinstance(self._raw.get("nodeAccessToken"), str): + errorMessages.append(f"Invalid configuration. Missing required field \"nodeAccessToken\".") isValid = False + isRamValid, nodeRam, message = self.validateRamField(ramLimit) + if not isRamValid: + errorMessages.append(message) + self.nodeRam = nodeRam + + isCPUCountValid, cpuCount, message = self.validateCPUCount(cpuLimit) + if not isCPUCountValid: + errorMessages.append(message) + self.cpuCount = cpuCount + + isSWAPMemoryValid, nodeSwap, message = self.validateSWAPMemory(swapLimit) + if not isSWAPMemoryValid: + errorMessages.append(message) + self.nodeSwap = nodeSwap + return isValid, errorMessages def getInitScriptPath(self) -> Optional[Path]: diff --git a/coretex/configuration/user.py b/coretex/configuration/user.py index a0f3145c..74bfc17d 100644 --- a/coretex/configuration/user.py +++ b/coretex/configuration/user.py @@ -17,11 +17,8 @@ from pathlib import Path from typing import Dict, Any, List, Optional, Tuple -from typing_extensions import Self from datetime import datetime, timezone -import os - from .base import BaseConfiguration, CONFIG_DIR from ..utils import decodeDate @@ -34,9 +31,6 @@ class UserConfiguration(BaseConfiguration): def __init__(self, raw: Dict[str, Any]) -> None: super().__init__(raw) - if not "CTX_API_URL" in os.environ: - os.environ["CTX_API_URL"] = self.serverUrl - @classmethod def getConfigPath(cls) -> Path: return CONFIG_DIR / "user_config.json" @@ -105,20 +99,17 @@ def projectId(self) -> Optional[int]: def projectId(self, value: Optional[int]) -> None: self._raw["projectId"] = value - def _isConfigured(self) -> bool: - return self._raw.get("username") is not None and self._raw.get("password") is not None - def _isConfigValid(self) -> Tuple[bool, List[str]]: isValid = True errorMessages = [] if self._raw.get("username") is None or not isinstance(self._raw.get("username"), str): - errorMessages.append("Field \"username\" not found in configuration.") isValid = False + errorMessages.append("Missing required field \"username\" in user configuration.") if self._raw.get("password") is None or not isinstance(self._raw.get("password"), str): - errorMessages.append("Field \"password\" not found in configuration.") isValid = False + errorMessages.append("Missing required field \"password\" in user configuration.") return isValid, errorMessages From 2d8f4ed0c6e5cac246827b0e509fa081ee80ab1c Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 7 Aug 2024 13:04:58 +0200 Subject: [PATCH 23/28] CTX-5430: Pipeline fix. --- coretex/_folder_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coretex/_folder_manager.py b/coretex/_folder_manager.py index a23ac1f9..72d42d02 100644 --- a/coretex/_folder_manager.py +++ b/coretex/_folder_manager.py @@ -185,4 +185,4 @@ def tempFile(self, name: Optional[str] = None) -> Iterator[Path]: path.unlink(missing_ok = True) -folder_manager = FolderManager(os.environ["CTX_STORAGE_PATH"]) +folder_manager = FolderManager(os.environ.get("CTX_STORAGE_PATH", "~/.coretex")) From 596d3e769a31163cb9b7f77a253be8343ee1cec8 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Fri, 9 Aug 2024 10:59:58 +0200 Subject: [PATCH 24/28] CTX-5430: Cleanup, changes regarding discussions on PR. --- coretex/_folder_manager.py | 2 +- coretex/cli/commands/node.py | 29 +-- coretex/cli/commands/project.py | 8 +- coretex/cli/commands/task.py | 11 +- coretex/cli/modules/config_defaults.py | 6 +- coretex/cli/modules/node.py | 55 +++--- coretex/cli/modules/project_utils.py | 27 +-- coretex/cli/modules/ui.py | 15 +- coretex/cli/modules/update.py | 2 - coretex/cli/modules/user.py | 1 + coretex/configuration/__init__.py | 55 ++---- coretex/configuration/base.py | 3 +- coretex/configuration/node.py | 250 ++++++++----------------- coretex/configuration/user.py | 25 ++- coretex/configuration/utils.py | 104 ++++++++++ coretex/utils/__init__.py | 1 - coretex/zkml/__init__.py | 16 ++ coretex/{utils => zkml}/inference.py | 20 +- 18 files changed, 325 insertions(+), 305 deletions(-) create mode 100644 coretex/configuration/utils.py create mode 100644 coretex/zkml/__init__.py rename coretex/{utils => zkml}/inference.py (77%) diff --git a/coretex/_folder_manager.py b/coretex/_folder_manager.py index 72d42d02..a23ac1f9 100644 --- a/coretex/_folder_manager.py +++ b/coretex/_folder_manager.py @@ -185,4 +185,4 @@ def tempFile(self, name: Optional[str] = None) -> Iterator[Path]: path.unlink(missing_ok = True) -folder_manager = FolderManager(os.environ.get("CTX_STORAGE_PATH", "~/.coretex")) +folder_manager = FolderManager(os.environ["CTX_STORAGE_PATH"]) diff --git a/coretex/cli/commands/node.py b/coretex/cli/commands/node.py index 9839ffc5..06809fbe 100644 --- a/coretex/cli/commands/node.py +++ b/coretex/cli/commands/node.py @@ -34,7 +34,6 @@ @onBeforeCommandExecute(node_module.initializeNodeConfiguration) def start(image: Optional[str]) -> None: nodeConfig = NodeConfiguration.load() - userConfig = UserConfiguration.load() if node_module.isRunning(): if not ui.clickPrompt( @@ -45,7 +44,7 @@ def start(image: Optional[str]) -> None: ): return - node_module.stop(nodeConfig.nodeId) + node_module.stop(nodeConfig.id) if node_module.exists(): node_module.clean() @@ -60,7 +59,7 @@ def start(image: Optional[str]) -> None: if node_module.shouldUpdate(dockerImage): node_module.pull(dockerImage) - node_module.start(dockerImage, userConfig, nodeConfig) + node_module.start(dockerImage, nodeConfig) docker.removeDanglingImages(node_module.getRepoFromImageUrl(dockerImage), node_module.getTagFromImageUrl(dockerImage)) activateAutoUpdate() @@ -73,7 +72,7 @@ def stop() -> None: ui.errorEcho("Node is already offline.") return - node_module.stop(nodeConfig.nodeId) + node_module.stop(nodeConfig.id) @click.command() @@ -85,9 +84,7 @@ def update(autoAccept: bool, autoDecline: bool) -> None: ui.errorEcho("Only one of the flags (\"-y\" or \"-n\") can be used at the same time.") return - userConfig = UserConfiguration.load() nodeConfig = NodeConfiguration.load() - nodeStatus = node_module.getNodeStatus() if nodeStatus == NodeStatus.inactive: @@ -110,13 +107,13 @@ def update(autoAccept: bool, autoDecline: bool) -> None: ): return - node_module.stop(nodeConfig.nodeId) + node_module.stop(nodeConfig.id) if not node_module.shouldUpdate(nodeConfig.image): ui.successEcho("Node is already up to date.") return - ui.stdEcho("Updating node...") + ui.stdEcho("Fetching latest node image...") node_module.pull(nodeConfig.image) if getNodeStatus() == NodeStatus.busy and not autoAccept: @@ -131,15 +128,16 @@ def update(autoAccept: bool, autoDecline: bool) -> None: ): return - node_module.stop(nodeConfig.nodeId) + node_module.stop(nodeConfig.id) ui.stdEcho("Updating node...") - node_module.start(nodeConfig.image, userConfig, nodeConfig) + node_module.start(nodeConfig.image, nodeConfig) docker.removeDanglingImages( node_module.getRepoFromImageUrl(nodeConfig.image), node_module.getTagFromImageUrl(nodeConfig.image) ) + activateAutoUpdate() @click.command() @@ -155,10 +153,13 @@ def config(verbose: bool) -> None: ui.errorEcho("If you wish to reconfigure your node, use coretex node stop commands first.") return - nodeConfig = NodeConfiguration.load() - node_module.stop(nodeConfig.nodeId) + try: + nodeConfig = NodeConfiguration.load() + node_module.stop(nodeConfig.id) + except (ConfigurationNotFound, InvalidConfiguration): + node_module.stop() + - userConfig = UserConfiguration.load() try: nodeConfig = NodeConfiguration.load() if not ui.clickPrompt( @@ -173,7 +174,7 @@ def config(verbose: bool) -> None: nodeConfig = node_module.configureNode(verbose) nodeConfig.save() - ui.previewConfig(userConfig, nodeConfig) + ui.previewNodeConfig(nodeConfig) ui.successEcho("Node successfully configured.") activateAutoUpdate() diff --git a/coretex/cli/commands/project.py b/coretex/cli/commands/project.py index 12d582ea..f701d881 100644 --- a/coretex/cli/commands/project.py +++ b/coretex/cli/commands/project.py @@ -33,10 +33,11 @@ @click.option("--description", "-d", type = str, help = "Project description") def create(name: Optional[str], projectType: Optional[int], description: Optional[str]) -> None: project = project_utils.createProject(name, projectType, description) + userConfig = UserConfiguration.load() selectNewProject = ui.clickPrompt("Do you want to select the new project as default? (Y/n)", type = bool, default = True) if selectNewProject: - project_utils.selectProject(project.id) + userConfig.selectProject(project.id) ui.successEcho(f"Project \"{project.name}\" successfully selected.") @@ -75,20 +76,21 @@ def edit(project: Optional[str], name: Optional[str], description: Optional[str] @click.argument("name", type = str) def select(name: str) -> None: project: Optional[Project] = None + userConfig = UserConfiguration.load() ui.progressEcho("Validating project...") try: project = Project.fetchOne(name = name) ui.successEcho(f"Project \"{name}\" selected successfully!") - project_utils.selectProject(project.id) + userConfig.selectProject(project.id) except ValueError: ui.errorEcho(f"Project \"{name}\" not found.") project = project_utils.promptProjectCreate("Do you want to create a project with that name?", name) if project is None: return - project_utils.selectProject(project.id) + userConfig.selectProject(project.id) @click.group() diff --git a/coretex/cli/commands/task.py b/coretex/cli/commands/task.py index 6d9fe2a3..ffb22dde 100644 --- a/coretex/cli/commands/task.py +++ b/coretex/cli/commands/task.py @@ -23,7 +23,7 @@ from ..modules.user import initializeUserSession from ..modules.utils import onBeforeCommandExecute from ..modules.project_utils import getProject -from ..._folder_manager import FolderManager +from ..._folder_manager import folder_manager from ..._task import TaskRunWorker, executeRunLocally, readTaskConfig, runLogger from ...configuration import UserConfiguration, NodeConfiguration from ...entities import TaskRun, TaskRunStatus @@ -44,19 +44,16 @@ class RunException(Exception): @click.option("--project", "-p", type = str) def run(path: str, name: Optional[str], description: Optional[str], snapshot: bool, project: Optional[str]) -> None: userConfig = UserConfiguration.load() - nodeConfig = NodeConfiguration.load() if userConfig.refreshToken is None: raise RunException(f"Failed to execute \"coretex run {path}\" command. Authenticate again using \"coretex login\" command and try again.") parameters = readTaskConfig() - # clearing temporary files in case that node was manually killed before - folderManager = FolderManager(nodeConfig.storagePath) - folderManager.clearTempFiles() + # clearing temporary files in case that local run was manually killed before + folder_manager.clearTempFiles() selectedProject = getProject(project, userConfig) - # clearing temporary files in case that node was manually killed before if selectedProject is None: return @@ -97,4 +94,4 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo else: taskRun.updateStatus(TaskRunStatus.completedWithSuccess) - folderManager.clearTempFiles() + folder_manager.clearTempFiles() diff --git a/coretex/cli/modules/config_defaults.py b/coretex/cli/modules/config_defaults.py index 3893ea7f..7c59b2f3 100644 --- a/coretex/cli/modules/config_defaults.py +++ b/coretex/cli/modules/config_defaults.py @@ -1,13 +1,11 @@ from pathlib import Path -import os +import multiprocessing from ...node import NodeMode from ...statistics import getAvailableRam, getAvailableCpuCount -cpuCount = os.cpu_count() - DOCKER_CONTAINER_NAME = "coretex_node" DOCKER_CONTAINER_NETWORK = "coretex_node" DEFAULT_STORAGE_PATH = str(Path.home() / ".coretex") @@ -15,7 +13,7 @@ MINIMUM_RAM = 6 DEFAULT_SWAP_MEMORY = DEFAULT_RAM * 2 DEFAULT_SHARED_MEMORY = 2 -DEFAULT_CPU_COUNT = getAvailableCpuCount() +DEFAULT_CPU_COUNT = multiprocessing.cpu_count() DEFAULT_NODE_MODE = NodeMode.execution DEFAULT_ALLOW_DOCKER = False DEFAULT_NODE_SECRET = "" diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index a4384e65..a09651aa 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -15,11 +15,12 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Any, Dict, Optional, Tuple, List +from typing import Any, Dict, Optional, Tuple from enum import Enum from pathlib import Path from base64 import b64encode +import os import logging import requests @@ -31,7 +32,7 @@ from ...networking import networkManager, NetworkRequestError from ...utils import CommandException, docker from ...node import NodeMode, NodeStatus -from ...configuration import UserConfiguration, NodeConfiguration, InvalidConfiguration, ConfigurationNotFound +from ...configuration import NodeConfiguration, InvalidConfiguration, ConfigurationNotFound class NodeException(Exception): @@ -62,22 +63,22 @@ def exists() -> bool: return docker.containerExists(config_defaults.DOCKER_CONTAINER_NAME) -def start(dockerImage: str, userConfig: UserConfiguration, nodeConfig: NodeConfiguration) -> None: +def start(dockerImage: str, nodeConfig: NodeConfiguration) -> None: try: ui.progressEcho("Starting Coretex Node...") docker.createNetwork(config_defaults.DOCKER_CONTAINER_NETWORK) environ = { - "CTX_API_URL": userConfig.serverUrl, + "CTX_API_URL": os.environ["CTX_API_URL"], "CTX_STORAGE_PATH": "/root/.coretex", - "CTX_NODE_ACCESS_TOKEN": nodeConfig.nodeAccessToken, - "CTX_NODE_MODE": str(nodeConfig.nodeMode) + "CTX_NODE_ACCESS_TOKEN": nodeConfig.accessToken, + "CTX_NODE_MODE": str(nodeConfig.mode) } if isinstance(nodeConfig.modelId, int): environ["CTX_MODEL_ID"] = str(nodeConfig.modelId) - nodeSecret = nodeConfig.nodeSecret if nodeConfig.nodeSecret is not None else config_defaults.DEFAULT_NODE_SECRET # change in configuration + nodeSecret = nodeConfig.secret if nodeConfig.secret is not None else config_defaults.DEFAULT_NODE_SECRET # change in configuration if isinstance(nodeSecret, str) and nodeSecret != config_defaults.DEFAULT_NODE_SECRET: environ["CTX_NODE_SECRET"] = nodeSecret @@ -96,9 +97,9 @@ def start(dockerImage: str, userConfig: UserConfiguration, nodeConfig: NodeConfi config_defaults.DOCKER_CONTAINER_NAME, dockerImage, nodeConfig.allowGpu, - nodeConfig.nodeRam, - nodeConfig.nodeSwap, - nodeConfig.nodeSharedMemory, + nodeConfig.ram, + nodeConfig.swap, + nodeConfig.sharedMemory, nodeConfig.cpuCount, environ, volumes @@ -129,11 +130,13 @@ def deactivateNode(id: Optional[int]) -> None: raise NetworkRequestError(response, "Failed to deactivate node.") -def stop(nodeId: int) -> None: +def stop(nodeId: Optional[int] = None) -> None: try: ui.progressEcho("Stopping Coretex Node...") docker.stopContainer(config_defaults.DOCKER_CONTAINER_NAME) - deactivateNode(nodeId) + + if nodeId is not None: + deactivateNode(nodeId) clean() ui.successEcho("Successfully stopped Coretex Node....") except BaseException as ex: @@ -372,7 +375,7 @@ def configureNode(advanced: bool) -> NodeConfiguration: cpuLimit, ramLimit = docker.getResourceLimits() swapLimit = docker.getDockerSwapLimit() - nodeConfig.nodeName = ui.clickPrompt("Node name", type = str) + nodeConfig.name = ui.clickPrompt("Node name", type = str) imageType = selectImageType() if imageType == ImageType.custom: @@ -390,13 +393,13 @@ def configureNode(advanced: bool) -> NodeConfiguration: nodeConfig.image += f":latest-{tag}" nodeConfig.storagePath = config_defaults.DEFAULT_STORAGE_PATH - nodeConfig.nodeRam = int(min(max(config_defaults.MINIMUM_RAM, ramLimit), config_defaults.DEFAULT_RAM)) - nodeConfig.nodeSwap = min(swapLimit, int(max(config_defaults.DEFAULT_SWAP_MEMORY, swapLimit))) - nodeConfig.nodeSharedMemory = config_defaults.DEFAULT_SHARED_MEMORY + nodeConfig.ram = int(min(max(config_defaults.MINIMUM_RAM, ramLimit), config_defaults.DEFAULT_RAM)) + nodeConfig.swap = min(swapLimit, int(max(config_defaults.DEFAULT_SWAP_MEMORY, swapLimit))) + nodeConfig.sharedMemory = config_defaults.DEFAULT_SHARED_MEMORY nodeConfig.cpuCount = int(min(cpuLimit, config_defaults.DEFAULT_CPU_COUNT)) - nodeConfig.nodeMode = config_defaults.DEFAULT_NODE_MODE + nodeConfig.mode = config_defaults.DEFAULT_NODE_MODE nodeConfig.allowDocker = config_defaults.DEFAULT_ALLOW_DOCKER - nodeConfig.nodeSecret = config_defaults.DEFAULT_NODE_SECRET + nodeConfig.secret = config_defaults.DEFAULT_NODE_SECRET nodeConfig.initScript = config_defaults.DEFAULT_INIT_SCRIPT publicKey: Optional[bytes] = None @@ -409,10 +412,10 @@ def configureNode(advanced: bool) -> NodeConfiguration: nodeConfig.storagePath = ui.clickPrompt("Storage path (press enter to use default)", config_defaults.DEFAULT_STORAGE_PATH, type = str) nodeConfig.cpuCount = promptCpu(cpuLimit) - nodeConfig.nodeRam = promptRam(ramLimit) - nodeConfig.nodeSwap = promptSwap(nodeConfig.nodeRam, swapLimit) + nodeConfig.ram = promptRam(ramLimit) + nodeConfig.swap = promptSwap(nodeConfig.ram, swapLimit) - nodeConfig.nodeSharedMemory = ui.clickPrompt( + nodeConfig.sharedMemory = ui.clickPrompt( "Node POSIX shared memory limit in GB (press enter to use default)", config_defaults.DEFAULT_SHARED_MEMORY, type = int @@ -432,7 +435,7 @@ def configureNode(advanced: bool) -> NodeConfiguration: type = str, hide_input = True ) - nodeConfig.nodeSecret = nodeSecret + nodeConfig.secret = nodeSecret if nodeSecret != config_defaults.DEFAULT_NODE_SECRET: ui.progressEcho("Generating RSA key-pair (2048 bits long) using provided node secret...") @@ -457,15 +460,15 @@ def configureNode(advanced: bool) -> NodeConfiguration: else: ui.stdEcho("To configure node manually run coretex node config with --verbose flag.") - nodeConfig.nodeId, nodeConfig.nodeAccessToken = registerNode(nodeConfig.nodeName, nodeMode, publicKey, nearWalletId, endpointInvocationPrice) - nodeConfig.nodeMode = nodeMode + nodeConfig.id, nodeConfig.accessToken = registerNode(nodeConfig.name, nodeMode, publicKey, nearWalletId, endpointInvocationPrice) + nodeConfig.mode = nodeMode return nodeConfig def initializeNodeConfiguration() -> None: try: - nodeConfig = NodeConfiguration.load() + NodeConfiguration.load() return except ConfigurationNotFound: ui.errorEcho("Node configuration not found.") @@ -484,7 +487,7 @@ def initializeNodeConfiguration() -> None: return nodeConfig = NodeConfiguration.load() - stop(nodeConfig.nodeId) + stop(nodeConfig.id) nodeConfig = configureNode(advanced = False) nodeConfig.save() diff --git a/coretex/cli/modules/project_utils.py b/coretex/cli/modules/project_utils.py index 91c5fef6..cbe86618 100644 --- a/coretex/cli/modules/project_utils.py +++ b/coretex/cli/modules/project_utils.py @@ -10,12 +10,6 @@ from ...networking import NetworkRequestError -def selectProject(projectId: int) -> None: - userConfiguration = UserConfiguration.load() - userConfiguration.projectId = projectId - userConfiguration.save() - - def selectProjectType() -> ProjectType: availableProjectTypes = { "Computer Vision": ProjectType.computerVision, @@ -62,21 +56,21 @@ def promptProjectCreate(message: str, name: str) -> Optional[Project]: raise click.ClickException(f"Failed to create project \"{name}\".") -def promptProjectSelect() -> Optional[Project]: +def promptProjectSelect(userConfig: UserConfiguration) -> Optional[Project]: name = ui.clickPrompt("Specify project name that you wish to select") ui.progressEcho("Validating project...") try: project = Project.fetchOne(name = name) ui.successEcho(f"Project \"{name}\" selected successfully!") - selectProject(project.id) + userConfig.selectProject(project.id) except ValueError: ui.errorEcho(f"Project \"{name}\" not found.") newProject = promptProjectCreate("Do you want to create a project with that name?", name) if newProject is None: return None - selectProject(project.id) + userConfig.selectProject(project.id) return project @@ -117,7 +111,7 @@ def getProject(name: Optional[str], userConfig: UserConfiguration) -> Optional[P if projectId is None: ui.stdEcho("To use this command you need to have a Project selected.") if click.confirm("Would you like to select an existing Project?", default = True): - return promptProjectSelect() + return promptProjectSelect(userConfig) if not click.confirm("Would you like to create a new Project?", default = True): return None @@ -125,16 +119,3 @@ def getProject(name: Optional[str], userConfig: UserConfiguration) -> Optional[P return createProject(name) return Project.fetchById(projectId) - - -def isProjectSelected() -> bool: - userConfig = UserConfiguration.load() - - if userConfig.projectId is None: - return False - - try: - Project.fetchById(userConfig.projectId) - return True - except NetworkRequestError: - return False diff --git a/coretex/cli/modules/ui.py b/coretex/cli/modules/ui.py index 5f824f60..e450d580 100644 --- a/coretex/cli/modules/ui.py +++ b/coretex/cli/modules/ui.py @@ -49,24 +49,23 @@ def arrowPrompt(choices: List[Any], message: str) -> Any: return answers["option"] -def previewConfig(userConfig: UserConfiguration, nodeConfig: NodeConfiguration) -> None: +def previewNodeConfig(nodeConfig: NodeConfiguration) -> None: allowDocker = "Yes" if nodeConfig.allowDocker else "No" - if nodeConfig.nodeSecret is None or nodeConfig.nodeSecret == "": + if nodeConfig.secret is None or nodeConfig.secret == "": nodeSecret = "" else: nodeSecret = "********" table = [ - ["Node name", nodeConfig.nodeName], - ["Server URL", userConfig.serverUrl], + ["Node name", nodeConfig.name], ["Coretex Node type", nodeConfig.image], ["Storage path", nodeConfig.storagePath], - ["RAM", f"{nodeConfig.nodeRam}GB"], - ["SWAP memory", f"{nodeConfig.nodeSwap}GB"], - ["POSIX shared memory", f"{nodeConfig.nodeSharedMemory}GB"], + ["RAM", f"{nodeConfig.ram}GB"], + ["SWAP memory", f"{nodeConfig.swap}GB"], + ["POSIX shared memory", f"{nodeConfig.sharedMemory}GB"], ["CPU cores allocated", f"{nodeConfig.cpuCount}"], - ["Coretex Node mode", f"{NodeMode(nodeConfig.nodeMode).name}"], + ["Coretex Node mode", f"{NodeMode(nodeConfig.mode).name}"], ["Docker access", allowDocker], ["Coretex Node secret", nodeSecret], ["Coretex Node init script", nodeConfig.initScript if nodeConfig.initScript is not None else ""] diff --git a/coretex/cli/modules/update.py b/coretex/cli/modules/update.py index 85931e98..d531dffd 100644 --- a/coretex/cli/modules/update.py +++ b/coretex/cli/modules/update.py @@ -18,8 +18,6 @@ from enum import IntEnum from pathlib import Path -import logging - import requests from .utils import getExecPath diff --git a/coretex/cli/modules/user.py b/coretex/cli/modules/user.py index cd69aace..825a80df 100644 --- a/coretex/cli/modules/user.py +++ b/coretex/cli/modules/user.py @@ -104,6 +104,7 @@ def initializeUserSession() -> None: if not ui.clickPrompt("Would you like to reconfigure the user? (Y/n)", type = bool, default = True, show_default = False): raise + userConfig = configUser() userConfig.save() diff --git a/coretex/configuration/__init__.py b/coretex/configuration/__init__.py index d46b9c9e..f83b0b42 100644 --- a/coretex/configuration/__init__.py +++ b/coretex/configuration/__init__.py @@ -30,9 +30,8 @@ def configMigration(configPath: Path) -> None: with configPath.open("r") as file: oldConfig = json.load(file) - logging.warning(f"[Coretex] >> WARNING: Old configuration:\n{oldConfig}") - userRaw = { + UserConfiguration({ "username": oldConfig.get("username"), "password": oldConfig.get("password"), "token": oldConfig.get("token"), @@ -41,30 +40,26 @@ def configMigration(configPath: Path) -> None: "refreshTokenExpirationDate": oldConfig.get("refreshTokenExpirationDate"), "serverUrl": oldConfig.get("serverUrl"), "projectId": oldConfig.get("projectId"), - } + }).save() - nodeRaw = { - "nodeName": oldConfig.get("nodeName"), - "nodeAccessToken": oldConfig.get("nodeAccessToken"), + NodeConfiguration({ + "name": oldConfig.get("nodeName"), + "accessToken": oldConfig.get("nodeAccessToken"), "storagePath": oldConfig.get("storagePath"), "image": oldConfig.get("image"), "allowGpu": oldConfig.get("allowGpu"), - "nodeRam": oldConfig.get("nodeRam"), - "nodeSharedMemory": oldConfig.get("nodeSharedMemory"), - "nodeSwap": oldConfig.get("nodeSwap"), + "ram": oldConfig.get("nodeRam"), + "sharedMemory": oldConfig.get("nodeSharedMemory"), + "swap": oldConfig.get("nodeSwap"), "cpuCount": oldConfig.get("cpuCount"), - "nodeMode": oldConfig.get("nodeMode"), + "mode": oldConfig.get("nodeMode"), "allowDocker": oldConfig.get("allowDocker"), - "nodeSecret": oldConfig.get("nodeSecret"), + "secret": oldConfig.get("nodeSecret"), "initScript": oldConfig.get("initScript"), "modelId": oldConfig.get("modelId"), - "nodeId": oldConfig.get("nodeId") - } + "id": oldConfig.get("nodeId") + }).save() - userConfig = UserConfiguration(userRaw) - nodeConfig = NodeConfiguration(nodeRaw) - userConfig.save() - nodeConfig.save() configPath.unlink() @@ -78,8 +73,7 @@ def _syncConfigWithEnv() -> None: oldConfigPath = CONFIG_DIR / "config.json" if oldConfigPath.exists(): logging.warning( - f"[Coretex] >> WARNING: Old configuration found at path: {oldConfigPath}. Migrating to new configuration." - f"\nFields with invalid values might be overrided in this process." + f">> [Coretex] Old configuration found at path: {oldConfigPath}. Migrating to new configuration." ) configMigration(oldConfigPath) @@ -87,22 +81,9 @@ def _syncConfigWithEnv() -> None: userConfig = UserConfiguration.load() if not "CTX_API_URL" in os.environ: os.environ["CTX_API_URL"] = userConfig.serverUrl - except (ConfigurationNotFound, InvalidConfiguration) as ex: - if not isCliRuntime(): - logging.error(f">> [Coretex] Error loading configuration. Reason: {ex}") - logging.info("\tIf this message came from Coretex Node you can safely ignore it.") + except (ConfigurationNotFound, InvalidConfiguration): + if not "CTX_API_URL" in os.environ: + os.environ["CTX_API_URL"] = "https://api.coretex.ai/" - try: - nodeConfig = NodeConfiguration.load() - - if isinstance(nodeConfig.nodeSecret, str) and nodeConfig.nodeSecret != "": - os.environ["CTX_SECRETS_KEY"] = nodeConfig.nodeSecret - - if not isCliRuntime(): - os.environ["CTX_STORAGE_PATH"] = nodeConfig.storagePath - else: - os.environ["CTX_STORAGE_PATH"] = f"{CONFIG_DIR}/data" - except (ConfigurationNotFound, InvalidConfiguration) as ex: - if not isCliRuntime(): - logging.error(f">> [Coretex] Error loading configuration. Reason: {ex}") - logging.info("\tIf this message came from Coretex Node you can safely ignore it.") + if not "CTX_STORAGE_PATH" in os.environ or isCliRuntime(): + os.environ["CTX_STORAGE_PATH"] = f"{CONFIG_DIR}/data" diff --git a/coretex/configuration/base.py b/coretex/configuration/base.py index 0ac56d68..dd4ed9a1 100644 --- a/coretex/configuration/base.py +++ b/coretex/configuration/base.py @@ -32,8 +32,9 @@ class InvalidConfiguration(Exception): - def __init__(self, message: str,errors: List[str]) -> None: + def __init__(self, message: str, errors: List[str]) -> None: super().__init__(message) + self.errors = errors diff --git a/coretex/configuration/node.py b/coretex/configuration/node.py index 9e66dd89..e5befed6 100644 --- a/coretex/configuration/node.py +++ b/coretex/configuration/node.py @@ -15,53 +15,42 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Dict, Any, List, Optional, Tuple +from typing import List, Optional, Tuple from pathlib import Path -import os - from . import config_defaults from .base import BaseConfiguration, CONFIG_DIR +from . import utils from ..utils import docker from ..node import NodeMode from ..networking import networkManager, NetworkRequestError -def getEnvVar(key: str, default: str) -> str: - if os.environ.get(key) is None: - return default - - return os.environ[key] - - class DockerConfigurationException(Exception): pass class NodeConfiguration(BaseConfiguration): - def __init__(self, raw: Dict[str, Any]) -> None: - super().__init__(raw) - @classmethod def getConfigPath(cls) -> Path: return CONFIG_DIR / "node_config.json" @property - def nodeName(self) -> str: - return self.getValue("nodeName", str, "CTX_NODE_NAME") + def name(self) -> str: + return self.getValue("name", str, "CTX_NODE_NAME") - @nodeName.setter - def nodeName(self, value: str) -> None: - self._raw["nodeName"] = value + @name.setter + def name(self, value: str) -> None: + self._raw["name"] = value @property - def nodeAccessToken(self) -> str: - return self.getValue("nodeAccessToken", str) + def accessToken(self) -> str: + return self.getValue("accessToken", str) - @nodeAccessToken.setter - def nodeAccessToken(self, value: str) -> None: - self._raw["nodeAccessToken"] = value + @accessToken.setter + def accessToken(self, value: str) -> None: + self._raw["accessToken"] = value @property def storagePath(self) -> str: @@ -88,43 +77,43 @@ def allowGpu(self, value: bool) -> None: self._raw["allowGpu"] = value @property - def nodeRam(self) -> int: - nodeRam = self.getOptValue("nodeRam", int) + def ram(self) -> int: + ram = self.getOptValue("ram", int) - if nodeRam is None: - nodeRam = config_defaults.DEFAULT_RAM + if ram is None: + ram = config_defaults.DEFAULT_RAM - return nodeRam + return ram - @nodeRam.setter - def nodeRam(self, value: int) -> None: - self._raw["nodeRam"] = value + @ram.setter + def ram(self, value: int) -> None: + self._raw["ram"] = value @property - def nodeSwap(self) -> int: - nodeSwap = self.getOptValue("nodeSwap", int) + def swap(self) -> int: + swap = self.getOptValue("swap", int) - if nodeSwap is None: - nodeSwap = config_defaults.DEFAULT_SWAP_MEMORY + if swap is None: + swap = config_defaults.DEFAULT_SWAP_MEMORY - return nodeSwap + return swap - @nodeSwap.setter - def nodeSwap(self, value: int) -> None: - self._raw["nodeSwap"] = value + @swap.setter + def swap(self, value: int) -> None: + self._raw["swap"] = value @property - def nodeSharedMemory(self) -> int: - nodeSharedMemory = self.getOptValue("nodeSharedMemory", int) + def sharedMemory(self) -> int: + sharedMemory = self.getOptValue("sharedMemory", int) - if nodeSharedMemory is None: - nodeSharedMemory = config_defaults.DEFAULT_SHARED_MEMORY + if sharedMemory is None: + sharedMemory = config_defaults.DEFAULT_SHARED_MEMORY - return nodeSharedMemory + return sharedMemory - @nodeSharedMemory.setter - def nodeSharedMemory(self, value: int) -> None: - self._raw["nodeSharedMemory"] = value + @sharedMemory.setter + def sharedMemory(self, value: int) -> None: + self._raw["sharedMemory"] = value @property def cpuCount(self) -> int: @@ -140,30 +129,30 @@ def cpuCount(self, value: int) -> None: self._raw["cpuCount"] = value @property - def nodeMode(self) -> int: - nodeMode = self.getOptValue("nodeMode", int) + def mode(self) -> int: + mode = self.getOptValue("mode", int) - if nodeMode is None: - nodeMode = NodeMode.any + if mode is None: + mode = NodeMode.any - return nodeMode + return mode - @nodeMode.setter - def nodeMode(self, value: int) -> None: - self._raw["nodeMode"] = value + @mode.setter + def mode(self, value: int) -> None: + self._raw["mode"] = value @property - def nodeId(self) -> int: - nodeId = self.getOptValue("nodeId", int) + def id(self) -> int: + id = self.getOptValue("id", int) - if nodeId is None: - nodeId = self.fetchNodeId() + if id is None: + id = self.fetchNodeId() - return nodeId + return id - @nodeId.setter - def nodeId(self, value: int) -> None: - self._raw["nodeId"] = value + @id.setter + def id(self, value: int) -> None: + self._raw["id"] = value @property def allowDocker(self) -> bool: @@ -174,12 +163,12 @@ def allowDocker(self, value: bool) -> None: self._raw["allowDocker"] = value @property - def nodeSecret(self) -> Optional[str]: - return self.getOptValue("nodeSecret", str) + def secret(self) -> Optional[str]: + return self.getOptValue("secret", str) - @nodeSecret.setter - def nodeSecret(self, value: Optional[str]) -> None: - self._raw["nodeSecret"] = value + @secret.setter + def secret(self, value: Optional[str]) -> None: + self._raw["secret"] = value @property def initScript(self) -> Optional[str]: @@ -213,121 +202,41 @@ def endpointInvocationPrice(self) -> Optional[float]: def endpointInvocationPrice(self, value: Optional[float]) -> None: self._raw["endpointInvocationPrice"] = value - def validateRamField(self, ramLimit: int) -> Tuple[bool, int, str]: - isValid = True - message = "" - - if ramLimit < config_defaults.MINIMUM_RAM: - isValid = False - raise DockerConfigurationException( - f"Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM}GB) " - f"is higher than your current Docker desktop RAM limit ({ramLimit}GB). " - "Please adjust resource limitations in Docker Desktop settings to match Node requirements." - ) - - defaultRamValue = int(min(max(config_defaults.MINIMUM_RAM, ramLimit), config_defaults.DEFAULT_RAM)) - - if not isinstance(self._raw.get("nodeRam"), int): - isValid = False - message = ( - f"Invalid config \"nodeRam\" field type \"{type(self._raw.get('nodeRam'))}\". Expected: \"int\"" - f"Using default value of {defaultRamValue} GB" - ) - - if self.nodeRam < config_defaults.MINIMUM_RAM: - isValid = False - message = ( - f"WARNING: Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM}GB) " - f"is higher than the configured value ({self._raw.get('nodeRam')}GB)" - f"Overriding \"nodeRam\" field to match node RAM requirements." - ) - - if self.nodeRam > ramLimit: - isValid = False - message = ( - f"WARNING: RAM limit in Docker Desktop ({ramLimit}GB) " - f"is lower than the configured value ({self._raw.get('nodeRam')}GB)" - f"Overriding \"nodeRam\" field to limit in Docker Desktop." - ) - - return isValid, defaultRamValue, message - - def validateCPUCount(self, cpuLimit: int) -> Tuple[bool, int, str]: - isValid = True - message = "" - defaultCPUCount = config_defaults.DEFAULT_CPU_COUNT if config_defaults.DEFAULT_CPU_COUNT <= cpuLimit else cpuLimit - - if not isinstance(self._raw.get("cpuCount"), int): - isValid = False - message = ( - f"Invalid config \"cpuCount\" field type \"{type(self._raw.get('cpuCount'))}\". Expected: \"int\"" - f"Using default value of {defaultCPUCount} cores" - ) - - if self.cpuCount > cpuLimit: - isValid = False - message = ( - f"WARNING: CPU limit in Docker Desktop ({cpuLimit}) " - f"is lower than the configured value ({self._raw.get('cpuCount')})" - f"Overriding \"cpuCount\" field to limit in Docker Desktop." - ) - - return isValid, cpuLimit, message - - def validateSWAPMemory(self, swapLimit: int) -> Tuple[bool, int, str]: - isValid = True - message = "" - defaultSWAPMemory = config_defaults.DEFAULT_SWAP_MEMORY if config_defaults.DEFAULT_SWAP_MEMORY <= swapLimit else swapLimit - - if not isinstance(self._raw.get("nodeSwap"), int): - isValid = False - message = ( - f"Invalid config \"nodeSwap\" field type \"{type(self._raw.get('nodeSwap'))}\". Expected: \"int\"" - f"Using default value of {defaultSWAPMemory} GB" - ) - - if self.nodeSwap > swapLimit: - isValid = False - message = ( - f"WARNING: SWAP limit in Docker Desktop ({swapLimit}GB) " - f"is lower than the configured value ({self.nodeSwap}GB)" - f"Overriding \"nodeSwap\" field to limit in Docker Desktop." - ) - - return isValid, defaultSWAPMemory, message - def _isConfigValid(self) -> Tuple[bool, List[str]]: isValid = True errorMessages = [] cpuLimit, ramLimit = docker.getResourceLimits() swapLimit = docker.getDockerSwapLimit() - if not isinstance(self._raw.get("nodeName"), str): + if not isinstance(self._raw.get("name"), str): isValid = False - errorMessages.append("Invalid configuration. Missing required field \"nodeName\".") + errorMessages.append("Invalid configuration. Missing required field \"name\".") if not isinstance(self._raw.get("image"), str): isValid = False errorMessages.append("Invalid configuration. Missing required field \"image\".") - if not isinstance(self._raw.get("nodeAccessToken"), str): + if not isinstance(self._raw.get("accessToken"), str): isValid = False - errorMessages.append("Invalid configuration. Missing required field \"nodeAccessToken\".") + errorMessages.append("Invalid configuration. Missing required field \"accessToken\".") - isRamValid, nodeRam, message = self.validateRamField(ramLimit) - if not isRamValid: + validateRamField = utils.validateRamField(self, ramLimit) + if isinstance(validateRamField, tuple): + ram, message = validateRamField errorMessages.append(message) - self.nodeRam = nodeRam + self.ram = ram - isCPUCountValid, cpuCount, message = self.validateCPUCount(cpuLimit) - if not isCPUCountValid: + validateCpuCount = utils.validateCpuCount(self, cpuLimit) + if isinstance(validateCpuCount, tuple): + cpuCount, message = validateCpuCount errorMessages.append(message) self.cpuCount = cpuCount - isSWAPMemoryValid, nodeSwap, message = self.validateSWAPMemory(swapLimit) - if not isSWAPMemoryValid: + validateSwapMemory = utils.validateSwapMemory(self, swapLimit) + if isinstance(validateSwapMemory, tuple): + swap, message = validateSwapMemory errorMessages.append(message) - self.nodeSwap = nodeSwap + self.swap = swap return isValid, errorMessages @@ -348,7 +257,7 @@ def getInitScriptPath(self) -> Optional[Path]: def fetchNodeId(self) -> int: params = { - "machine_name": f"={self.nodeName}" + "machine_name": f"={self.name}" } response = networkManager.get("service", params) @@ -362,15 +271,16 @@ def fetchNodeId(self) -> int: raise TypeError(f"Invalid \"data\" type {type(data)}. Expected: \"list\"") if len(data) == 0: - raise ValueError(f"Node with name \"{self.nodeName}\" not found.") + raise ValueError(f"Node with name \"{self.name}\" not found.") nodeJson = data[0] if not isinstance(nodeJson, dict): raise TypeError(f"Invalid \"nodeJson\" type {type(nodeJson)}. Expected: \"dict\"") - nodeId = nodeJson.get("id") - if not isinstance(nodeId, str): - raise TypeError(f"Invalid \"nodeId\" type {type(nodeId)}. Expected: \"str\"") + id = nodeJson.get("id") + if not isinstance(id, str): + raise TypeError(f"Invalid \"id\" type {type(id)}. Expected: \"str\"") - self.nodeId = int(nodeId) + self.id = int(id) self.save() - return int(nodeId) \ No newline at end of file + + return int(id) diff --git a/coretex/configuration/user.py b/coretex/configuration/user.py index 74bfc17d..553003a1 100644 --- a/coretex/configuration/user.py +++ b/coretex/configuration/user.py @@ -16,21 +16,17 @@ # along with this program. If not, see . from pathlib import Path -from typing import Dict, Any, List, Optional, Tuple +from typing import List, Optional, Tuple from datetime import datetime, timezone from .base import BaseConfiguration, CONFIG_DIR from ..utils import decodeDate - - -USER_CONFIG_PATH = CONFIG_DIR / "user_config.json" +from ..entities import Project +from ..networking import NetworkRequestError class UserConfiguration(BaseConfiguration): - def __init__(self, raw: Dict[str, Any]) -> None: - super().__init__(raw) - @classmethod def getConfigPath(cls) -> Path: return CONFIG_DIR / "user_config.json" @@ -126,3 +122,18 @@ def isTokenValid(self, tokenName: str) -> bool: return datetime.now(timezone.utc) > decodeDate(tokenExpirationDate) except ValueError: return False + + @property + def isProjectSelected(self) -> bool: + if self.projectId is None: + return False + + try: + Project.fetchById(self.projectId) + return True + except NetworkRequestError: + return False + + def selectProject(self, id: int) -> None: + self.projectId = id + self.save() diff --git a/coretex/configuration/utils.py b/coretex/configuration/utils.py new file mode 100644 index 00000000..b220e41d --- /dev/null +++ b/coretex/configuration/utils.py @@ -0,0 +1,104 @@ +# Copyright (C) 2023 Coretex LLC + +# This file is part of Coretex.ai + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from typing import Optional, Tuple + +from . import config_defaults +from .node import NodeConfiguration, DockerConfigurationException + + +def validateRamField(nodeConfig: NodeConfiguration, ramLimit: int) -> Optional[Tuple[int, str]]: + if ramLimit < config_defaults.MINIMUM_RAM: + raise DockerConfigurationException( + f"Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM}GB) " + f"is higher than your current Docker desktop RAM limit ({ramLimit}GB). " + "Please adjust resource limitations in Docker Desktop settings to match Node requirements." + ) + + defaultRamValue = int(min(max(config_defaults.MINIMUM_RAM, ramLimit), config_defaults.DEFAULT_RAM)) + + if not isinstance(nodeConfig._raw.get("ram"), int): + message = ( + f"Invalid config \"ram\" field type \"{type(nodeConfig._raw.get('ram'))}\". Expected: \"int\"" + f"Using default value of {defaultRamValue} GB" + ) + + return defaultRamValue, message + + if nodeConfig.ram < config_defaults.MINIMUM_RAM: + message = ( + f"WARNING: Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM}GB) " + f"is higher than the configured value ({nodeConfig._raw.get('ram')}GB)" + f"Overriding \"ram\" field to match node RAM requirements." + ) + + return defaultRamValue, message + + if nodeConfig.ram > ramLimit: + message = ( + f"WARNING: RAM limit in Docker Desktop ({ramLimit}GB) " + f"is lower than the configured value ({nodeConfig._raw.get('ram')}GB)" + f"Overriding \"ram\" field to limit in Docker Desktop." + ) + + return defaultRamValue, message + + return None + +def validateCpuCount(nodeConfig: NodeConfiguration, cpuLimit: int) -> Optional[Tuple[int, str]]: + defaultCpuCount = config_defaults.DEFAULT_CPU_COUNT if config_defaults.DEFAULT_CPU_COUNT <= cpuLimit else cpuLimit + + if not isinstance(nodeConfig._raw.get("cpuCount"), int): + message = ( + f"Invalid config \"cpuCount\" field type \"{type(nodeConfig._raw.get('cpuCount'))}\". Expected: \"int\"" + f"Using default value of {defaultCpuCount} cores" + ) + + return defaultCpuCount, message + + if nodeConfig.cpuCount > cpuLimit: + message = ( + f"WARNING: CPU limit in Docker Desktop ({cpuLimit}) " + f"is lower than the configured value ({nodeConfig._raw.get('cpuCount')})" + f"Overriding \"cpuCount\" field to limit in Docker Desktop." + ) + + return defaultCpuCount, message + + return None + +def validateSwapMemory(nodeConfig: NodeConfiguration, swapLimit: int) -> Optional[Tuple[int, str]]: + defaultSwapMemory = config_defaults.DEFAULT_SWAP_MEMORY if config_defaults.DEFAULT_SWAP_MEMORY <= swapLimit else swapLimit + + if not isinstance(nodeConfig._raw.get("swap"), int): + message = ( + f"Invalid config \"swap\" field type \"{type(nodeConfig._raw.get('swap'))}\". Expected: \"int\"" + f"Using default value of {defaultSwapMemory} GB" + ) + + return defaultSwapMemory, message + + if nodeConfig.swap > swapLimit: + message = ( + f"WARNING: SWAP limit in Docker Desktop ({swapLimit}GB) " + f"is lower than the configured value ({nodeConfig.swap}GB)" + f"Overriding \"swap\" field to limit in Docker Desktop." + ) + + return defaultSwapMemory, message + + return None \ No newline at end of file diff --git a/coretex/utils/__init__.py b/coretex/utils/__init__.py index 7de780df..8d9a3af6 100644 --- a/coretex/utils/__init__.py +++ b/coretex/utils/__init__.py @@ -23,4 +23,3 @@ from .process import logProcessOutput, command, CommandException from .logs import createFileHandler from .misc import isCliRuntime -from .inference import runOnnxInference diff --git a/coretex/zkml/__init__.py b/coretex/zkml/__init__.py new file mode 100644 index 00000000..4872ce17 --- /dev/null +++ b/coretex/zkml/__init__.py @@ -0,0 +1,16 @@ +# Copyright (C) 2023 Coretex LLC + +# This file is part of Coretex.ai + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . diff --git a/coretex/utils/inference.py b/coretex/zkml/inference.py similarity index 77% rename from coretex/utils/inference.py rename to coretex/zkml/inference.py index 7fb34acc..29af7e49 100644 --- a/coretex/utils/inference.py +++ b/coretex/zkml/inference.py @@ -1,4 +1,21 @@ -from typing import Tuple, Optional, Union, Any +# Copyright (C) 2023 Coretex LLC + +# This file is part of Coretex.ai + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from typing import Tuple, Optional, Union from pathlib import Path import json @@ -16,6 +33,7 @@ async def genWitness(inputPath: Path, circuit: Path, witnessPath: Path) -> None: await ezkl.gen_witness(inputPath, circuit, witnessPath) + async def getSrs(settings: Path) -> None: await ezkl.get_srs(settings) From a761b912491ff584d0c20474710326a5f05fbcc3 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Fri, 9 Aug 2024 14:09:00 +0200 Subject: [PATCH 25/28] CTX-5430: Changes regarding pr discussions. --- coretex/cli/commands/node.py | 1 - coretex/cli/modules/config_defaults.py | 22 ------------ coretex/cli/modules/node.py | 45 +++++++++++------------- coretex/configuration/config_defaults.py | 25 +++---------- coretex/configuration/utils.py | 2 +- 5 files changed, 26 insertions(+), 69 deletions(-) delete mode 100644 coretex/cli/modules/config_defaults.py diff --git a/coretex/cli/commands/node.py b/coretex/cli/commands/node.py index 06809fbe..9c0d55cd 100644 --- a/coretex/cli/commands/node.py +++ b/coretex/cli/commands/node.py @@ -159,7 +159,6 @@ def config(verbose: bool) -> None: except (ConfigurationNotFound, InvalidConfiguration): node_module.stop() - try: nodeConfig = NodeConfiguration.load() if not ui.clickPrompt( diff --git a/coretex/cli/modules/config_defaults.py b/coretex/cli/modules/config_defaults.py deleted file mode 100644 index 7c59b2f3..00000000 --- a/coretex/cli/modules/config_defaults.py +++ /dev/null @@ -1,22 +0,0 @@ -from pathlib import Path - -import multiprocessing - -from ...node import NodeMode -from ...statistics import getAvailableRam, getAvailableCpuCount - - -DOCKER_CONTAINER_NAME = "coretex_node" -DOCKER_CONTAINER_NETWORK = "coretex_node" -DEFAULT_STORAGE_PATH = str(Path.home() / ".coretex") -DEFAULT_RAM = getAvailableRam() -MINIMUM_RAM = 6 -DEFAULT_SWAP_MEMORY = DEFAULT_RAM * 2 -DEFAULT_SHARED_MEMORY = 2 -DEFAULT_CPU_COUNT = multiprocessing.cpu_count() -DEFAULT_NODE_MODE = NodeMode.execution -DEFAULT_ALLOW_DOCKER = False -DEFAULT_NODE_SECRET = "" -DEFAULT_INIT_SCRIPT = "" -DEFAULT_NEAR_WALLET_ID = "" -DEFAULT_ENDPOINT_INVOCATION_PRICE = 0.0 diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index a09651aa..06d6127f 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -26,13 +26,13 @@ import click -from . import ui, config_defaults +from . import ui from .utils import isGPUAvailable from ...cryptography import rsa from ...networking import networkManager, NetworkRequestError from ...utils import CommandException, docker from ...node import NodeMode, NodeStatus -from ...configuration import NodeConfiguration, InvalidConfiguration, ConfigurationNotFound +from ...configuration import config_defaults, NodeConfiguration, InvalidConfiguration, ConfigurationNotFound class NodeException(Exception): @@ -75,7 +75,7 @@ def start(dockerImage: str, nodeConfig: NodeConfiguration) -> None: "CTX_NODE_MODE": str(nodeConfig.mode) } - if isinstance(nodeConfig.modelId, int): + if nodeConfig.modelId is not None: environ["CTX_MODEL_ID"] = str(nodeConfig.modelId) nodeSecret = nodeConfig.secret if nodeConfig.secret is not None else config_defaults.DEFAULT_NODE_SECRET # change in configuration @@ -137,6 +137,7 @@ def stop(nodeId: Optional[int] = None) -> None: if nodeId is not None: deactivateNode(nodeId) + clean() ui.successEcho("Successfully stopped Coretex Node....") except BaseException as ex: @@ -146,7 +147,7 @@ def stop(nodeId: Optional[int] = None) -> None: def getNodeStatus() -> NodeStatus: try: - response = requests.get(f"http://localhost:21000/status", timeout = 1) + response = requests.get("http://localhost:21000/status", timeout = 1) status = response.json()["status"] return NodeStatus(status) except: @@ -230,7 +231,6 @@ def registerNode( if not isinstance(accessToken, str) or not isinstance(nodeId, int): raise TypeError("Something went wrong. Please try again...") - print(nodeId) return nodeId, accessToken @@ -402,11 +402,6 @@ def configureNode(advanced: bool) -> NodeConfiguration: nodeConfig.secret = config_defaults.DEFAULT_NODE_SECRET nodeConfig.initScript = config_defaults.DEFAULT_INIT_SCRIPT - publicKey: Optional[bytes] = None - nearWalletId: Optional[str] = None - endpointInvocationPrice: Optional[float] = None - nodeMode = NodeMode.any - if advanced: nodeMode = selectNodeMode() nodeConfig.storagePath = ui.clickPrompt("Storage path (press enter to use default)", config_defaults.DEFAULT_STORAGE_PATH, type = str) @@ -437,30 +432,32 @@ def configureNode(advanced: bool) -> NodeConfiguration: ) nodeConfig.secret = nodeSecret - if nodeSecret != config_defaults.DEFAULT_NODE_SECRET: - ui.progressEcho("Generating RSA key-pair (2048 bits long) using provided node secret...") - rsaKey = rsa.generateKey(2048, nodeSecret.encode("utf-8")) - publicKey = rsa.getPublicKeyBytes(rsaKey.public_key()) + if nodeMode in [NodeMode.endpointReserved, NodeMode.endpointShared]: - nearWalletId = ui.clickPrompt( + nodeConfig.nearWalletId = ui.clickPrompt( "Enter a NEAR wallet id to which the funds will be transfered when executing endpoints", config_defaults.DEFAULT_NEAR_WALLET_ID, type = str ) - if nearWalletId != config_defaults.DEFAULT_NEAR_WALLET_ID: - nodeConfig.nearWalletId = nearWalletId - endpointInvocationPrice = promptInvocationPrice() - else: - nodeConfig.nearWalletId = None - nodeConfig.endpointInvocationPrice = None - nearWalletId = None - endpointInvocationPrice = None + nodeConfig.endpointInvocationPrice = promptInvocationPrice() else: ui.stdEcho("To configure node manually run coretex node config with --verbose flag.") - nodeConfig.id, nodeConfig.accessToken = registerNode(nodeConfig.name, nodeMode, publicKey, nearWalletId, endpointInvocationPrice) + publicKey: Optional[bytes] = None + if nodeSecret != config_defaults.DEFAULT_NODE_SECRET: + ui.progressEcho("Generating RSA key-pair (2048 bits long) using provided node secret...") + rsaKey = rsa.generateKey(2048, nodeSecret.encode("utf-8")) + publicKey = rsa.getPublicKeyBytes(rsaKey.public_key()) + + nodeConfig.id, nodeConfig.accessToken = registerNode( + nodeConfig.name, + nodeMode, + publicKey, + nodeConfig.nearWalletId, + nodeConfig.endpointInvocationPrice + ) nodeConfig.mode = nodeMode return nodeConfig diff --git a/coretex/configuration/config_defaults.py b/coretex/configuration/config_defaults.py index 57feaffc..735e9b43 100644 --- a/coretex/configuration/config_defaults.py +++ b/coretex/configuration/config_defaults.py @@ -1,30 +1,11 @@ -# Copyright (C) 2023 Coretex LLC - -# This file is part of Coretex.ai - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - from pathlib import Path -import os +import multiprocessing from ..node import NodeMode from ..statistics import getAvailableRam -cpuCount = os.cpu_count() - DOCKER_CONTAINER_NAME = "coretex_node" DOCKER_CONTAINER_NETWORK = "coretex_node" DEFAULT_STORAGE_PATH = str(Path.home() / ".coretex") @@ -32,8 +13,10 @@ MINIMUM_RAM = 6 DEFAULT_SWAP_MEMORY = DEFAULT_RAM * 2 DEFAULT_SHARED_MEMORY = 2 -DEFAULT_CPU_COUNT = cpuCount if cpuCount is not None else 0 +DEFAULT_CPU_COUNT = multiprocessing.cpu_count() DEFAULT_NODE_MODE = NodeMode.execution DEFAULT_ALLOW_DOCKER = False DEFAULT_NODE_SECRET = "" DEFAULT_INIT_SCRIPT = "" +DEFAULT_NEAR_WALLET_ID = "" +DEFAULT_ENDPOINT_INVOCATION_PRICE = 0.0 diff --git a/coretex/configuration/utils.py b/coretex/configuration/utils.py index b220e41d..786e044b 100644 --- a/coretex/configuration/utils.py +++ b/coretex/configuration/utils.py @@ -101,4 +101,4 @@ def validateSwapMemory(nodeConfig: NodeConfiguration, swapLimit: int) -> Optiona return defaultSwapMemory, message - return None \ No newline at end of file + return None From 702baa05e039f0fe6adc7c295144943d33389d0a Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Fri, 9 Aug 2024 14:47:40 +0200 Subject: [PATCH 26/28] CTX-5430: Code and pipeline fix, imports issue solved. --- coretex/cli/modules/node.py | 38 ---------- coretex/configuration/node.py | 4 -- coretex/configuration/user.py | 13 ---- coretex/configuration/utils.py | 68 +++++++++++++----- coretex/utils/docker.py | 4 ++ tests/resources/computer_vision_dataset/1.zip | Bin 30505 -> 30505 bytes .../computer_vision_dataset/classes.json | 2 +- 7 files changed, 54 insertions(+), 75 deletions(-) diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index 991bf4c0..0bb4bf23 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -120,43 +120,7 @@ def clean() -> None: raise NodeException("Failed to clean inactive Coretex Node.") -<<<<<<< HEAD def deactivateNode(id: Optional[int]) -> None: -======= -def fetchNodeId(name: str) -> int: - config = loadConfig() - params = { - "name": f"={name}" - } - - response = networkManager.get("service/directory", params) - if response.hasFailed(): - raise NetworkRequestError(response, "Failed to fetch node id.") - - responseJson = response.getJson(dict) - data = responseJson.get("data") - - if not isinstance(data, list): - raise TypeError(f"Invalid \"data\" type {type(data)}. Expected: \"list\"") - - if len(data) == 0: - raise ValueError(f"Node with name \"{name}\" not found.") - - nodeJson = data[0] - if not isinstance(nodeJson, dict): - raise TypeError(f"Invalid \"nodeJson\" type {type(nodeJson)}. Expected: \"dict\"") - - nodeId = nodeJson.get("id") - if not isinstance(nodeId, int): - raise TypeError(f"Invalid \"nodeId\" type {type(nodeId)}. Expected: \"int\"") - - config["nodeId"] = nodeId - saveConfig(config) - return nodeId - - -def deactivateNode(id: int) -> None: ->>>>>>> develop params = { "id": id } @@ -468,8 +432,6 @@ def configureNode(advanced: bool) -> NodeConfiguration: ) nodeConfig.secret = nodeSecret - - if nodeMode in [NodeMode.endpointReserved, NodeMode.endpointShared]: nodeConfig.nearWalletId = ui.clickPrompt( "Enter a NEAR wallet id to which the funds will be transfered when executing endpoints", diff --git a/coretex/configuration/node.py b/coretex/configuration/node.py index 085505df..783edd8f 100644 --- a/coretex/configuration/node.py +++ b/coretex/configuration/node.py @@ -26,10 +26,6 @@ from ..networking import networkManager, NetworkRequestError -class DockerConfigurationException(Exception): - pass - - class NodeConfiguration(BaseConfiguration): @classmethod diff --git a/coretex/configuration/user.py b/coretex/configuration/user.py index 553003a1..b7a650dc 100644 --- a/coretex/configuration/user.py +++ b/coretex/configuration/user.py @@ -21,8 +21,6 @@ from .base import BaseConfiguration, CONFIG_DIR from ..utils import decodeDate -from ..entities import Project -from ..networking import NetworkRequestError class UserConfiguration(BaseConfiguration): @@ -123,17 +121,6 @@ def isTokenValid(self, tokenName: str) -> bool: except ValueError: return False - @property - def isProjectSelected(self) -> bool: - if self.projectId is None: - return False - - try: - Project.fetchById(self.projectId) - return True - except NetworkRequestError: - return False - def selectProject(self, id: int) -> None: self.projectId = id self.save() diff --git a/coretex/configuration/utils.py b/coretex/configuration/utils.py index 786e044b..2f090a23 100644 --- a/coretex/configuration/utils.py +++ b/coretex/configuration/utils.py @@ -15,13 +15,14 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Optional, Tuple +from typing import Optional, Tuple, Any from . import config_defaults -from .node import NodeConfiguration, DockerConfigurationException +from ..networking import networkManager, NetworkRequestError +from ..utils.docker import DockerConfigurationException -def validateRamField(nodeConfig: NodeConfiguration, ramLimit: int) -> Optional[Tuple[int, str]]: +def validateRamField(ram: Optional[Any], ramLimit: int) -> Optional[Tuple[int, str]]: if ramLimit < config_defaults.MINIMUM_RAM: raise DockerConfigurationException( f"Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM}GB) " @@ -31,27 +32,27 @@ def validateRamField(nodeConfig: NodeConfiguration, ramLimit: int) -> Optional[T defaultRamValue = int(min(max(config_defaults.MINIMUM_RAM, ramLimit), config_defaults.DEFAULT_RAM)) - if not isinstance(nodeConfig._raw.get("ram"), int): + if not isinstance(ram, int): message = ( - f"Invalid config \"ram\" field type \"{type(nodeConfig._raw.get('ram'))}\". Expected: \"int\"" + f"Invalid config \"ram\" field type \"{type(ram)}\". Expected: \"int\"" f"Using default value of {defaultRamValue} GB" ) return defaultRamValue, message - if nodeConfig.ram < config_defaults.MINIMUM_RAM: + if ram < config_defaults.MINIMUM_RAM: message = ( f"WARNING: Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM}GB) " - f"is higher than the configured value ({nodeConfig._raw.get('ram')}GB)" + f"is higher than the configured value ({ram}GB)" f"Overriding \"ram\" field to match node RAM requirements." ) return defaultRamValue, message - if nodeConfig.ram > ramLimit: + if ram > ramLimit: message = ( f"WARNING: RAM limit in Docker Desktop ({ramLimit}GB) " - f"is lower than the configured value ({nodeConfig._raw.get('ram')}GB)" + f"is lower than the configured value ({ram}GB)" f"Overriding \"ram\" field to limit in Docker Desktop." ) @@ -59,21 +60,21 @@ def validateRamField(nodeConfig: NodeConfiguration, ramLimit: int) -> Optional[T return None -def validateCpuCount(nodeConfig: NodeConfiguration, cpuLimit: int) -> Optional[Tuple[int, str]]: +def validateCpuCount(cpuCount: Optional[Any], cpuLimit: int) -> Optional[Tuple[int, str]]: defaultCpuCount = config_defaults.DEFAULT_CPU_COUNT if config_defaults.DEFAULT_CPU_COUNT <= cpuLimit else cpuLimit - if not isinstance(nodeConfig._raw.get("cpuCount"), int): + if not isinstance(cpuCount, int): message = ( - f"Invalid config \"cpuCount\" field type \"{type(nodeConfig._raw.get('cpuCount'))}\". Expected: \"int\"" + f"Invalid config \"cpuCount\" field type \"{type(cpuCount)}\". Expected: \"int\"" f"Using default value of {defaultCpuCount} cores" ) return defaultCpuCount, message - if nodeConfig.cpuCount > cpuLimit: + if cpuCount > cpuLimit: message = ( f"WARNING: CPU limit in Docker Desktop ({cpuLimit}) " - f"is lower than the configured value ({nodeConfig._raw.get('cpuCount')})" + f"is lower than the configured value ({cpuCount})" f"Overriding \"cpuCount\" field to limit in Docker Desktop." ) @@ -81,24 +82,53 @@ def validateCpuCount(nodeConfig: NodeConfiguration, cpuLimit: int) -> Optional[T return None -def validateSwapMemory(nodeConfig: NodeConfiguration, swapLimit: int) -> Optional[Tuple[int, str]]: +def validateSwapMemory(swap: Optional[Any], swapLimit: int) -> Optional[Tuple[int, str]]: defaultSwapMemory = config_defaults.DEFAULT_SWAP_MEMORY if config_defaults.DEFAULT_SWAP_MEMORY <= swapLimit else swapLimit - if not isinstance(nodeConfig._raw.get("swap"), int): + if not isinstance(swap, int): message = ( - f"Invalid config \"swap\" field type \"{type(nodeConfig._raw.get('swap'))}\". Expected: \"int\"" + f"Invalid config \"swap\" field type \"{type(swap)}\". Expected: \"int\"" f"Using default value of {defaultSwapMemory} GB" ) return defaultSwapMemory, message - if nodeConfig.swap > swapLimit: + if swap > swapLimit: message = ( f"WARNING: SWAP limit in Docker Desktop ({swapLimit}GB) " - f"is lower than the configured value ({nodeConfig.swap}GB)" + f"is lower than the configured value ({swap}GB)" f"Overriding \"swap\" field to limit in Docker Desktop." ) return defaultSwapMemory, message return None + + +def fetchNodeId(name: str) -> int: + params = { + "name": f"={name}" + } + + response = networkManager.get("service/directory", params) + if response.hasFailed(): + raise NetworkRequestError(response, "Failed to fetch node id.") + + responseJson = response.getJson(dict) + data = responseJson.get("data") + + if not isinstance(data, list): + raise TypeError(f"Invalid \"data\" type {type(data)}. Expected: \"list\"") + + if len(data) == 0: + raise ValueError(f"Node with name \"{name}\" not found.") + + nodeJson = data[0] + if not isinstance(nodeJson, dict): + raise TypeError(f"Invalid \"nodeJson\" type {type(nodeJson)}. Expected: \"dict\"") + + id = nodeJson.get("id") + if not isinstance(id, int): + raise TypeError(f"Invalid \"id\" type {type(id)}. Expected: \"int\"") + + return id diff --git a/coretex/utils/docker.py b/coretex/utils/docker.py index 42893b55..f7d6785d 100644 --- a/coretex/utils/docker.py +++ b/coretex/utils/docker.py @@ -8,6 +8,10 @@ from ..statistics import getTotalSwapMemory +class DockerConfigurationException(Exception): + pass + + def isDockerAvailable() -> None: try: # Run the command to check if Docker exists and is available diff --git a/tests/resources/computer_vision_dataset/1.zip b/tests/resources/computer_vision_dataset/1.zip index f3ec7f86c13cd3b50f321ea47643585fb03fb627..93acb584706fe0e5b61ec4fe77ce6c0fec411948 100644 GIT binary patch delta 52 vcmZ4aj&bEXM!o=VW)=|!5O`F|8M%>9zKjLLm~2>90b#5yGl4Lkmbm}`w*wLf delta 52 vcmZ4aj&bEXM!o=VW)=|!5D05L7PgU3zKjLLm~2>90b#5yGl4Lkmbm}`#ETN| diff --git a/tests/resources/computer_vision_dataset/classes.json b/tests/resources/computer_vision_dataset/classes.json index 9d0fb389..1f76c1c8 100644 --- a/tests/resources/computer_vision_dataset/classes.json +++ b/tests/resources/computer_vision_dataset/classes.json @@ -1 +1 @@ -[{"ids": ["58257e5c-d33f-425d-8ac1-f21a7a62d2fd"], "name": "test_class_1", "color": "#a12997"}, {"ids": ["e3cc941a-2fbe-481c-8493-5cef24ecc30e"], "name": "test_class_2", "color": "#557633"}] \ No newline at end of file +[{"ids": ["98753b71-3b6f-4366-9e03-3f88d8532c77"], "name": "test_class_1", "color": "#33717c"}, {"ids": ["91107daf-301e-4399-b27a-18cd85ab9011"], "name": "test_class_2", "color": "#e33b57"}] \ No newline at end of file From c1b8e7712d96a5ceb1731f7e4a68fe246c8020bc Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Mon, 12 Aug 2024 13:02:09 +0200 Subject: [PATCH 27/28] CTX-5430: Discussion changes and minor bug fixes (variables not declared when they are supposed to and cause crash) --- coretex/cli/commands/node.py | 6 +++--- coretex/cli/modules/node.py | 14 ++++++-------- coretex/configuration/__init__.py | 2 ++ coretex/configuration/node.py | 4 +--- .../resources/computer_vision_dataset/classes.json | 2 +- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/coretex/cli/commands/node.py b/coretex/cli/commands/node.py index 9c0d55cd..f5b2dcdd 100644 --- a/coretex/cli/commands/node.py +++ b/coretex/cli/commands/node.py @@ -141,8 +141,8 @@ def update(autoAccept: bool, autoDecline: bool) -> None: @click.command() -@click.option("--verbose", is_flag = True, help = "Configure node settings manually.") -def config(verbose: bool) -> None: +@click.option("--advanced", is_flag = True, help = "Configure node settings manually.") +def config(advanced: bool) -> None: if node_module.isRunning(): if not ui.clickPrompt( "Node is already running. Do you wish to stop the Node? (Y/n)", @@ -171,7 +171,7 @@ def config(verbose: bool) -> None: except (ConfigurationNotFound, InvalidConfiguration): pass - nodeConfig = node_module.configureNode(verbose) + nodeConfig = node_module.configureNode(advanced) nodeConfig.save() ui.previewNodeConfig(nodeConfig) diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index 0bb4bf23..ad2af94d 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -403,7 +403,7 @@ def configureNode(advanced: bool) -> NodeConfiguration: nodeConfig.initScript = config_defaults.DEFAULT_INIT_SCRIPT if advanced: - nodeMode = selectNodeMode() + nodeConfig.mode = selectNodeMode() nodeConfig.storagePath = ui.clickPrompt("Storage path (press enter to use default)", config_defaults.DEFAULT_STORAGE_PATH, type = str) nodeConfig.cpuCount = promptCpu(cpuLimit) @@ -424,15 +424,14 @@ def configureNode(advanced: bool) -> NodeConfiguration: nodeConfig.initScript = _configureInitScript() - nodeSecret: str = ui.clickPrompt( + nodeConfig.secret = ui.clickPrompt( "Enter a secret which will be used to generate RSA key-pair for Node", config_defaults.DEFAULT_NODE_SECRET, type = str, hide_input = True ) - nodeConfig.secret = nodeSecret - if nodeMode in [NodeMode.endpointReserved, NodeMode.endpointShared]: + if nodeConfig.mode in [NodeMode.endpointReserved, NodeMode.endpointShared]: nodeConfig.nearWalletId = ui.clickPrompt( "Enter a NEAR wallet id to which the funds will be transfered when executing endpoints", config_defaults.DEFAULT_NEAR_WALLET_ID, @@ -444,19 +443,18 @@ def configureNode(advanced: bool) -> NodeConfiguration: ui.stdEcho("To configure node manually run coretex node config with --verbose flag.") publicKey: Optional[bytes] = None - if nodeSecret != config_defaults.DEFAULT_NODE_SECRET: + if isinstance(nodeConfig.secret, str) and nodeConfig.secret != config_defaults.DEFAULT_NODE_SECRET: ui.progressEcho("Generating RSA key-pair (2048 bits long) using provided node secret...") - rsaKey = rsa.generateKey(2048, nodeSecret.encode("utf-8")) + rsaKey = rsa.generateKey(2048, nodeConfig.secret.encode("utf-8")) publicKey = rsa.getPublicKeyBytes(rsaKey.public_key()) nodeConfig.id, nodeConfig.accessToken = registerNode( nodeConfig.name, - nodeMode, + nodeConfig.mode, publicKey, nodeConfig.nearWalletId, nodeConfig.endpointInvocationPrice ) - nodeConfig.mode = nodeMode return nodeConfig diff --git a/coretex/configuration/__init__.py b/coretex/configuration/__init__.py index f83b0b42..c14ca8c1 100644 --- a/coretex/configuration/__init__.py +++ b/coretex/configuration/__init__.py @@ -19,6 +19,7 @@ import os import json +import shutil import logging from .user import UserConfiguration @@ -61,6 +62,7 @@ def configMigration(configPath: Path) -> None: }).save() configPath.unlink() + shutil.rmtree(DEFAULT_VENV_PATH) def _syncConfigWithEnv() -> None: diff --git a/coretex/configuration/node.py b/coretex/configuration/node.py index 783edd8f..247a25bc 100644 --- a/coretex/configuration/node.py +++ b/coretex/configuration/node.py @@ -18,9 +18,8 @@ from typing import List, Optional, Tuple from pathlib import Path -from . import config_defaults +from . import utils, config_defaults from .base import BaseConfiguration, CONFIG_DIR -from . import utils from ..utils import docker from ..node import NodeMode from ..networking import networkManager, NetworkRequestError @@ -277,7 +276,6 @@ def fetchNodeId(self) -> int: if not isinstance(id, int): raise TypeError(f"Invalid \"id\" type {type(id)}. Expected: \"int\"") - self.id = int(id) self.save() diff --git a/tests/resources/computer_vision_dataset/classes.json b/tests/resources/computer_vision_dataset/classes.json index 1f76c1c8..16f91911 100644 --- a/tests/resources/computer_vision_dataset/classes.json +++ b/tests/resources/computer_vision_dataset/classes.json @@ -1 +1 @@ -[{"ids": ["98753b71-3b6f-4366-9e03-3f88d8532c77"], "name": "test_class_1", "color": "#33717c"}, {"ids": ["91107daf-301e-4399-b27a-18cd85ab9011"], "name": "test_class_2", "color": "#e33b57"}] \ No newline at end of file +[{"ids": ["58257e5c-d33f-425d-8ac1-f21a7a62d2fd"], "name": "test_class_1", "color": "#a12997"}, {"ids": ["e3cc941a-2fbe-481c-8493-5cef24ecc30e"], "name": "test_class_2", "color": "#557633"}] From 2d5c7747f0b47975212cb21b071f585ecbd30486 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Tue, 13 Aug 2024 11:47:39 +0200 Subject: [PATCH 28/28] CTX-5430: replaced modified files with ones from dev. --- tests/resources/computer_vision_dataset/1.zip | Bin 30505 -> 30505 bytes .../computer_vision_dataset/classes.json | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/resources/computer_vision_dataset/1.zip b/tests/resources/computer_vision_dataset/1.zip index 93acb584706fe0e5b61ec4fe77ce6c0fec411948..f3ec7f86c13cd3b50f321ea47643585fb03fb627 100644 GIT binary patch delta 52 vcmZ4aj&bEXM!o=VW)=|!5D05L7PgU3zKjLLm~2>90b#5yGl4Lkmbm}`#ETN| delta 52 vcmZ4aj&bEXM!o=VW)=|!5O`F|8M%>9zKjLLm~2>90b#5yGl4Lkmbm}`w*wLf diff --git a/tests/resources/computer_vision_dataset/classes.json b/tests/resources/computer_vision_dataset/classes.json index 16f91911..9d0fb389 100644 --- a/tests/resources/computer_vision_dataset/classes.json +++ b/tests/resources/computer_vision_dataset/classes.json @@ -1 +1 @@ -[{"ids": ["58257e5c-d33f-425d-8ac1-f21a7a62d2fd"], "name": "test_class_1", "color": "#a12997"}, {"ids": ["e3cc941a-2fbe-481c-8493-5cef24ecc30e"], "name": "test_class_2", "color": "#557633"}] +[{"ids": ["58257e5c-d33f-425d-8ac1-f21a7a62d2fd"], "name": "test_class_1", "color": "#a12997"}, {"ids": ["e3cc941a-2fbe-481c-8493-5cef24ecc30e"], "name": "test_class_2", "color": "#557633"}] \ No newline at end of file