Merge branch 'develop' into CTX-6577-v1

# Conflicts:
#	coretex/networking/utils.py

coretex/_task/run_logger/run_logger.py

@@ -39,12 +39,7 @@ def uploadTaskRunLogs(taskRunId: int, logs: List[Log]) -> bool:
             "logs": [log.encode() for log in logs]
         }
 
-        response = networkManager.post(
-            "model-queue/add-console-log",
-            params,
-            timeout = (5, 600)  # connection timeout 5 seconds, log upload timeout 600 seconds
-        )
-
+        response = networkManager.post("model-queue/add-console-log", params)
         return not response.hasFailed()
     except RequestFailedError as ex:
         logging.getLogger("coretexpylib").error(f">> Failed to upload console logs to Coretex. Reason: {ex}")

coretex/cli/commands/login.py

@@ -18,7 +18,7 @@
 import click
 
 from ..modules import user, ui
-from ...configuration import UserConfiguration, InvalidConfiguration, ConfigurationNotFound
+from ...configuration import UserConfiguration, InvalidConfiguration, ConfigurationNotFound, utils
 
 
 @click.command()
@@ -38,5 +38,9 @@ def login() -> None:
 
     ui.stdEcho("Please enter your credentials:")
     userConfig = user.configUser()
+
+    initialData = utils.fetchInitialData()
+    userConfig.frontendUrl = initialData.get("frontend_url", "app.coretex.ai/")
+
     userConfig.save()
     ui.successEcho(f"User {userConfig.username} successfully logged in.")

coretex/cli/commands/model.py

@@ -31,7 +31,7 @@ def create(name: str, path: str, project: Optional[str], accuracy: float) -> Non
     model.upload(path)
 
     ui.successEcho(f"Model \"{model.name}\" created successfully")
-    ui.stdEcho(f"A new model has been created. You can open it by clicking on this URL {ui.outputUrl(model.entityUrl())}.")
+    ui.stdEcho(f"A new model has been created. You can open it by clicking on this URL {ui.outputUrl(userConfig.frontendUrl, model.entityUrl())}.")
 
 
 @click.group()

coretex/cli/commands/project.py

@@ -32,8 +32,8 @@
 @click.option("--type", "-t", "projectType", type = int, help = "Project type")
 @click.option("--description", "-d", type = str, help = "Project description")
 def create(name: Optional[str], projectType: Optional[int], description: Optional[str]) -> None:
-    project = project_utils.createProject(name, projectType, description)
     userConfig = UserConfiguration.load()
+    project = project_utils.createProject(userConfig.frontendUrl, name, projectType, description)
 
     selectNewProject = ui.clickPrompt("Do you want to select the new project as default? (Y/n)", type = bool, default = True)
     if selectNewProject:
@@ -86,7 +86,11 @@ def select(name: str) -> None:
         userConfig.selectProject(project.id)
     except ValueError:
         ui.errorEcho(f"Project \"{name}\" not found.")
-        project = project_utils.promptProjectCreate("Do you want to create a project with that name?", name)
+        project = project_utils.promptProjectCreate(
+            "Do you want to create a project with that name?",
+            name,
+            userConfig.frontendUrl
+        )
         if project is None:
             return
 

coretex/cli/commands/task.py

@@ -18,14 +18,16 @@
 from typing import Optional
 
 import click
+import webbrowser
 
+from ..modules import ui
 from ..modules.project_utils import getProject
 from ..modules.user import initializeUserSession
 from ..modules.utils import onBeforeCommandExecute
 from ..modules.project_utils import getProject
 from ..._folder_manager import folder_manager
 from ..._task import TaskRunWorker, executeRunLocally, readTaskConfig, runLogger
-from ...configuration import UserConfiguration, NodeConfiguration
+from ...configuration import UserConfiguration
 from ...entities import TaskRun, TaskRunStatus
 from ...resources import PYTHON_ENTRY_POINT_PATH
 from ..._task import TaskRunWorker, executeRunLocally, readTaskConfig, runLogger
@@ -36,7 +38,6 @@ class RunException(Exception):
 
 
 @click.command()
-@onBeforeCommandExecute(initializeUserSession)
 @click.argument("path", type = click.Path(exists = True, dir_okay = False))
 @click.option("--name", type = str, default = None)
 @click.option("--description", type = str, default = None)
@@ -58,6 +59,14 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo
     if selectedProject is None:
         return
 
+    ui.stdEcho(
+        "Project info: "
+        f"\n\tName: {selectedProject.name}"
+        f"\n\tProject type: {selectedProject.projectType.name}"
+        f"\n\tDescription: {selectedProject.description}"
+        f"\n\tCreated on: {selectedProject.createdOn}"
+    )
+
     taskRun: TaskRun = TaskRun.runLocal(
         selectedProject.id,
         snapshot,
@@ -67,6 +76,12 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo
         entryPoint = path
     )
 
+    ui.stdEcho(
+        "Task Run successfully started. "
+        f"You can open it by clicking on this URL {ui.outputUrl(userConfig.frontendUrl, taskRun.entityUrl())}."
+    )
+    webbrowser.open(f"{userConfig.frontendUrl}/{taskRun.entityUrl()}")
+
     taskRun.updateStatus(TaskRunStatus.preparingToStart)
 
     with TaskRunWorker(userConfig.refreshToken, taskRun.id):
@@ -95,3 +110,12 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo
         taskRun.updateStatus(TaskRunStatus.completedWithSuccess)
 
     folder_manager.clearTempFiles()
+
+
+@click.group()
+@onBeforeCommandExecute(initializeUserSession)
+def task() -> None:
+    pass
+
+
+task.add_command(run, "run")

coretex/cli/main.py

@@ -22,7 +22,7 @@
 from .commands.login import login
 from .commands.model import model
 from .commands.node import node
-from .commands.task import run
+from .commands.task import task
 from .commands.project import project
 
 from .modules import ui, utils
@@ -67,6 +67,6 @@ def cli() -> None:
 cli.add_command(model)
 cli.add_command(project)
 cli.add_command(node)
-cli.add_command(run)
+cli.add_command(task)
 cli.add_command(version)
 cli.add_command(update)

coretex/cli/modules/node.py

@@ -72,7 +72,8 @@ def start(dockerImage: str, nodeConfig: NodeConfiguration) -> None:
             "CTX_API_URL": os.environ["CTX_API_URL"],
             "CTX_STORAGE_PATH": "/root/.coretex",
             "CTX_NODE_ACCESS_TOKEN": nodeConfig.accessToken,
-            "CTX_NODE_MODE": str(nodeConfig.mode)
+            "CTX_NODE_MODE": str(nodeConfig.mode),
+            "CTX_HEARTBEAT_INTERVAL": str(nodeConfig.heartbeatInterval)
         }
 
         if nodeConfig.modelId is not None:
@@ -443,11 +444,17 @@ def configureNode(advanced: bool) -> NodeConfiguration:
             )
 
             nodeConfig.endpointInvocationPrice = promptInvocationPrice()
+
+        nodeConfig.heartbeatInterval = ui.clickPrompt(
+            "Enter interval (seconds) at which the Node will send heartbeat to Coretex Server",
+            config_defaults.HEARTBEAT_INTERVAL // 1000,
+            type = int
+        ) * 1000  # Node expects the value in ms
     else:
-        ui.stdEcho("To configure node manually run coretex node config with --verbose flag.")
+        ui.stdEcho("To configure node manually run coretex node config with --advanced flag.")
 
     publicKey: Optional[bytes] = None
-    if isinstance(nodeConfig.secret, str) and nodeConfig.secret != config_defaults.DEFAULT_NODE_SECRET:
+    if nodeConfig.secret is not None and nodeConfig.secret != config_defaults.DEFAULT_NODE_SECRET:
         ui.progressEcho("Generating RSA key-pair (2048 bits long) using provided node secret...")
         rsaKey = rsa.generateKey(2048, nodeConfig.secret.encode("utf-8"))
         publicKey = rsa.getPublicKeyBytes(rsaKey.public_key())

coretex/cli/modules/project_utils.py

@@ -40,7 +40,7 @@ def selectProjectVisibility() -> ProjectVisibility:
     return selectedProjectVisibility
 
 
-def promptProjectCreate(message: str, name: str) -> Optional[Project]:
+def promptProjectCreate(message: str, name: str, frontendUrl: str) -> Optional[Project]:
     if not click.confirm(message, default = True):
         return None
 
@@ -49,7 +49,10 @@ def promptProjectCreate(message: str, name: str) -> Optional[Project]:
     try:
         project = Project.createProject(name, selectedProjectType)
         ui.successEcho(f"Project \"{name}\" created successfully.")
-        ui.stdEcho(f"A new Project has been created. You can open it by clicking on this URL {ui.outputUrl(project.entityUrl())}.")
+        ui.stdEcho(
+            "A new Project has been created. "
+            f"You can open it by clicking on this URL {ui.outputUrl(frontendUrl, project.entityUrl())}."
+        )
         return project
     except NetworkRequestError as ex:
         logging.getLogger("cli").debug(ex, exc_info = ex)
@@ -66,7 +69,11 @@ def promptProjectSelect(userConfig: UserConfiguration) -> Optional[Project]:
         userConfig.selectProject(project.id)
     except ValueError:
         ui.errorEcho(f"Project \"{name}\" not found.")
-        newProject = promptProjectCreate("Do you want to create a project with that name?", name)
+        newProject = promptProjectCreate(
+            "Do you want to create a project with that name?",
+            name,
+            userConfig.frontendUrl
+        )
         if newProject is None:
             return None
 
@@ -75,7 +82,7 @@ def promptProjectSelect(userConfig: UserConfiguration) -> Optional[Project]:
     return project
 
 
-def createProject(name: Optional[str] = None, projectType: Optional[int] = None, description: Optional[str] = None) -> Project:
+def createProject(frontendUrl: str, name: Optional[str] = None, projectType: Optional[int] = None, description: Optional[str] = None) -> Project:
     if name is None:
         name = ui.clickPrompt("Please enter name of the project you want to create", type = str)
 
@@ -90,7 +97,10 @@ def createProject(name: Optional[str] = None, projectType: Optional[int] = None,
     try:
         project = Project.createProject(name, projectType, description = description)
         ui.successEcho(f"Project \"{name}\" created successfully.")
-        ui.stdEcho(f"A new Project has been created. You can open it by clicking on this URL {ui.outputUrl(project.entityUrl())}.")
+        ui.stdEcho(
+            "A new Project has been created. "
+            f"You can open it by clicking on this URL {ui.outputUrl(frontendUrl, project.entityUrl())}."
+        )
         return project
     except NetworkRequestError as ex:
         logging.getLogger("cli").debug(ex, exc_info = ex)
@@ -104,7 +114,11 @@ def getProject(name: Optional[str], userConfig: UserConfiguration) -> Optional[P
             return Project.fetchOne(name = name)
         except:
             if projectId is None:
-                return promptProjectCreate("Project not found. Do you want to create a new Project with that name?", name)
+                return promptProjectCreate(
+                    "Project not found. Do you want to create a new Project with that name?",
+                    name,
+                    userConfig.frontendUrl
+                )
 
             return Project.fetchById(projectId)
 
@@ -116,6 +130,6 @@ def getProject(name: Optional[str], userConfig: UserConfiguration) -> Optional[P
         if not click.confirm("Would you like to create a new Project?", default = True):
             return None
 
-        return createProject(name)
+        return createProject(userConfig.frontendUrl, name)
 
     return Project.fetchById(projectId)

coretex/cli/modules/ui.py

@@ -58,17 +58,18 @@ def previewNodeConfig(nodeConfig: NodeConfiguration) -> None:
         nodeSecret = "********"
 
     table = [
-        ["Node name",                   nodeConfig.name],
-        ["Coretex Node type",           nodeConfig.image],
-        ["Storage path",                nodeConfig.storagePath],
-        ["RAM",                         f"{nodeConfig.ram}GB"],
-        ["SWAP memory",                 f"{nodeConfig.swap}GB"],
-        ["POSIX shared memory",         f"{nodeConfig.sharedMemory}GB"],
-        ["CPU cores allocated",         f"{nodeConfig.cpuCount}"],
-        ["Coretex Node mode",           f"{NodeMode(nodeConfig.mode).name}"],
-        ["Docker access",               allowDocker],
-        ["Coretex Node secret",         nodeSecret],
-        ["Coretex Node init script",    nodeConfig.initScript if nodeConfig.initScript is not None else ""]
+        ["Node name",               nodeConfig.name],
+        ["Node image",              nodeConfig.image],
+        ["Storage path",            nodeConfig.storagePath],
+        ["RAM",                     f"{nodeConfig.ram}GB"],
+        ["SWAP memory",             f"{nodeConfig.swap}GB"],
+        ["POSIX shared memory",     f"{nodeConfig.sharedMemory}GB"],
+        ["CPU cores allocated",     f"{nodeConfig.cpuCount}"],
+        ["Node mode",               f"{NodeMode(nodeConfig.mode).name}"],
+        ["Docker access",           allowDocker],
+        ["Node secret",             nodeSecret],
+        ["Node init script",        nodeConfig.initScript if nodeConfig.initScript is not None else ""],
+        ["Node heartbeat interval", f"{nodeConfig.heartbeatInterval // 1000}s"]
     ]
     if nodeConfig.modelId is not None:
         table.append(["Coretex Model ID", f"{nodeConfig.modelId}"])
@@ -82,8 +83,8 @@ def previewNodeConfig(nodeConfig: NodeConfiguration) -> None:
     stdEcho(tabulate(table))
 
 
-def outputUrl(entityUrl: str) -> str:
-    return ("\033[4m" + f"https://app.coretex.ai/{entityUrl}" + "\033[0m")
+def outputUrl(baseUrl: str, entityUrl: str) -> str:
+    return ("\033[4m" + f"{baseUrl}/{entityUrl}" + "\033[0m")
 
 
 def stdEcho(text: str) -> None:

coretex/cli/modules/user.py

@@ -15,6 +15,8 @@
 #     You should have received a copy of the GNU Affero General Public License
 #     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+from requests.exceptions import RequestException
+
 import requests
 
 from . import ui
@@ -27,7 +29,7 @@ def validateServerUrl(serverUrl: str) -> bool:
         endpoint = baseUrl(serverUrl) + "/info.json"
         response = requests.get(endpoint, timeout = 5)
         return response.ok
-    except BaseException as ex:
+    except RequestException:
         return False
 
 
@@ -38,7 +40,7 @@ def configureServerUrl() -> str:
     if not "Official" in selectedChoice:
         serverUrl: str = ui.clickPrompt("Enter server url that you wish to use", type = str)
 
-        if not validateServerUrl(serverUrl):
+        while not validateServerUrl(serverUrl):
             serverUrl = ui.clickPrompt("You've entered invalid server url. Please try again.", type = str)
 
         return serverUrl

coretex/configuration/config_defaults.py

@@ -20,3 +20,4 @@
 DEFAULT_INIT_SCRIPT = ""
 DEFAULT_NEAR_WALLET_ID = ""
 DEFAULT_ENDPOINT_INVOCATION_PRICE = 0.0
+HEARTBEAT_INTERVAL = 5000  # ms

coretex/configuration/node.py

@@ -197,6 +197,14 @@ def endpointInvocationPrice(self) -> Optional[float]:
     def endpointInvocationPrice(self, value: Optional[float]) -> None:
         self._raw["endpointInvocationPrice"] = value
 
+    @property
+    def heartbeatInterval(self) -> int:
+        return self.getValue("heartbeatInterval", int, default = config_defaults.HEARTBEAT_INTERVAL)
+
+    @heartbeatInterval.setter
+    def heartbeatInterval(self, value: int) -> None:
+        self._raw["heartbeatInterval"] = value
+
     def _isConfigValid(self) -> Tuple[bool, List[str]]:
         isValid = True
         errorMessages = []

coretex/configuration/user.py

@@ -96,6 +96,14 @@ def projectId(self) -> Optional[int]:
     def projectId(self, value: Optional[int]) -> None:
         self._raw["projectId"] = value
 
+    @property
+    def frontendUrl(self) -> str:
+        return self.getValue("frontendUrl", str, default = "app.coretex.ai")
+
+    @frontendUrl.setter
+    def frontendUrl(self, value: Optional[str]) -> None:
+        self._raw["frontendUrl"] = value
+
     def _isConfigValid(self) -> Tuple[bool, List[str]]:
         isValid = True
         errorMessages = []

coretex/configuration/utils.py

@@ -15,7 +15,7 @@
 #     You should have received a copy of the GNU Affero General Public License
 #     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from typing import Optional, Tuple, Any
+from typing import Optional, Tuple, Any, Dict
 
 from . import config_defaults
 from ..networking import networkManager, NetworkRequestError
@@ -132,3 +132,12 @@ def fetchNodeId(name: str) -> int:
         raise TypeError(f"Invalid \"id\" type {type(id)}. Expected: \"int\"")
 
     return id
+
+
+def fetchInitialData() -> Dict[str, Any]:
+    response = networkManager.get("user/initial-data")
+
+    if response.hasFailed():
+        raise NetworkRequestError(response, "Failed to fetch user's initial data.")
+
+    return response.getJson(dict)

coretex/entities/dataset/network_dataset.py

@@ -65,7 +65,7 @@ def _chunkSampleImport(sampleType: Type[SampleType], sampleName: str, samplePath
         "file_id": fileChunkUpload(samplePath)
     }
 
-    response = networkManager.formData("session/import", parameters, timeout = (5, 300))
+    response = networkManager.formData("session/import", parameters)
     if response.hasFailed():
         raise NetworkRequestError(response, f"Failed to create sample from \"{samplePath}\"")