From 12256d1245bd748099c7e4f5c6f33fd9ade2bfd5 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Mon, 26 Aug 2024 10:02:21 +0200 Subject: [PATCH 1/9] saving changes --- coretex/cli/commands/task.py | 9 +++++++++ coretex/cli/main.py | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/coretex/cli/commands/task.py b/coretex/cli/commands/task.py index ffb22dde..be75cb34 100644 --- a/coretex/cli/commands/task.py +++ b/coretex/cli/commands/task.py @@ -95,3 +95,12 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo taskRun.updateStatus(TaskRunStatus.completedWithSuccess) folder_manager.clearTempFiles() + + +@click.group() +@onBeforeCommandExecute(initializeUserSession) +def task() -> None: + pass + + +task.add_command(run, "run") diff --git a/coretex/cli/main.py b/coretex/cli/main.py index 96da0e0b..98c22d90 100644 --- a/coretex/cli/main.py +++ b/coretex/cli/main.py @@ -22,7 +22,7 @@ from .commands.login import login from .commands.model import model from .commands.node import node -from .commands.task import run +from .commands.task import task from .commands.project import project from .modules import ui, utils @@ -67,6 +67,6 @@ def cli() -> None: cli.add_command(model) cli.add_command(project) cli.add_command(node) -cli.add_command(run) +cli.add_command(task) cli.add_command(version) cli.add_command(update) From cda3b2bec3a33bcb14f02f7033a20fd21fbce9b3 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Thu, 29 Aug 2024 10:59:38 +0200 Subject: [PATCH 2/9] CTX-6563: saving changes regarding task pull command implementation. --- coretex/cli/commands/task.py | 10 ++++++ coretex/cli/modules/task.py | 50 +++++++++++++++++++++++++++ coretex/entities/task_run/task_run.py | 20 +++++++++++ 3 files changed, 80 insertions(+) create mode 100644 coretex/cli/modules/task.py diff --git a/coretex/cli/commands/task.py b/coretex/cli/commands/task.py index 18a1e17e..42c472a3 100644 --- a/coretex/cli/commands/task.py +++ b/coretex/cli/commands/task.py @@ -21,6 +21,7 @@ import webbrowser from ..modules import ui +from ..modules import task as task_utils from ..modules.project_utils import getProject from ..modules.user import initializeUserSession from ..modules.utils import onBeforeCommandExecute @@ -112,6 +113,14 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo folder_manager.clearTempFiles() +@click.command() +@click.argument("id", type = int, default = None) +def pull(id: int) -> None: + taskRun: TaskRun = TaskRun.fetchById(id) + taskRun.pull() + task_utils.createMetadata(id) + + @click.group() @onBeforeCommandExecute(initializeUserSession) def task() -> None: @@ -119,3 +128,4 @@ def task() -> None: task.add_command(run, "run") +task.add_command(pull, "pull") \ No newline at end of file diff --git a/coretex/cli/modules/task.py b/coretex/cli/modules/task.py new file mode 100644 index 00000000..68959209 --- /dev/null +++ b/coretex/cli/modules/task.py @@ -0,0 +1,50 @@ +# Copyright (C) 2023 Coretex LLC + +# This file is part of Coretex.ai + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from zipfile import ZipFile + +import os +import json + +from ...entities import TaskRun +from ...networking import networkManager, NetworkRequestError + + +def pull(id: int) -> None: + params = { + "sub_project_id": id + } + + zipFilePath = f"{id}.zip" + response = networkManager.download(f"workspace/download", zipFilePath, params) + + if response.hasFailed(): + raise NetworkRequestError(response, ">> [Coretex] Task download has failed") + + with ZipFile(zipFilePath) as zipFile: + zipFile.extractall(str(id)) + + # remove zip file after extract + os.unlink(zipFilePath) + + +def createMetadata(id: int) -> None: + with open(f"{id}/.metadata.json", "r") as initialMetadataFile: + initialMetadata = json.load(initialMetadataFile) + + with open(f"{id}/.coretex.json", "w") as metadataFile: + json.dump(initialMetadata, metadataFile, indent = 4) diff --git a/coretex/entities/task_run/task_run.py b/coretex/entities/task_run/task_run.py index 2d5beb84..0137a1aa 100644 --- a/coretex/entities/task_run/task_run.py +++ b/coretex/entities/task_run/task_run.py @@ -666,3 +666,23 @@ def generateEntityName(self) -> str: name = f"{self.id}-{self.name}" return name[:50] + + def pull(self) -> bool: + params = { + "sub_project_id": self.id + } + + zipFilePath = f"{self.id}.zip" + response = networkManager.download(f"workspace/download", zipFilePath, params) + + if response.hasFailed(): + logging.getLogger("coretexpylib").error(">> [Coretex] Task download has failed") + return False + + with ZipFile(zipFilePath) as zipFile: + zipFile.extractall(str(self.id)) + + # remove zip file after extract + os.unlink(zipFilePath) + + return not response.hasFailed() From 99cb21d746f995b9c575373f5a843d00efd2345a Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Thu, 29 Aug 2024 16:32:40 +0200 Subject: [PATCH 3/9] CTX-6563: Task pull (not task run) correction. --- coretex/cli/commands/task.py | 15 ++++++++------ coretex/cli/modules/task.py | 30 ++++++++++++++++++++++----- coretex/entities/project/task.py | 26 +++++++++++++++++++++++ coretex/entities/task_run/task_run.py | 22 +------------------- 4 files changed, 61 insertions(+), 32 deletions(-) diff --git a/coretex/cli/commands/task.py b/coretex/cli/commands/task.py index 42c472a3..67449e9d 100644 --- a/coretex/cli/commands/task.py +++ b/coretex/cli/commands/task.py @@ -16,6 +16,7 @@ # along with this program. If not, see . from typing import Optional +from pathlib import Path import click import webbrowser @@ -25,13 +26,11 @@ from ..modules.project_utils import getProject from ..modules.user import initializeUserSession from ..modules.utils import onBeforeCommandExecute -from ..modules.project_utils import getProject from ..._folder_manager import folder_manager from ..._task import TaskRunWorker, executeRunLocally, readTaskConfig, runLogger from ...configuration import UserConfiguration -from ...entities import TaskRun, TaskRunStatus +from ...entities import Task, TaskRun, TaskRunStatus from ...resources import PYTHON_ENTRY_POINT_PATH -from ..._task import TaskRunWorker, executeRunLocally, readTaskConfig, runLogger class RunException(Exception): @@ -116,9 +115,13 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo @click.command() @click.argument("id", type = int, default = None) def pull(id: int) -> None: - taskRun: TaskRun = TaskRun.fetchById(id) - taskRun.pull() - task_utils.createMetadata(id) + initialMetadataPath = Path(f"{id}/.metadata.json") + coretexMetadataPath = Path(f"{id}/.coretex.json") + + task = Task.fetchById(id) + task.pull() + task_utils.createMetadata(initialMetadataPath, coretexMetadataPath) + task_utils.fillTaskMetadata(task, initialMetadataPath, coretexMetadataPath) @click.group() diff --git a/coretex/cli/modules/task.py b/coretex/cli/modules/task.py index 68959209..45531166 100644 --- a/coretex/cli/modules/task.py +++ b/coretex/cli/modules/task.py @@ -15,12 +15,14 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Dict, Any +from pathlib import Path from zipfile import ZipFile import os import json -from ...entities import TaskRun +from ...entities import Task from ...networking import networkManager, NetworkRequestError @@ -42,9 +44,27 @@ def pull(id: int) -> None: os.unlink(zipFilePath) -def createMetadata(id: int) -> None: - with open(f"{id}/.metadata.json", "r") as initialMetadataFile: +def createMetadata(initialMetadataPath: Path, coretexMetadataPath: Path) -> None: + with open(initialMetadataPath, "r") as initialMetadataFile: initialMetadata = json.load(initialMetadataFile) - with open(f"{id}/.coretex.json", "w") as metadataFile: - json.dump(initialMetadata, metadataFile, indent = 4) + newMetadata = { + "checksums": initialMetadata + } + + with open(coretexMetadataPath, "w") as coretexMetadataFile: + json.dump(newMetadata, coretexMetadataFile, indent=4) + + +def fillTaskMetadata(task: Task, initialMetadataPath: Path, coretexMetadataPath: Path) -> None: + metadata = task.encode() + + with coretexMetadataPath.open("r") as coretexMetadataFile: + existing_metadata = json.load(coretexMetadataFile) + + existing_metadata.update(metadata) + + with coretexMetadataPath.open("w") as coretexMetadataFile: + json.dump(existing_metadata, coretexMetadataFile, indent=4) + + initialMetadataPath.unlink() \ No newline at end of file diff --git a/coretex/entities/project/task.py b/coretex/entities/project/task.py index 29e567a5..5c837985 100644 --- a/coretex/entities/project/task.py +++ b/coretex/entities/project/task.py @@ -17,10 +17,15 @@ from typing import Optional, Dict from typing_extensions import Self +from zipfile import ZipFile + +import os +import logging from .base import BaseObject from ..utils import isEntityNameValid from ...codable import KeyDescriptor +from ...networking import networkManager class Task(BaseObject): @@ -80,3 +85,24 @@ def createTask(cls, name: str, projectId: int, description: Optional[str]=None) parent_id = projectId, description = description ) + + + def pull(self) -> bool: + params = { + "sub_project_id": self.id + } + + zipFilePath = f"{self.id}.zip" + response = networkManager.download(f"workspace/download", zipFilePath, params) + + if response.hasFailed(): + logging.getLogger("coretexpylib").error(">> [Coretex] Task download has failed") + return False + + with ZipFile(zipFilePath) as zipFile: + zipFile.extractall(str(self.id)) + + # remove zip file after extract + os.unlink(zipFilePath) + + return not response.hasFailed() diff --git a/coretex/entities/task_run/task_run.py b/coretex/entities/task_run/task_run.py index 0137a1aa..75776d90 100644 --- a/coretex/entities/task_run/task_run.py +++ b/coretex/entities/task_run/task_run.py @@ -17,7 +17,7 @@ from typing import Optional, Any, List, Dict, Union, Tuple, TypeVar, Generic, Type from typing_extensions import Self, override -from zipfile import ZipFile, ZIP_DEFLATED +from zipfile import ZipFile from pathlib import Path import os @@ -666,23 +666,3 @@ def generateEntityName(self) -> str: name = f"{self.id}-{self.name}" return name[:50] - - def pull(self) -> bool: - params = { - "sub_project_id": self.id - } - - zipFilePath = f"{self.id}.zip" - response = networkManager.download(f"workspace/download", zipFilePath, params) - - if response.hasFailed(): - logging.getLogger("coretexpylib").error(">> [Coretex] Task download has failed") - return False - - with ZipFile(zipFilePath) as zipFile: - zipFile.extractall(str(self.id)) - - # remove zip file after extract - os.unlink(zipFilePath) - - return not response.hasFailed() From dcd2429b31db061290a5d050fbdf312f028697ba Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Mon, 2 Sep 2024 09:46:22 +0200 Subject: [PATCH 4/9] CTX-6563: saving changes --- coretex/cli/commands/task.py | 45 ++++++++++++++++--- coretex/cli/modules/task.py | 38 ++++++++++++++-- coretex/entities/project/task.py | 14 +++++- .../entities/repository/coretex_repository.py | 40 +++++++++++++++++ coretex/networking/network_response.py | 4 +- coretex/utils/misc.py | 14 ++++++ 6 files changed, 143 insertions(+), 12 deletions(-) create mode 100644 coretex/entities/repository/coretex_repository.py diff --git a/coretex/cli/commands/task.py b/coretex/cli/commands/task.py index 67449e9d..b50db7dc 100644 --- a/coretex/cli/commands/task.py +++ b/coretex/cli/commands/task.py @@ -26,6 +26,7 @@ from ..modules.project_utils import getProject from ..modules.user import initializeUserSession from ..modules.utils import onBeforeCommandExecute +from ...networking import NetworkRequestError from ..._folder_manager import folder_manager from ..._task import TaskRunWorker, executeRunLocally, readTaskConfig, runLogger from ...configuration import UserConfiguration @@ -113,15 +114,47 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo @click.command() -@click.argument("id", type = int, default = None) -def pull(id: int) -> None: +@click.argument("id", type = int, default = None, required = False) +def pull(id: Optional[int]) -> None: initialMetadataPath = Path(f"{id}/.metadata.json") coretexMetadataPath = Path(f"{id}/.coretex.json") - task = Task.fetchById(id) - task.pull() - task_utils.createMetadata(initialMetadataPath, coretexMetadataPath) - task_utils.fillTaskMetadata(task, initialMetadataPath, coretexMetadataPath) + if id is None and not coretexMetadataPath.exists(): + id = ui.clickPrompt(f"There is no existing Task repository. Please specify id of Task you want to pull:", type = int) + + if id is not None: + try: + task = Task.fetchById(id) + except NetworkRequestError as ex: + ui.errorEcho(f"Failed to fetch Task id {id}. Reason: {ex.response}") + return + + if not coretexMetadataPath.exists(): + task.pull() + task_utils.createMetadata(initialMetadataPath, coretexMetadataPath) + task_utils.fillTaskMetadata(task, initialMetadataPath, coretexMetadataPath) + else: + remoteMetadata = task.getMetadata() + differences = task_utils.checkMetadataDifference(remoteMetadata, coretexMetadataPath) + if len(differences) == 0: + ui.stdEcho("Your repository is already updated.") + return + + ui.stdEcho("There are conflicts between your and remote repository.") + for diff in differences: + ui.stdEcho(f"File: {diff['path']} differs") + ui.stdEcho(f" Local checksum: {diff['local_checksum']}") + ui.stdEcho(f" Remote checksum: {diff['remote_checksum']}") + + if not ui.clickPrompt("Do you want to pull the changes and update your local repository? (Y/n):", type = bool, default = True): + ui.stdEcho("No changes were made to your local repository.") + return + + task.pull() + task_utils.createMetadata(initialMetadataPath, coretexMetadataPath) + task_utils.fillTaskMetadata(task, initialMetadataPath, coretexMetadataPath) + ui.stdEcho("Repository updated successfully.") + @click.group() diff --git a/coretex/cli/modules/task.py b/coretex/cli/modules/task.py index 45531166..4cf24d44 100644 --- a/coretex/cli/modules/task.py +++ b/coretex/cli/modules/task.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Dict, Any +from typing import Dict, Any, Tuple, List from pathlib import Path from zipfile import ZipFile @@ -24,6 +24,7 @@ from ...entities import Task from ...networking import networkManager, NetworkRequestError +from ...utils.misc import generateSha256Checksum def pull(id: int) -> None: @@ -48,12 +49,19 @@ def createMetadata(initialMetadataPath: Path, coretexMetadataPath: Path) -> None with open(initialMetadataPath, "r") as initialMetadataFile: initialMetadata = json.load(initialMetadataFile) + # if backend returns null for checksum of file, generate checksum + for file in initialMetadata: + if file["checksum"] is None: + filePath = initialMetadataPath.parent.joinpath(file["path"]) + if filePath.exists(): + file["checksum"] = generateSha256Checksum(filePath) + newMetadata = { "checksums": initialMetadata } with open(coretexMetadataPath, "w") as coretexMetadataFile: - json.dump(newMetadata, coretexMetadataFile, indent=4) + json.dump(newMetadata, coretexMetadataFile, indent = 4) def fillTaskMetadata(task: Task, initialMetadataPath: Path, coretexMetadataPath: Path) -> None: @@ -67,4 +75,28 @@ def fillTaskMetadata(task: Task, initialMetadataPath: Path, coretexMetadataPath: with coretexMetadataPath.open("w") as coretexMetadataFile: json.dump(existing_metadata, coretexMetadataFile, indent=4) - initialMetadataPath.unlink() \ No newline at end of file + initialMetadataPath.unlink() + + +def checkMetadataDifference(remoteMetadata: list, coretexMetadataPath: Path) -> List[Dict[str, Any]]: + with coretexMetadataPath.open("r") as localMetadataFile: + localMetadata = json.load(localMetadataFile) + + localChecksums = {file['path']: file['checksum'] for file in localMetadata['checksums']} + + differences = [] + + for remoteFile in remoteMetadata: + remotePath = remoteFile['path'] + remoteChecksum = remoteFile['checksum'] + + localChecksum = localChecksums.get(remotePath) + + if localChecksum != remoteChecksum: + differences.append({ + 'path': remotePath, + 'local_checksum': localChecksum, + 'remote_checksum': remoteChecksum + }) + + return differences diff --git a/coretex/entities/project/task.py b/coretex/entities/project/task.py index 5c837985..a71a92d2 100644 --- a/coretex/entities/project/task.py +++ b/coretex/entities/project/task.py @@ -25,7 +25,7 @@ from .base import BaseObject from ..utils import isEntityNameValid from ...codable import KeyDescriptor -from ...networking import networkManager +from ...networking import networkManager, NetworkResponse class Task(BaseObject): @@ -86,6 +86,18 @@ def createTask(cls, name: str, projectId: int, description: Optional[str]=None) description = description ) + def getMetadata(self) -> Optional[list]: + params = { + "sub_project_id": self.id + } + + response = networkManager.get("workspace/metadata", params) + + if response.hasFailed(): + logging.getLogger("coretexpylib").error(">> [Coretex] Failed to fetch task metadata") + return None + + return response.getJson(list, force = True) def pull(self) -> bool: params = { diff --git a/coretex/entities/repository/coretex_repository.py b/coretex/entities/repository/coretex_repository.py new file mode 100644 index 00000000..0da695fc --- /dev/null +++ b/coretex/entities/repository/coretex_repository.py @@ -0,0 +1,40 @@ +# Copyright (C) 2023 Coretex LLC + +# This file is part of Coretex.ai + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from enum import IntEnum +from abc import abstractmethod, ABC + + +class EntityCoretexRepositoryType(IntEnum): + + task = 1 + + +class CoretexRepository(ABC): + + id: int + projectId: int + + + @property + @abstractmethod + def entityTagType(self) -> EntityCoretexRepositoryType: + pass + + + def pull(self): + pass \ No newline at end of file diff --git a/coretex/networking/network_response.py b/coretex/networking/network_response.py index d187c1f2..6cc3c205 100644 --- a/coretex/networking/network_response.py +++ b/coretex/networking/network_response.py @@ -90,7 +90,7 @@ def isHead(self) -> bool: return self._raw.request.method == RequestType.head.value - def getJson(self, type_: Type[JsonType]) -> JsonType: + def getJson(self, type_: Type[JsonType], force: bool = False) -> JsonType: """ Converts HTTP response body to json @@ -109,7 +109,7 @@ def getJson(self, type_: Type[JsonType]) -> JsonType: TypeError -> If it was not possible to convert body to type of passed "type_" parameter """ - if not "application/json" in self.headers.get("Content-Type", ""): + if not force and not "application/json" in self.headers.get("Content-Type", ""): raise ValueError(f">> [Coretex] Trying to convert request response to json but response \"Content-Type\" was \"{self.headers.get('Content-Type')}\"") value = self._raw.json() diff --git a/coretex/utils/misc.py b/coretex/utils/misc.py index 9df00d44..82382350 100644 --- a/coretex/utils/misc.py +++ b/coretex/utils/misc.py @@ -15,8 +15,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from pathlib import Path + import os import sys +import hashlib def isCliRuntime() -> bool: @@ -25,3 +28,14 @@ def isCliRuntime() -> bool: executablePath.endswith("/bin/coretex") and os.access(executablePath, os.X_OK) ) + + +def generateSha256Checksum(path: Path) -> str: + sha256 = hashlib.sha256() + chunkSize = 64 * 1024 # 65536 bytes + + with path.open("rb") as file: + while chunk := file.read(chunkSize): + sha256.update(chunk) + + return sha256.hexdigest() \ No newline at end of file From 0f7696dd44dcb91053ff01676b52ea18ab9f4184 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Mon, 2 Sep 2024 10:56:55 +0200 Subject: [PATCH 5/9] CTX-6563: CoretexRepository class created that will be responsible for "git-like" management inside coretexpylib. Working my way to stable and sustainable codebase. --- coretex/entities/project/task.py | 53 +++----- coretex/entities/repository/__init__.py | 1 + .../entities/repository/coretex_repository.py | 116 +++++++++++++++++- coretex/utils/__init__.py | 2 +- 4 files changed, 128 insertions(+), 44 deletions(-) create mode 100644 coretex/entities/repository/__init__.py diff --git a/coretex/entities/project/task.py b/coretex/entities/project/task.py index a71a92d2..5b2fb181 100644 --- a/coretex/entities/project/task.py +++ b/coretex/entities/project/task.py @@ -17,18 +17,14 @@ from typing import Optional, Dict from typing_extensions import Self -from zipfile import ZipFile - -import os -import logging from .base import BaseObject from ..utils import isEntityNameValid +from ..repository import CoretexRepository, EntityCoretexRepositoryType from ...codable import KeyDescriptor -from ...networking import networkManager, NetworkResponse -class Task(BaseObject): +class Task(BaseObject, CoretexRepository): """ Represents the task entity from Coretex.ai\n @@ -38,6 +34,18 @@ class Task(BaseObject): isDefault: bool taskId: int + @property + def entityCoretexRepositoryType(self) -> EntityCoretexRepositoryType: + return EntityCoretexRepositoryType.task + + @property + def paramKey(self) -> str: + return "sub_project_id" + + @property + def endpoint(self) -> str: + return "workspace" + @classmethod def _keyDescriptors(cls) -> Dict[str, KeyDescriptor]: descriptors = super()._keyDescriptors() @@ -85,36 +93,3 @@ def createTask(cls, name: str, projectId: int, description: Optional[str]=None) parent_id = projectId, description = description ) - - def getMetadata(self) -> Optional[list]: - params = { - "sub_project_id": self.id - } - - response = networkManager.get("workspace/metadata", params) - - if response.hasFailed(): - logging.getLogger("coretexpylib").error(">> [Coretex] Failed to fetch task metadata") - return None - - return response.getJson(list, force = True) - - def pull(self) -> bool: - params = { - "sub_project_id": self.id - } - - zipFilePath = f"{self.id}.zip" - response = networkManager.download(f"workspace/download", zipFilePath, params) - - if response.hasFailed(): - logging.getLogger("coretexpylib").error(">> [Coretex] Task download has failed") - return False - - with ZipFile(zipFilePath) as zipFile: - zipFile.extractall(str(self.id)) - - # remove zip file after extract - os.unlink(zipFilePath) - - return not response.hasFailed() diff --git a/coretex/entities/repository/__init__.py b/coretex/entities/repository/__init__.py new file mode 100644 index 00000000..bd6a388c --- /dev/null +++ b/coretex/entities/repository/__init__.py @@ -0,0 +1 @@ +from .coretex_repository import CoretexRepository, EntityCoretexRepositoryType, CORETEX_METADATA_PATH diff --git a/coretex/entities/repository/coretex_repository.py b/coretex/entities/repository/coretex_repository.py index 0da695fc..a02bd046 100644 --- a/coretex/entities/repository/coretex_repository.py +++ b/coretex/entities/repository/coretex_repository.py @@ -15,8 +15,23 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Optional, List, Dict, Any from enum import IntEnum from abc import abstractmethod, ABC +from pathlib import Path +from zipfile import ZipFile + +import os +import json +import logging + +from ...codable import Codable +from ...utils import generateSha256Checksum +from ...networking import networkManager, NetworkRequestError + + +INITIAL_METADATA_PATH = Path(f"{id}/.metadata.json") +CORETEX_METADATA_PATH = Path(f"{id}/.coretex.json") class EntityCoretexRepositoryType(IntEnum): @@ -24,7 +39,7 @@ class EntityCoretexRepositoryType(IntEnum): task = 1 -class CoretexRepository(ABC): +class CoretexRepository(ABC, Codable): id: int projectId: int @@ -32,9 +47,102 @@ class CoretexRepository(ABC): @property @abstractmethod - def entityTagType(self) -> EntityCoretexRepositoryType: + def entityCoretexRepositoryType(self) -> EntityCoretexRepositoryType: + pass + + @property + @abstractmethod + def paramKey(self) -> str: + pass + + @property + @abstractmethod + def endpoint(self) -> str: pass + def pull(self) -> bool: + params = { + self.paramKey: self.id + } + + zipFilePath = f"{self.id}.zip" + response = networkManager.download(f"{self.endpoint}/download", zipFilePath, params) + + if response.hasFailed(): + logging.getLogger("coretexpylib").error(f">> [Coretex] {self.entityCoretexRepositoryType.name.capitalize()} download has failed") + return False + + with ZipFile(zipFilePath) as zipFile: + zipFile.extractall(str(self.id)) + + # remove zip file after extract + os.unlink(zipFilePath) + + return not response.hasFailed() + + def getRemoteMetadata(self) -> List: + params = { + self.paramKey: self.id + } + + response = networkManager.get(f"{self.endpoint}/metadata", params) + if response.hasFailed(): + return NetworkRequestError(response, "Failed to fetch task metadata.") + + return response.getJson(list, force = True) + + def createMetadata(self) -> None: + with open(INITIAL_METADATA_PATH, "r") as initialMetadataFile: + initialMetadata = json.load(initialMetadataFile) + + # if backend returns null for checksum of file, generate checksum + for file in initialMetadata: + if file["checksum"] is None: + filePath = INITIAL_METADATA_PATH.parent.joinpath(file["path"]) + if filePath.exists(): + file["checksum"] = generateSha256Checksum(filePath) + + newMetadata = { + "checksums": initialMetadata + } + + with open(CORETEX_METADATA_PATH, "w") as coretexMetadataFile: + json.dump(newMetadata, coretexMetadataFile, indent = 4) + + def fillMetadata(self) -> None: + localMetadata: Dict[str, Any] = {} + metadata = self.encode() + + with CORETEX_METADATA_PATH.open("r") as coretexMetadataFile: + localMetadata = json.load(coretexMetadataFile) + + localMetadata.update(metadata) + + with CORETEX_METADATA_PATH.open("w") as coretexMetadataFile: + json.dump(localMetadata, coretexMetadataFile, indent = 4) + + INITIAL_METADATA_PATH.unlink() + + def checkDiff(self) -> List[Dict[str, Any]]: + with CORETEX_METADATA_PATH.open("r") as localMetadataFile: + localMetadata = json.load(localMetadataFile) + + localChecksums = {file["path"]: file["checksum"] for file in localMetadata["checksums"]} + + differences = [] + + remoteMetadata = self.getRemoteMetadata() + for remoteFile in remoteMetadata: + remotePath = remoteFile["path"] + remoteChecksum = remoteFile["checksum"] + + localChecksum = localChecksums.get(remotePath) + + if localChecksum != remoteChecksum: + differences.append({ + "path": remotePath, + "local_checksum": localChecksum, + "remote_checksum": remoteChecksum + }) - def pull(self): - pass \ No newline at end of file + return differences diff --git a/coretex/utils/__init__.py b/coretex/utils/__init__.py index 80dd2753..b2788707 100644 --- a/coretex/utils/__init__.py +++ b/coretex/utils/__init__.py @@ -22,5 +22,5 @@ from .image import resizeWithPadding, cropToWidth from .process import logProcessOutput, command, CommandException from .logs import createFileHandler -from .misc import isCliRuntime +from .misc import isCliRuntime, generateSha256Checksum from .error_handling import Throws From 55a35098bdba636e9a9c8114b0edf190d6258540 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Tue, 3 Sep 2024 10:52:13 +0200 Subject: [PATCH 6/9] CTX-6563: Saving changes --- coretex/cli/commands/task.py | 24 ++--- coretex/cli/modules/task.py | 102 ------------------ .../entities/repository/coretex_repository.py | 7 +- 3 files changed, 15 insertions(+), 118 deletions(-) delete mode 100644 coretex/cli/modules/task.py diff --git a/coretex/cli/commands/task.py b/coretex/cli/commands/task.py index b50db7dc..c468dcb2 100644 --- a/coretex/cli/commands/task.py +++ b/coretex/cli/commands/task.py @@ -22,7 +22,6 @@ import webbrowser from ..modules import ui -from ..modules import task as task_utils from ..modules.project_utils import getProject from ..modules.user import initializeUserSession from ..modules.utils import onBeforeCommandExecute @@ -31,6 +30,7 @@ from ..._task import TaskRunWorker, executeRunLocally, readTaskConfig, runLogger from ...configuration import UserConfiguration from ...entities import Task, TaskRun, TaskRunStatus +from ...entities.repository import CORETEX_METADATA_PATH from ...resources import PYTHON_ENTRY_POINT_PATH @@ -116,10 +116,7 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo @click.command() @click.argument("id", type = int, default = None, required = False) def pull(id: Optional[int]) -> None: - initialMetadataPath = Path(f"{id}/.metadata.json") - coretexMetadataPath = Path(f"{id}/.coretex.json") - - if id is None and not coretexMetadataPath.exists(): + if id is None and not CORETEX_METADATA_PATH.exists(): id = ui.clickPrompt(f"There is no existing Task repository. Please specify id of Task you want to pull:", type = int) if id is not None: @@ -129,13 +126,12 @@ def pull(id: Optional[int]) -> None: ui.errorEcho(f"Failed to fetch Task id {id}. Reason: {ex.response}") return - if not coretexMetadataPath.exists(): + if not CORETEX_METADATA_PATH.exists(): task.pull() - task_utils.createMetadata(initialMetadataPath, coretexMetadataPath) - task_utils.fillTaskMetadata(task, initialMetadataPath, coretexMetadataPath) + task.createMetadata() + task.fillMetadata() else: - remoteMetadata = task.getMetadata() - differences = task_utils.checkMetadataDifference(remoteMetadata, coretexMetadataPath) + differences = task.getDiff() if len(differences) == 0: ui.stdEcho("Your repository is already updated.") return @@ -143,16 +139,16 @@ def pull(id: Optional[int]) -> None: ui.stdEcho("There are conflicts between your and remote repository.") for diff in differences: ui.stdEcho(f"File: {diff['path']} differs") - ui.stdEcho(f" Local checksum: {diff['local_checksum']}") - ui.stdEcho(f" Remote checksum: {diff['remote_checksum']}") + ui.stdEcho(f"\tLocal checksum: {diff['local_checksum']}") + ui.stdEcho(f"\tRemote checksum: {diff['remote_checksum']}") if not ui.clickPrompt("Do you want to pull the changes and update your local repository? (Y/n):", type = bool, default = True): ui.stdEcho("No changes were made to your local repository.") return task.pull() - task_utils.createMetadata(initialMetadataPath, coretexMetadataPath) - task_utils.fillTaskMetadata(task, initialMetadataPath, coretexMetadataPath) + task.createMetadata() + task.fillMetadata() ui.stdEcho("Repository updated successfully.") diff --git a/coretex/cli/modules/task.py b/coretex/cli/modules/task.py deleted file mode 100644 index 4cf24d44..00000000 --- a/coretex/cli/modules/task.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (C) 2023 Coretex LLC - -# This file is part of Coretex.ai - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from typing import Dict, Any, Tuple, List -from pathlib import Path -from zipfile import ZipFile - -import os -import json - -from ...entities import Task -from ...networking import networkManager, NetworkRequestError -from ...utils.misc import generateSha256Checksum - - -def pull(id: int) -> None: - params = { - "sub_project_id": id - } - - zipFilePath = f"{id}.zip" - response = networkManager.download(f"workspace/download", zipFilePath, params) - - if response.hasFailed(): - raise NetworkRequestError(response, ">> [Coretex] Task download has failed") - - with ZipFile(zipFilePath) as zipFile: - zipFile.extractall(str(id)) - - # remove zip file after extract - os.unlink(zipFilePath) - - -def createMetadata(initialMetadataPath: Path, coretexMetadataPath: Path) -> None: - with open(initialMetadataPath, "r") as initialMetadataFile: - initialMetadata = json.load(initialMetadataFile) - - # if backend returns null for checksum of file, generate checksum - for file in initialMetadata: - if file["checksum"] is None: - filePath = initialMetadataPath.parent.joinpath(file["path"]) - if filePath.exists(): - file["checksum"] = generateSha256Checksum(filePath) - - newMetadata = { - "checksums": initialMetadata - } - - with open(coretexMetadataPath, "w") as coretexMetadataFile: - json.dump(newMetadata, coretexMetadataFile, indent = 4) - - -def fillTaskMetadata(task: Task, initialMetadataPath: Path, coretexMetadataPath: Path) -> None: - metadata = task.encode() - - with coretexMetadataPath.open("r") as coretexMetadataFile: - existing_metadata = json.load(coretexMetadataFile) - - existing_metadata.update(metadata) - - with coretexMetadataPath.open("w") as coretexMetadataFile: - json.dump(existing_metadata, coretexMetadataFile, indent=4) - - initialMetadataPath.unlink() - - -def checkMetadataDifference(remoteMetadata: list, coretexMetadataPath: Path) -> List[Dict[str, Any]]: - with coretexMetadataPath.open("r") as localMetadataFile: - localMetadata = json.load(localMetadataFile) - - localChecksums = {file['path']: file['checksum'] for file in localMetadata['checksums']} - - differences = [] - - for remoteFile in remoteMetadata: - remotePath = remoteFile['path'] - remoteChecksum = remoteFile['checksum'] - - localChecksum = localChecksums.get(remotePath) - - if localChecksum != remoteChecksum: - differences.append({ - 'path': remotePath, - 'local_checksum': localChecksum, - 'remote_checksum': remoteChecksum - }) - - return differences diff --git a/coretex/entities/repository/coretex_repository.py b/coretex/entities/repository/coretex_repository.py index a02bd046..e58fd67c 100644 --- a/coretex/entities/repository/coretex_repository.py +++ b/coretex/entities/repository/coretex_repository.py @@ -87,7 +87,7 @@ def getRemoteMetadata(self) -> List: response = networkManager.get(f"{self.endpoint}/metadata", params) if response.hasFailed(): - return NetworkRequestError(response, "Failed to fetch task metadata.") + raise NetworkRequestError(response, "Failed to fetch task metadata.") return response.getJson(list, force = True) @@ -123,7 +123,7 @@ def fillMetadata(self) -> None: INITIAL_METADATA_PATH.unlink() - def checkDiff(self) -> List[Dict[str, Any]]: + def getDiff(self) -> List[Dict[str, Any]]: with CORETEX_METADATA_PATH.open("r") as localMetadataFile: localMetadata = json.load(localMetadataFile) @@ -146,3 +146,6 @@ def checkDiff(self) -> List[Dict[str, Any]]: }) return differences + + # def updateRepository(self, differences: List[Dict[str, Any]]) -> None: + # pass From 55170ee75cea5f7822a4b6aa2390bec2bdc3c6a7 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 4 Sep 2024 09:56:17 +0200 Subject: [PATCH 7/9] CTX-6563: coretex task pull implementation. --- coretex/entities/repository/coretex_repository.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/coretex/entities/repository/coretex_repository.py b/coretex/entities/repository/coretex_repository.py index e58fd67c..fceb2ec2 100644 --- a/coretex/entities/repository/coretex_repository.py +++ b/coretex/entities/repository/coretex_repository.py @@ -146,6 +146,3 @@ def getDiff(self) -> List[Dict[str, Any]]: }) return differences - - # def updateRepository(self, differences: List[Dict[str, Any]]) -> None: - # pass From 70fbc551e16cbc5906dca0dacded5f5e1375fda4 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 4 Sep 2024 10:16:34 +0200 Subject: [PATCH 8/9] CTX-6563: Added comments for clarification and added coretexMetadataPath and initialMetadataPath as properties of base Repository class. --- coretex/cli/commands/task.py | 7 ++- coretex/entities/repository/__init__.py | 2 +- .../entities/repository/coretex_repository.py | 44 ++++++++++++++----- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/coretex/cli/commands/task.py b/coretex/cli/commands/task.py index c468dcb2..93d1f2cc 100644 --- a/coretex/cli/commands/task.py +++ b/coretex/cli/commands/task.py @@ -30,7 +30,7 @@ from ..._task import TaskRunWorker, executeRunLocally, readTaskConfig, runLogger from ...configuration import UserConfiguration from ...entities import Task, TaskRun, TaskRunStatus -from ...entities.repository import CORETEX_METADATA_PATH +from ...entities.repository import checkIfCoretexRepoExists from ...resources import PYTHON_ENTRY_POINT_PATH @@ -116,7 +116,7 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo @click.command() @click.argument("id", type = int, default = None, required = False) def pull(id: Optional[int]) -> None: - if id is None and not CORETEX_METADATA_PATH.exists(): + if id is None and not checkIfCoretexRepoExists(): id = ui.clickPrompt(f"There is no existing Task repository. Please specify id of Task you want to pull:", type = int) if id is not None: @@ -126,7 +126,7 @@ def pull(id: Optional[int]) -> None: ui.errorEcho(f"Failed to fetch Task id {id}. Reason: {ex.response}") return - if not CORETEX_METADATA_PATH.exists(): + if not task.coretexMetadataPath.exists(): task.pull() task.createMetadata() task.fillMetadata() @@ -152,7 +152,6 @@ def pull(id: Optional[int]) -> None: ui.stdEcho("Repository updated successfully.") - @click.group() @onBeforeCommandExecute(initializeUserSession) def task() -> None: diff --git a/coretex/entities/repository/__init__.py b/coretex/entities/repository/__init__.py index bd6a388c..bea739e1 100644 --- a/coretex/entities/repository/__init__.py +++ b/coretex/entities/repository/__init__.py @@ -1 +1 @@ -from .coretex_repository import CoretexRepository, EntityCoretexRepositoryType, CORETEX_METADATA_PATH +from .coretex_repository import CoretexRepository, EntityCoretexRepositoryType, checkIfCoretexRepoExists diff --git a/coretex/entities/repository/coretex_repository.py b/coretex/entities/repository/coretex_repository.py index fceb2ec2..b382afad 100644 --- a/coretex/entities/repository/coretex_repository.py +++ b/coretex/entities/repository/coretex_repository.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Optional, List, Dict, Any +from typing import List, Dict, Any from enum import IntEnum from abc import abstractmethod, ABC from pathlib import Path @@ -30,8 +30,9 @@ from ...networking import networkManager, NetworkRequestError -INITIAL_METADATA_PATH = Path(f"{id}/.metadata.json") -CORETEX_METADATA_PATH = Path(f"{id}/.coretex.json") +def checkIfCoretexRepoExists() -> bool: + print(Path.cwd().joinpath(".coretex")) + return Path.cwd().joinpath(".coretex").exists() class EntityCoretexRepositoryType(IntEnum): @@ -44,6 +45,13 @@ class CoretexRepository(ABC, Codable): id: int projectId: int + @property + def __initialMetadataPath(self) -> Path: + return Path(f"{self.id}/.metadata.json") + + @property + def coretexMetadataPath(self) -> Path: + return Path(f"{self.id}/.coretex.json") @property @abstractmethod @@ -75,12 +83,14 @@ def pull(self) -> bool: with ZipFile(zipFilePath) as zipFile: zipFile.extractall(str(self.id)) - # remove zip file after extract os.unlink(zipFilePath) return not response.hasFailed() def getRemoteMetadata(self) -> List: + # getRemoteMetadata downloads only .metadata file, this was needed so we can + # synchronize changes if multiple people work on the same Entity at the same time + params = { self.paramKey: self.id } @@ -92,13 +102,19 @@ def getRemoteMetadata(self) -> List: return response.getJson(list, force = True) def createMetadata(self) -> None: - with open(INITIAL_METADATA_PATH, "r") as initialMetadataFile: + # createMetadata() function will store metadata of files that backend returns + # currently all files on backend (that weren't uploaded after checksum calculation change + # for files that is implemented recently on backend) return null/None for their checksum + # if backend returns None for checksum of some file we need to calculate initial checksum of + # the file so we can track changes + + with self.__initialMetadataPath.open("r") as initialMetadataFile: initialMetadata = json.load(initialMetadataFile) # if backend returns null for checksum of file, generate checksum for file in initialMetadata: if file["checksum"] is None: - filePath = INITIAL_METADATA_PATH.parent.joinpath(file["path"]) + filePath = self.__initialMetadataPath.parent.joinpath(file["path"]) if filePath.exists(): file["checksum"] = generateSha256Checksum(filePath) @@ -106,25 +122,31 @@ def createMetadata(self) -> None: "checksums": initialMetadata } - with open(CORETEX_METADATA_PATH, "w") as coretexMetadataFile: + with self.coretexMetadataPath.open("w") as coretexMetadataFile: json.dump(newMetadata, coretexMetadataFile, indent = 4) def fillMetadata(self) -> None: + # fillMetadata() function will update initial metadata returned from backend + # (file paths and their checksums) with other important Entity info (e.g. name, id, description...) + localMetadata: Dict[str, Any] = {} metadata = self.encode() - with CORETEX_METADATA_PATH.open("r") as coretexMetadataFile: + with self.coretexMetadataPath.open("r") as coretexMetadataFile: localMetadata = json.load(coretexMetadataFile) localMetadata.update(metadata) - with CORETEX_METADATA_PATH.open("w") as coretexMetadataFile: + with self.coretexMetadataPath.open("w") as coretexMetadataFile: json.dump(localMetadata, coretexMetadataFile, indent = 4) - INITIAL_METADATA_PATH.unlink() + self.__initialMetadataPath.unlink() def getDiff(self) -> List[Dict[str, Any]]: - with CORETEX_METADATA_PATH.open("r") as localMetadataFile: + # getDiff() checks initial checksums of files stored in .coretex file + # and compares them with their current checksums, based on that we know what files have changes + + with self.coretexMetadataPath.open("r") as localMetadataFile: localMetadata = json.load(localMetadataFile) localChecksums = {file["path"]: file["checksum"] for file in localMetadata["checksums"]} From f3639b3252572d809fb599d9eaae22f8c9cd6014 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 4 Sep 2024 10:18:24 +0200 Subject: [PATCH 9/9] CTX-6563: Added missing newline at the end of the file. --- coretex/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coretex/utils/misc.py b/coretex/utils/misc.py index 82382350..f6b57aa8 100644 --- a/coretex/utils/misc.py +++ b/coretex/utils/misc.py @@ -38,4 +38,4 @@ def generateSha256Checksum(path: Path) -> str: while chunk := file.read(chunkSize): sha256.update(chunk) - return sha256.hexdigest() \ No newline at end of file + return sha256.hexdigest()