diff --git a/coretex/cli/commands/task.py b/coretex/cli/commands/task.py index 18a1e17e..93d1f2cc 100644 --- a/coretex/cli/commands/task.py +++ b/coretex/cli/commands/task.py @@ -16,6 +16,7 @@ # along with this program. If not, see . from typing import Optional +from pathlib import Path import click import webbrowser @@ -24,13 +25,13 @@ from ..modules.project_utils import getProject from ..modules.user import initializeUserSession from ..modules.utils import onBeforeCommandExecute -from ..modules.project_utils import getProject +from ...networking import NetworkRequestError from ..._folder_manager import folder_manager from ..._task import TaskRunWorker, executeRunLocally, readTaskConfig, runLogger from ...configuration import UserConfiguration -from ...entities import TaskRun, TaskRunStatus +from ...entities import Task, TaskRun, TaskRunStatus +from ...entities.repository import checkIfCoretexRepoExists from ...resources import PYTHON_ENTRY_POINT_PATH -from ..._task import TaskRunWorker, executeRunLocally, readTaskConfig, runLogger class RunException(Exception): @@ -112,6 +113,45 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo folder_manager.clearTempFiles() +@click.command() +@click.argument("id", type = int, default = None, required = False) +def pull(id: Optional[int]) -> None: + if id is None and not checkIfCoretexRepoExists(): + id = ui.clickPrompt(f"There is no existing Task repository. Please specify id of Task you want to pull:", type = int) + + if id is not None: + try: + task = Task.fetchById(id) + except NetworkRequestError as ex: + ui.errorEcho(f"Failed to fetch Task id {id}. Reason: {ex.response}") + return + + if not task.coretexMetadataPath.exists(): + task.pull() + task.createMetadata() + task.fillMetadata() + else: + differences = task.getDiff() + if len(differences) == 0: + ui.stdEcho("Your repository is already updated.") + return + + ui.stdEcho("There are conflicts between your and remote repository.") + for diff in differences: + ui.stdEcho(f"File: {diff['path']} differs") + ui.stdEcho(f"\tLocal checksum: {diff['local_checksum']}") + ui.stdEcho(f"\tRemote checksum: {diff['remote_checksum']}") + + if not ui.clickPrompt("Do you want to pull the changes and update your local repository? (Y/n):", type = bool, default = True): + ui.stdEcho("No changes were made to your local repository.") + return + + task.pull() + task.createMetadata() + task.fillMetadata() + ui.stdEcho("Repository updated successfully.") + + @click.group() @onBeforeCommandExecute(initializeUserSession) def task() -> None: @@ -119,3 +159,4 @@ def task() -> None: task.add_command(run, "run") +task.add_command(pull, "pull") \ No newline at end of file diff --git a/coretex/entities/project/task.py b/coretex/entities/project/task.py index 29e567a5..5b2fb181 100644 --- a/coretex/entities/project/task.py +++ b/coretex/entities/project/task.py @@ -20,10 +20,11 @@ from .base import BaseObject from ..utils import isEntityNameValid +from ..repository import CoretexRepository, EntityCoretexRepositoryType from ...codable import KeyDescriptor -class Task(BaseObject): +class Task(BaseObject, CoretexRepository): """ Represents the task entity from Coretex.ai\n @@ -33,6 +34,18 @@ class Task(BaseObject): isDefault: bool taskId: int + @property + def entityCoretexRepositoryType(self) -> EntityCoretexRepositoryType: + return EntityCoretexRepositoryType.task + + @property + def paramKey(self) -> str: + return "sub_project_id" + + @property + def endpoint(self) -> str: + return "workspace" + @classmethod def _keyDescriptors(cls) -> Dict[str, KeyDescriptor]: descriptors = super()._keyDescriptors() diff --git a/coretex/entities/repository/__init__.py b/coretex/entities/repository/__init__.py new file mode 100644 index 00000000..bea739e1 --- /dev/null +++ b/coretex/entities/repository/__init__.py @@ -0,0 +1 @@ +from .coretex_repository import CoretexRepository, EntityCoretexRepositoryType, checkIfCoretexRepoExists diff --git a/coretex/entities/repository/coretex_repository.py b/coretex/entities/repository/coretex_repository.py new file mode 100644 index 00000000..b382afad --- /dev/null +++ b/coretex/entities/repository/coretex_repository.py @@ -0,0 +1,170 @@ +# Copyright (C) 2023 Coretex LLC + +# This file is part of Coretex.ai + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from typing import List, Dict, Any +from enum import IntEnum +from abc import abstractmethod, ABC +from pathlib import Path +from zipfile import ZipFile + +import os +import json +import logging + +from ...codable import Codable +from ...utils import generateSha256Checksum +from ...networking import networkManager, NetworkRequestError + + +def checkIfCoretexRepoExists() -> bool: + print(Path.cwd().joinpath(".coretex")) + return Path.cwd().joinpath(".coretex").exists() + + +class EntityCoretexRepositoryType(IntEnum): + + task = 1 + + +class CoretexRepository(ABC, Codable): + + id: int + projectId: int + + @property + def __initialMetadataPath(self) -> Path: + return Path(f"{self.id}/.metadata.json") + + @property + def coretexMetadataPath(self) -> Path: + return Path(f"{self.id}/.coretex.json") + + @property + @abstractmethod + def entityCoretexRepositoryType(self) -> EntityCoretexRepositoryType: + pass + + @property + @abstractmethod + def paramKey(self) -> str: + pass + + @property + @abstractmethod + def endpoint(self) -> str: + pass + + def pull(self) -> bool: + params = { + self.paramKey: self.id + } + + zipFilePath = f"{self.id}.zip" + response = networkManager.download(f"{self.endpoint}/download", zipFilePath, params) + + if response.hasFailed(): + logging.getLogger("coretexpylib").error(f">> [Coretex] {self.entityCoretexRepositoryType.name.capitalize()} download has failed") + return False + + with ZipFile(zipFilePath) as zipFile: + zipFile.extractall(str(self.id)) + + os.unlink(zipFilePath) + + return not response.hasFailed() + + def getRemoteMetadata(self) -> List: + # getRemoteMetadata downloads only .metadata file, this was needed so we can + # synchronize changes if multiple people work on the same Entity at the same time + + params = { + self.paramKey: self.id + } + + response = networkManager.get(f"{self.endpoint}/metadata", params) + if response.hasFailed(): + raise NetworkRequestError(response, "Failed to fetch task metadata.") + + return response.getJson(list, force = True) + + def createMetadata(self) -> None: + # createMetadata() function will store metadata of files that backend returns + # currently all files on backend (that weren't uploaded after checksum calculation change + # for files that is implemented recently on backend) return null/None for their checksum + # if backend returns None for checksum of some file we need to calculate initial checksum of + # the file so we can track changes + + with self.__initialMetadataPath.open("r") as initialMetadataFile: + initialMetadata = json.load(initialMetadataFile) + + # if backend returns null for checksum of file, generate checksum + for file in initialMetadata: + if file["checksum"] is None: + filePath = self.__initialMetadataPath.parent.joinpath(file["path"]) + if filePath.exists(): + file["checksum"] = generateSha256Checksum(filePath) + + newMetadata = { + "checksums": initialMetadata + } + + with self.coretexMetadataPath.open("w") as coretexMetadataFile: + json.dump(newMetadata, coretexMetadataFile, indent = 4) + + def fillMetadata(self) -> None: + # fillMetadata() function will update initial metadata returned from backend + # (file paths and their checksums) with other important Entity info (e.g. name, id, description...) + + localMetadata: Dict[str, Any] = {} + metadata = self.encode() + + with self.coretexMetadataPath.open("r") as coretexMetadataFile: + localMetadata = json.load(coretexMetadataFile) + + localMetadata.update(metadata) + + with self.coretexMetadataPath.open("w") as coretexMetadataFile: + json.dump(localMetadata, coretexMetadataFile, indent = 4) + + self.__initialMetadataPath.unlink() + + def getDiff(self) -> List[Dict[str, Any]]: + # getDiff() checks initial checksums of files stored in .coretex file + # and compares them with their current checksums, based on that we know what files have changes + + with self.coretexMetadataPath.open("r") as localMetadataFile: + localMetadata = json.load(localMetadataFile) + + localChecksums = {file["path"]: file["checksum"] for file in localMetadata["checksums"]} + + differences = [] + + remoteMetadata = self.getRemoteMetadata() + for remoteFile in remoteMetadata: + remotePath = remoteFile["path"] + remoteChecksum = remoteFile["checksum"] + + localChecksum = localChecksums.get(remotePath) + + if localChecksum != remoteChecksum: + differences.append({ + "path": remotePath, + "local_checksum": localChecksum, + "remote_checksum": remoteChecksum + }) + + return differences diff --git a/coretex/entities/task_run/task_run.py b/coretex/entities/task_run/task_run.py index f2263cb2..3889f957 100644 --- a/coretex/entities/task_run/task_run.py +++ b/coretex/entities/task_run/task_run.py @@ -17,7 +17,7 @@ from typing import Optional, Any, List, Dict, Union, Tuple, TypeVar, Generic, Type from typing_extensions import Self, override -from zipfile import ZipFile, ZIP_DEFLATED +from zipfile import ZipFile from pathlib import Path import os diff --git a/coretex/networking/network_response.py b/coretex/networking/network_response.py index d187c1f2..6cc3c205 100644 --- a/coretex/networking/network_response.py +++ b/coretex/networking/network_response.py @@ -90,7 +90,7 @@ def isHead(self) -> bool: return self._raw.request.method == RequestType.head.value - def getJson(self, type_: Type[JsonType]) -> JsonType: + def getJson(self, type_: Type[JsonType], force: bool = False) -> JsonType: """ Converts HTTP response body to json @@ -109,7 +109,7 @@ def getJson(self, type_: Type[JsonType]) -> JsonType: TypeError -> If it was not possible to convert body to type of passed "type_" parameter """ - if not "application/json" in self.headers.get("Content-Type", ""): + if not force and not "application/json" in self.headers.get("Content-Type", ""): raise ValueError(f">> [Coretex] Trying to convert request response to json but response \"Content-Type\" was \"{self.headers.get('Content-Type')}\"") value = self._raw.json() diff --git a/coretex/utils/__init__.py b/coretex/utils/__init__.py index 80dd2753..b2788707 100644 --- a/coretex/utils/__init__.py +++ b/coretex/utils/__init__.py @@ -22,5 +22,5 @@ from .image import resizeWithPadding, cropToWidth from .process import logProcessOutput, command, CommandException from .logs import createFileHandler -from .misc import isCliRuntime +from .misc import isCliRuntime, generateSha256Checksum from .error_handling import Throws diff --git a/coretex/utils/misc.py b/coretex/utils/misc.py index 9df00d44..f6b57aa8 100644 --- a/coretex/utils/misc.py +++ b/coretex/utils/misc.py @@ -15,8 +15,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from pathlib import Path + import os import sys +import hashlib def isCliRuntime() -> bool: @@ -25,3 +28,14 @@ def isCliRuntime() -> bool: executablePath.endswith("/bin/coretex") and os.access(executablePath, os.X_OK) ) + + +def generateSha256Checksum(path: Path) -> str: + sha256 = hashlib.sha256() + chunkSize = 64 * 1024 # 65536 bytes + + with path.open("rb") as file: + while chunk := file.read(chunkSize): + sha256.update(chunk) + + return sha256.hexdigest()