CTX-6563: coretex task pull implementation

coretex/cli/commands/task.py

@@ -16,6 +16,7 @@
 #     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 from typing import Optional
+from pathlib import Path
 
 import click
 import webbrowser
@@ -24,13 +25,13 @@
 from ..modules.project_utils import getProject
 from ..modules.user import initializeUserSession
 from ..modules.utils import onBeforeCommandExecute
-from ..modules.project_utils import getProject
+from ...networking import NetworkRequestError
 from ..._folder_manager import folder_manager
 from ..._task import TaskRunWorker, executeRunLocally, readTaskConfig, runLogger
 from ...configuration import UserConfiguration
-from ...entities import TaskRun, TaskRunStatus
+from ...entities import Task, TaskRun, TaskRunStatus
+from ...entities.repository import checkIfCoretexRepoExists
 from ...resources import PYTHON_ENTRY_POINT_PATH
-from ..._task import TaskRunWorker, executeRunLocally, readTaskConfig, runLogger
 
 
 class RunException(Exception):
@@ -112,10 +113,50 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo
     folder_manager.clearTempFiles()
 
 
+@click.command()
+@click.argument("id", type = int, default = None, required = False)
+def pull(id: Optional[int]) -> None:
+    if id is None and not checkIfCoretexRepoExists():
+        id = ui.clickPrompt(f"There is no existing Task repository. Please specify id of Task you want to pull:", type = int)
+
+    if id is not None:
+        try:
+            task = Task.fetchById(id)
+        except NetworkRequestError as ex:
+            ui.errorEcho(f"Failed to fetch Task id {id}. Reason: {ex.response}")
+            return
+
+    if not task.coretexMetadataPath.exists():
+        task.pull()
+        task.createMetadata()
+        task.fillMetadata()
+    else:
+        differences = task.getDiff()
+        if len(differences) == 0:
+            ui.stdEcho("Your repository is already updated.")
+            return
+
+        ui.stdEcho("There are conflicts between your and remote repository.")
+        for diff in differences:
+            ui.stdEcho(f"File: {diff['path']} differs")
+            ui.stdEcho(f"\tLocal checksum:  {diff['local_checksum']}")
+            ui.stdEcho(f"\tRemote checksum: {diff['remote_checksum']}")
+
+        if not ui.clickPrompt("Do you want to pull the changes and update your local repository? (Y/n):", type = bool, default = True):
+            ui.stdEcho("No changes were made to your local repository.")
+            return
+
+        task.pull()
+        task.createMetadata()
+        task.fillMetadata()
+        ui.stdEcho("Repository updated successfully.")
+
+
 @click.group()
 @onBeforeCommandExecute(initializeUserSession)
 def task() -> None:
     pass
 
 
 task.add_command(run, "run")
+task.add_command(pull, "pull")

coretex/entities/project/task.py

@@ -20,10 +20,11 @@
 
 from .base import BaseObject
 from ..utils import isEntityNameValid
+from ..repository import CoretexRepository, EntityCoretexRepositoryType
 from ...codable import KeyDescriptor
 
 
-class Task(BaseObject):
+class Task(BaseObject, CoretexRepository):
 
     """
         Represents the task entity from Coretex.ai\n
@@ -33,6 +34,18 @@ class Task(BaseObject):
     isDefault: bool
     taskId: int
 
+    @property
+    def entityCoretexRepositoryType(self) -> EntityCoretexRepositoryType:
+        return EntityCoretexRepositoryType.task
+
+    @property
+    def paramKey(self) -> str:
+        return "sub_project_id"
+
+    @property
+    def endpoint(self) -> str:
+        return "workspace"
+
     @classmethod
     def _keyDescriptors(cls) -> Dict[str, KeyDescriptor]:
         descriptors = super()._keyDescriptors()

coretex/entities/repository/__init__.py

@@ -0,0 +1 @@
+from .coretex_repository import CoretexRepository, EntityCoretexRepositoryType, checkIfCoretexRepoExists

coretex/entities/repository/coretex_repository.py

@@ -0,0 +1,170 @@
+#     Copyright (C) 2023  Coretex LLC
+
+#     This file is part of Coretex.ai
+
+#     This program is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU Affero General Public License as
+#     published by the Free Software Foundation, either version 3 of the
+#     License, or (at your option) any later version.
+
+#     This program is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU Affero General Public License for more details.
+
+#     You should have received a copy of the GNU Affero General Public License
+#     along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from typing import List, Dict, Any
+from enum import IntEnum
+from abc import abstractmethod, ABC
+from pathlib import Path
+from zipfile import ZipFile
+
+import os
+import json
+import logging
+
+from ...codable import Codable
+from ...utils import generateSha256Checksum
+from ...networking import networkManager, NetworkRequestError
+
+
+def checkIfCoretexRepoExists() -> bool:
+    print(Path.cwd().joinpath(".coretex"))
+    return Path.cwd().joinpath(".coretex").exists()
+
+
+class EntityCoretexRepositoryType(IntEnum):
+
+    task   = 1
+
+
+class CoretexRepository(ABC, Codable):
+
+    id: int
+    projectId: int
+
+    @property
+    def __initialMetadataPath(self) -> Path:
+        return Path(f"{self.id}/.metadata.json")
+
+    @property
+    def coretexMetadataPath(self) -> Path:
+        return Path(f"{self.id}/.coretex.json")
+
+    @property
+    @abstractmethod
+    def entityCoretexRepositoryType(self) -> EntityCoretexRepositoryType:
+        pass
+
+    @property
+    @abstractmethod
+    def paramKey(self) -> str:
+        pass
+
+    @property
+    @abstractmethod
+    def endpoint(self) -> str:
+        pass
+
+    def pull(self) -> bool:
+        params = {
+            self.paramKey: self.id
+        }
+
+        zipFilePath = f"{self.id}.zip"
+        response = networkManager.download(f"{self.endpoint}/download", zipFilePath, params)
+
+        if response.hasFailed():
+            logging.getLogger("coretexpylib").error(f">> [Coretex] {self.entityCoretexRepositoryType.name.capitalize()} download has failed")
+            return False
+
+        with ZipFile(zipFilePath) as zipFile:
+            zipFile.extractall(str(self.id))
+
+        os.unlink(zipFilePath)
+
+        return not response.hasFailed()
+
+    def getRemoteMetadata(self) -> List:
+        # getRemoteMetadata downloads only .metadata file, this was needed so we can
+        # synchronize changes if multiple people work on the same Entity at the same time
+
+        params = {
+            self.paramKey: self.id
+        }
+
+        response = networkManager.get(f"{self.endpoint}/metadata", params)
+        if response.hasFailed():
+            raise NetworkRequestError(response, "Failed to fetch task metadata.")
+
+        return response.getJson(list, force = True)
+
+    def createMetadata(self) -> None:
+        # createMetadata() function will store metadata of files that backend returns
+        # currently all files on backend (that weren't uploaded after checksum calculation change
+        # for files that is implemented recently on backend) return null/None for their checksum
+        # if backend returns None for checksum of some file we need to calculate initial checksum of
+        # the file so we can track changes
+
+        with self.__initialMetadataPath.open("r") as initialMetadataFile:
+            initialMetadata = json.load(initialMetadataFile)
+
+            # if backend returns null for checksum of file, generate checksum
+            for file in initialMetadata:
+                if file["checksum"] is None:
+                    filePath = self.__initialMetadataPath.parent.joinpath(file["path"])
+                    if filePath.exists():
+                        file["checksum"] = generateSha256Checksum(filePath)
+
+        newMetadata = {
+            "checksums": initialMetadata
+        }
+
+        with self.coretexMetadataPath.open("w") as coretexMetadataFile:
+            json.dump(newMetadata, coretexMetadataFile, indent = 4)
+
+    def fillMetadata(self) -> None:
+        # fillMetadata() function will update initial metadata returned from backend
+        # (file paths and their checksums) with other important Entity info (e.g. name, id, description...)
+
+        localMetadata: Dict[str, Any] = {}
+        metadata = self.encode()
+
+        with self.coretexMetadataPath.open("r") as coretexMetadataFile:
+            localMetadata = json.load(coretexMetadataFile)
+
+        localMetadata.update(metadata)
+
+        with self.coretexMetadataPath.open("w") as coretexMetadataFile:
+            json.dump(localMetadata, coretexMetadataFile, indent = 4)
+
+        self.__initialMetadataPath.unlink()
+
+    def getDiff(self) ->  List[Dict[str, Any]]:
+        # getDiff() checks initial checksums of files stored in .coretex file
+        # and compares them with their current checksums, based on that we know what files have changes
+
+        with self.coretexMetadataPath.open("r") as localMetadataFile:
+            localMetadata = json.load(localMetadataFile)
+
+        localChecksums = {file["path"]: file["checksum"] for file in localMetadata["checksums"]}
+
+        differences = []
+
+        remoteMetadata = self.getRemoteMetadata()
+        for remoteFile in remoteMetadata:
+            remotePath = remoteFile["path"]
+            remoteChecksum = remoteFile["checksum"]
+
+            localChecksum = localChecksums.get(remotePath)
+
+            if localChecksum != remoteChecksum:
+                differences.append({
+                    "path": remotePath,
+                    "local_checksum": localChecksum,
+                    "remote_checksum": remoteChecksum
+                })
+
+        return differences

coretex/entities/task_run/task_run.py

@@ -17,7 +17,7 @@
 
 from typing import Optional, Any, List, Dict, Union, Tuple, TypeVar, Generic, Type
 from typing_extensions import Self, override
-from zipfile import ZipFile, ZIP_DEFLATED
+from zipfile import ZipFile
 from pathlib import Path
 
 import os

coretex/networking/network_response.py

@@ -90,7 +90,7 @@ def isHead(self) -> bool:
 
         return self._raw.request.method == RequestType.head.value
 
-    def getJson(self, type_: Type[JsonType]) -> JsonType:
+    def getJson(self, type_: Type[JsonType], force: bool = False) -> JsonType:
         """
             Converts HTTP response body to json
 
@@ -109,7 +109,7 @@ def getJson(self, type_: Type[JsonType]) -> JsonType:
             TypeError -> If it was not possible to convert body to type of passed "type_" parameter
         """
 
-        if not "application/json" in self.headers.get("Content-Type", ""):
+        if not force and not "application/json" in self.headers.get("Content-Type", ""):
             raise ValueError(f">> [Coretex] Trying to convert request response to json but response \"Content-Type\" was \"{self.headers.get('Content-Type')}\"")
 
         value = self._raw.json()

coretex/utils/__init__.py

@@ -22,5 +22,5 @@
 from .image import resizeWithPadding, cropToWidth
 from .process import logProcessOutput, command, CommandException
 from .logs import createFileHandler
-from .misc import isCliRuntime
+from .misc import isCliRuntime, generateSha256Checksum
 from .error_handling import Throws

coretex/utils/misc.py

@@ -15,8 +15,11 @@
 #     You should have received a copy of the GNU Affero General Public License
 #     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+from pathlib import Path
+
 import os
 import sys
+import hashlib
 
 
 def isCliRuntime() -> bool:
@@ -25,3 +28,14 @@ def isCliRuntime() -> bool:
         executablePath.endswith("/bin/coretex") and
         os.access(executablePath, os.X_OK)
     )
+
+
+def generateSha256Checksum(path: Path) -> str:
+    sha256 = hashlib.sha256()
+    chunkSize = 64 * 1024  # 65536 bytes
+
+    with path.open("rb") as file:
+        while chunk := file.read(chunkSize):
+            sha256.update(chunk)
+
+    return sha256.hexdigest()