From e5e8a21d5883fd178702c8b59083eb175dea7159 Mon Sep 17 00:00:00 2001 From: Vuk Manojlovic Date: Mon, 15 Jul 2024 13:30:15 +0200 Subject: [PATCH 01/37] CT-5524: Added tag management functions to Model and NetworkDataset --- coretex/entities/dataset/network_dataset.py | 44 ++++++++- coretex/entities/model/model.py | 44 ++++++++- coretex/entities/tag.py | 99 +++++++++++++++++++++ 3 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 coretex/entities/tag.py diff --git a/coretex/entities/dataset/network_dataset.py b/coretex/entities/dataset/network_dataset.py index 413e3766..20934e75 100644 --- a/coretex/entities/dataset/network_dataset.py +++ b/coretex/entities/dataset/network_dataset.py @@ -28,6 +28,7 @@ from .dataset import Dataset from .state import DatasetState +from ..tag import EntityTagType, Taggable from ..sample import NetworkSample from ..utils import isEntityNameValid from ... import folder_manager @@ -91,7 +92,7 @@ def _encryptedSampleImport(sampleType: Type[SampleType], sampleName: str, sample raise RuntimeError("Unreachable statement was reached.") -class NetworkDataset(Generic[SampleType], Dataset[SampleType], NetworkObject, ABC): +class NetworkDataset(Generic[SampleType], Dataset[SampleType], NetworkObject, ABC, Taggable): """ Represents the base class for all Dataset classes which are @@ -389,3 +390,44 @@ def add(self, samplePath: Union[Path, str], sampleName: Optional[str] = None, ** self.samples.append(sample) return sample + + def addTag(self, tag: str, color: Optional[str] = None) -> None: + """ + Add a tag to this dataset + + Parameters + ---------- + tag : str + name of the tag + color : Optional[str] + a hexadecimal color code for the new tag\n + if tag already exists in project, this will be ignored\n + if left empty and tag does not already exist, a random color will be picked + + Raises + ------ + ValueError + if tag name or color are invalid + NetworkRequestError + if request to add tag failed + """ + + self.addTagToEntity(tag, self.id, self.projectId, EntityTagType.dataset, color) + + + def removeTag(self, tag: str) -> None: + """ + Remove tag with provided name from the dataset + + Parameters + ---------- + tag : str + name of the tag + + Raises + ------ + NetworkRequestError + if tag removal request failed + """ + + self.removeTagFromEntity(tag, self.id, self.projectId, EntityTagType.dataset) diff --git a/coretex/entities/model/model.py b/coretex/entities/model/model.py index 9c8cfad2..95fa9c25 100644 --- a/coretex/entities/model/model.py +++ b/coretex/entities/model/model.py @@ -24,13 +24,14 @@ import json import logging +from ..tag import Taggable, EntityTagType from ..utils import isEntityNameValid from ... import folder_manager from ...networking import networkManager, NetworkObject, ChunkUploadSession, MAX_CHUNK_SIZE, NetworkRequestError from ...codable import KeyDescriptor -class Model(NetworkObject): +class Model(NetworkObject, Taggable): """ Represents a machine learning model object on Coretex.ai @@ -282,3 +283,44 @@ def upload(self, path: Union[Path, str]) -> None: response = networkManager.formData("model/upload", parameters) if response.hasFailed(): raise NetworkRequestError(response, "Failed to upload model") + + def addTag(self, tag: str, color: Optional[str] = None) -> None: + """ + Add a tag to this model + + Parameters + ---------- + tag : str + name of the tag + color : Optional[str] + a hexadecimal color code for the new tag\n + if tag already exists in project, this will be ignored\n + if left empty and tag does not already exist, a random color will be picked + + Raises + ------ + ValueError + if tag name or color are invalid + NetworkRequestError + if request to add tag failed + """ + + self.addTagToEntity(tag, self.id, self.projectId, EntityTagType.model, color) + + + def removeTag(self, tag: str) -> None: + """ + Remove tag with provided name from the model + + Parameters + ---------- + tag : str + name of the tag + + Raises + ------ + NetworkRequestError + if tag removal request failed + """ + + self.removeTagFromEntity(tag, self.id, self.projectId, EntityTagType.model) diff --git a/coretex/entities/tag.py b/coretex/entities/tag.py new file mode 100644 index 00000000..c5f869ef --- /dev/null +++ b/coretex/entities/tag.py @@ -0,0 +1,99 @@ +# Copyright (C) 2023 Coretex LLC + +# This file is part of Coretex.ai + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from enum import IntEnum +from typing import Optional + +import re +import random +import logging + +from ..networking import networkManager, NetworkRequestError + + +class EntityTagType(IntEnum): + + model = 1 + dataset = 2 + + +class Taggable: + + def _getTagId(self, tagName: str, projectId: int, entityTypeEnum: int) -> Optional[int]: + parameters = { + "name": tagName, + "type": int(entityTypeEnum), + "project_id": projectId, + } + + response = networkManager.get("tag", parameters) + if response.hasFailed(): + raise NetworkRequestError(response, "Failed to check existing tags") + + tags = response.getJson(dict)["data"] + if len(tags) == 0: + return None + + tagId: int = tags[0]["id"] + + return tagId + + + def addTagToEntity(self, tag: str, entityId: int, projectId: int, entityTypeEnum: int, color: Optional[str] = None) -> None: + if re.match(r"^[a-z0-9-]{1,10}$", tag) is None: + raise ValueError(">> [Coretex] Tag has to be alphanumeric") + + if color is None: + color = f"#{random.randint(0, 0xFFFFFF):06x}" + else: + if re.match(r'^#([A-Fa-f0-9]{3}|[A-Fa-f0-9]{6})$', color) is None: + raise ValueError(">> [Coretex] Tag color has to follow hexadecimal color code") + + parameters = { + "entity_id": entityId, + "type": entityTypeEnum, + "tags": {} + } + + tagId = self._getTagId(tag, projectId, entityTypeEnum) + if tagId is not None: + parameters["tags"]["existing"] = [tagId] # type: ignore[index] + else: + parameters["tags"]["new"] = [{ # type: ignore[index] + "name": tag, + "color": color + }] + + response = networkManager.post("tag/entity", parameters) + if response.hasFailed(): + raise NetworkRequestError(response, "Failed to create tag") + + def removeTagFromEntity(self, tag: str, entityId: int, projectId: int, entityTypeEnum: int) -> None: + tagId = self._getTagId(tag, projectId, entityTypeEnum) + if tagId is None: + logging.error(f">> [Coretex] Tag \"{tag}\" not found on entity id {entityId}") + return + + parameters = { + "entity_id": entityId, + "tag_id": tagId, + "type": entityTypeEnum + } + + response = networkManager.post("tag/remove", parameters) + if response.hasFailed(): + raise NetworkRequestError(response, "Failed to remove tag") From e2e2b6caca9768a8e78e3bfba781a25b57166ecf Mon Sep 17 00:00:00 2001 From: Vuk Manojlovic Date: Tue, 16 Jul 2024 11:33:41 +0200 Subject: [PATCH 02/37] CTX-5524: Improved tag code via abstract class --- coretex/entities/dataset/network_dataset.py | 45 +---------- coretex/entities/model/model.py | 47 ++--------- coretex/entities/tag.py | 89 ++++++++++++++++----- 3 files changed, 77 insertions(+), 104 deletions(-) diff --git a/coretex/entities/dataset/network_dataset.py b/coretex/entities/dataset/network_dataset.py index 20934e75..b636f168 100644 --- a/coretex/entities/dataset/network_dataset.py +++ b/coretex/entities/dataset/network_dataset.py @@ -92,7 +92,7 @@ def _encryptedSampleImport(sampleType: Type[SampleType], sampleName: str, sample raise RuntimeError("Unreachable statement was reached.") -class NetworkDataset(Generic[SampleType], Dataset[SampleType], NetworkObject, ABC, Taggable): +class NetworkDataset(Generic[SampleType], Dataset[SampleType], Taggable, ABC): """ Represents the base class for all Dataset classes which are @@ -391,43 +391,6 @@ def add(self, samplePath: Union[Path, str], sampleName: Optional[str] = None, ** return sample - def addTag(self, tag: str, color: Optional[str] = None) -> None: - """ - Add a tag to this dataset - - Parameters - ---------- - tag : str - name of the tag - color : Optional[str] - a hexadecimal color code for the new tag\n - if tag already exists in project, this will be ignored\n - if left empty and tag does not already exist, a random color will be picked - - Raises - ------ - ValueError - if tag name or color are invalid - NetworkRequestError - if request to add tag failed - """ - - self.addTagToEntity(tag, self.id, self.projectId, EntityTagType.dataset, color) - - - def removeTag(self, tag: str) -> None: - """ - Remove tag with provided name from the dataset - - Parameters - ---------- - tag : str - name of the tag - - Raises - ------ - NetworkRequestError - if tag removal request failed - """ - - self.removeTagFromEntity(tag, self.id, self.projectId, EntityTagType.dataset) + @property + def entityTagType(self) -> int: + return EntityTagType.dataset diff --git a/coretex/entities/model/model.py b/coretex/entities/model/model.py index 95fa9c25..48622bc8 100644 --- a/coretex/entities/model/model.py +++ b/coretex/entities/model/model.py @@ -27,11 +27,11 @@ from ..tag import Taggable, EntityTagType from ..utils import isEntityNameValid from ... import folder_manager -from ...networking import networkManager, NetworkObject, ChunkUploadSession, MAX_CHUNK_SIZE, NetworkRequestError +from ...networking import networkManager, ChunkUploadSession, MAX_CHUNK_SIZE, NetworkRequestError from ...codable import KeyDescriptor -class Model(NetworkObject, Taggable): +class Model(Taggable): """ Represents a machine learning model object on Coretex.ai @@ -284,43 +284,6 @@ def upload(self, path: Union[Path, str]) -> None: if response.hasFailed(): raise NetworkRequestError(response, "Failed to upload model") - def addTag(self, tag: str, color: Optional[str] = None) -> None: - """ - Add a tag to this model - - Parameters - ---------- - tag : str - name of the tag - color : Optional[str] - a hexadecimal color code for the new tag\n - if tag already exists in project, this will be ignored\n - if left empty and tag does not already exist, a random color will be picked - - Raises - ------ - ValueError - if tag name or color are invalid - NetworkRequestError - if request to add tag failed - """ - - self.addTagToEntity(tag, self.id, self.projectId, EntityTagType.model, color) - - - def removeTag(self, tag: str) -> None: - """ - Remove tag with provided name from the model - - Parameters - ---------- - tag : str - name of the tag - - Raises - ------ - NetworkRequestError - if tag removal request failed - """ - - self.removeTagFromEntity(tag, self.id, self.projectId, EntityTagType.model) + @property + def entityTagType(self) -> int: + return EntityTagType.model diff --git a/coretex/entities/tag.py b/coretex/entities/tag.py index c5f869ef..adce39ce 100644 --- a/coretex/entities/tag.py +++ b/coretex/entities/tag.py @@ -16,13 +16,14 @@ # along with this program. If not, see . from enum import IntEnum -from typing import Optional +from typing import Optional, Dict, Any +from abc import abstractmethod import re import random import logging -from ..networking import networkManager, NetworkRequestError +from ..networking import networkManager, NetworkRequestError, NetworkObject class EntityTagType(IntEnum): @@ -31,13 +32,23 @@ class EntityTagType(IntEnum): dataset = 2 -class Taggable: +class Taggable(NetworkObject): - def _getTagId(self, tagName: str, projectId: int, entityTypeEnum: int) -> Optional[int]: + @property + @abstractmethod + def projectId(self) -> int: + pass + + @property + @abstractmethod + def entityTagType(self) -> int: + pass + + def _getTagId(self, tagName: str) -> Optional[int]: parameters = { "name": tagName, - "type": int(entityTypeEnum), - "project_id": projectId, + "type": int(self.entityTagType), + "project_id": self.projectId, } response = networkManager.get("tag", parameters) @@ -53,8 +64,28 @@ def _getTagId(self, tagName: str, projectId: int, entityTypeEnum: int) -> Option return tagId - def addTagToEntity(self, tag: str, entityId: int, projectId: int, entityTypeEnum: int, color: Optional[str] = None) -> None: - if re.match(r"^[a-z0-9-]{1,10}$", tag) is None: + def addTag(self, tag: str, color: Optional[str] = None) -> None: + """ + Add a tag to this entity + + Parameters + ---------- + tag : str + name of the tag + color : Optional[str] + a hexadecimal color code for the new tag\n + if tag already exists in project, this will be ignored\n + if left empty and tag does not already exist, a random color will be picked + + Raises + ------ + ValueError + if tag name or color are invalid + NetworkRequestError + if request to add tag failed + """ + + if re.match(r"^[a-z0-9-]{1,30}$", tag) is None: raise ValueError(">> [Coretex] Tag has to be alphanumeric") if color is None: @@ -63,35 +94,51 @@ def addTagToEntity(self, tag: str, entityId: int, projectId: int, entityTypeEnum if re.match(r'^#([A-Fa-f0-9]{3}|[A-Fa-f0-9]{6})$', color) is None: raise ValueError(">> [Coretex] Tag color has to follow hexadecimal color code") - parameters = { - "entity_id": entityId, - "type": entityTypeEnum, - "tags": {} - } + tags: Dict[str, Any] = {} - tagId = self._getTagId(tag, projectId, entityTypeEnum) + tagId = self._getTagId(tag) if tagId is not None: - parameters["tags"]["existing"] = [tagId] # type: ignore[index] + tags["existing"] = [tagId] else: - parameters["tags"]["new"] = [{ # type: ignore[index] + tags["new"] = [{ "name": tag, "color": color }] + parameters = { + "entity_id": self.id, + "type": self.entityTagType, + "tags": tags + } + response = networkManager.post("tag/entity", parameters) if response.hasFailed(): raise NetworkRequestError(response, "Failed to create tag") - def removeTagFromEntity(self, tag: str, entityId: int, projectId: int, entityTypeEnum: int) -> None: - tagId = self._getTagId(tag, projectId, entityTypeEnum) + def removeTag(self, tag: str) -> None: + """ + Remove tag with provided name from the model + + Parameters + ---------- + tag : str + name of the tag + + Raises + ------ + NetworkRequestError + if tag removal request failed + """ + + tagId = self._getTagId(tag) if tagId is None: - logging.error(f">> [Coretex] Tag \"{tag}\" not found on entity id {entityId}") + logging.error(f">> [Coretex] Tag \"{tag}\" not found on entity id {self.id}") return parameters = { - "entity_id": entityId, + "entity_id": self.id, "tag_id": tagId, - "type": entityTypeEnum + "type": self.entityTagType } response = networkManager.post("tag/remove", parameters) From 68c1df336f300c3bd264b5b86fff51d81515088a Mon Sep 17 00:00:00 2001 From: Dusko Vesic Date: Tue, 16 Jul 2024 14:42:00 +0200 Subject: [PATCH 03/37] Documentation update for createCacheDataset() --- coretex/entities/dataset/network_dataset.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/coretex/entities/dataset/network_dataset.py b/coretex/entities/dataset/network_dataset.py index 413e3766..7fad979d 100644 --- a/coretex/entities/dataset/network_dataset.py +++ b/coretex/entities/dataset/network_dataset.py @@ -271,6 +271,20 @@ def createCacheDataset(cls, prefix: str, dependencies: List[str], projectId: int parameters which affect the contents of the cache projectId : int project for which the dataset will be created + + Returns + ------- + The created dataset object + + Raises + ------ + ValueError -> If prefix of the name is invalid or if failed to create cache dataset + + Example + ------- + >>> from coretex import NetworkDataset + \b + >>> dummyDataset = NetworkDataset.createDataset("dummyDataset", dependencies, 123) """ if not isEntityNameValid(prefix): From 753ce33ddba8225ab3dccdb9c66b9b18bcbe4e39 Mon Sep 17 00:00:00 2001 From: Vuk Manojlovic Date: Tue, 16 Jul 2024 17:10:06 +0200 Subject: [PATCH 04/37] CTX-5524: Added srs retrieval to runOnnxInference --- coretex/utils/inference.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/coretex/utils/inference.py b/coretex/utils/inference.py index 8a6f1058..b9b9a620 100644 --- a/coretex/utils/inference.py +++ b/coretex/utils/inference.py @@ -16,12 +16,16 @@ async def genWitness(inputPath: Path, circuit: Path, witnessPath: Path) -> None: await ezkl.gen_witness(inputPath, circuit, witnessPath) +async def getSrs(settings: Path): + await ezkl.get_srs(settings) + def runOnnxInference( data: np.ndarray, onnxPath: Path, compiledModelPath: Optional[Path] = None, - proveKey: Optional[Path] = None + proveKey: Optional[Path] = None, + settingsPath: Optional[Path] = None, ) -> Union[np.ndarray, Tuple[np.ndarray, Path]]: """ @@ -36,6 +40,8 @@ def runOnnxInference( data which will be directly fed to the model onnxPath : Path path to the onnx model + settingsPath : Path + path to the settigs.json file compiledModelPath : Optional[Path] path to the compiled model proveKey : Optional[Path] @@ -53,11 +59,11 @@ def runOnnxInference( inputName = session.get_inputs()[0].name result = np.array(session.run(None, {inputName: data})) - if compiledModelPath is None and proveKey is None: + if compiledModelPath is None and proveKey is None and settingsPath is None: return result - if compiledModelPath is None or proveKey is None: - raise ValueError(f">> [Coretex] Parameters compiledModelPath and proveKey have to either both be passed or None") + if compiledModelPath is None or proveKey is None or settingsPath is None: + raise ValueError(f">> [Coretex] Parameters compiledModelPath, proveKey and settingsPath have to either all be passed (for verified inference) or none of them (for regula inference)") inferenceDir = folder_manager.createTempFolder(inferenceId) witnessPath = inferenceDir / "witness.json" @@ -70,6 +76,7 @@ def runOnnxInference( json.dump(inputData, file) asyncio.run(genWitness(inputPath, compiledModelPath, witnessPath)) + asyncio.run(getSrs(settingsPath)) ezkl.prove( witnessPath, compiledModelPath, From 7a8f822238672a031eaa183309f4fbf394728633 Mon Sep 17 00:00:00 2001 From: Vuk Manojlovic Date: Wed, 17 Jul 2024 09:50:20 +0200 Subject: [PATCH 05/37] CTX-5524: Added return type --- coretex/utils/inference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coretex/utils/inference.py b/coretex/utils/inference.py index b9b9a620..19b0933a 100644 --- a/coretex/utils/inference.py +++ b/coretex/utils/inference.py @@ -16,7 +16,7 @@ async def genWitness(inputPath: Path, circuit: Path, witnessPath: Path) -> None: await ezkl.gen_witness(inputPath, circuit, witnessPath) -async def getSrs(settings: Path): +async def getSrs(settings: Path) -> None: await ezkl.get_srs(settings) From 4c0bc72b9b2cc3db6f0082d7e03efec5b490f167 Mon Sep 17 00:00:00 2001 From: Vuk Manojlovic Date: Wed, 17 Jul 2024 10:13:13 +0200 Subject: [PATCH 06/37] CTX-5524: Improved inheritance structure --- coretex/entities/dataset/network_dataset.py | 10 +++++----- coretex/entities/model/model.py | 12 ++++++------ coretex/entities/tag.py | 12 +++++------- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/coretex/entities/dataset/network_dataset.py b/coretex/entities/dataset/network_dataset.py index b636f168..645cd6a6 100644 --- a/coretex/entities/dataset/network_dataset.py +++ b/coretex/entities/dataset/network_dataset.py @@ -92,7 +92,7 @@ def _encryptedSampleImport(sampleType: Type[SampleType], sampleName: str, sample raise RuntimeError("Unreachable statement was reached.") -class NetworkDataset(Generic[SampleType], Dataset[SampleType], Taggable, ABC): +class NetworkDataset(Generic[SampleType], Dataset[SampleType], NetworkObject, Taggable, ABC): """ Represents the base class for all Dataset classes which are @@ -130,6 +130,10 @@ def path(self) -> Path: return folder_manager.datasetsFolder / str(self.id) + @property + def entityTagType(self) -> int: + return EntityTagType.dataset + # Codable overrides @classmethod @@ -390,7 +394,3 @@ def add(self, samplePath: Union[Path, str], sampleName: Optional[str] = None, ** self.samples.append(sample) return sample - - @property - def entityTagType(self) -> int: - return EntityTagType.dataset diff --git a/coretex/entities/model/model.py b/coretex/entities/model/model.py index 48622bc8..5812dbf4 100644 --- a/coretex/entities/model/model.py +++ b/coretex/entities/model/model.py @@ -27,11 +27,11 @@ from ..tag import Taggable, EntityTagType from ..utils import isEntityNameValid from ... import folder_manager -from ...networking import networkManager, ChunkUploadSession, MAX_CHUNK_SIZE, NetworkRequestError +from ...networking import networkManager, NetworkObject, ChunkUploadSession, MAX_CHUNK_SIZE, NetworkRequestError from ...codable import KeyDescriptor -class Model(Taggable): +class Model(NetworkObject, Taggable): """ Represents a machine learning model object on Coretex.ai @@ -83,6 +83,10 @@ def path(self) -> Path: def zipPath(self) -> Path: return self.path.with_suffix(".zip") + @property + def entityTagType(self) -> int: + return EntityTagType.model + @classmethod def modelDescriptorFileName(cls) -> str: """ @@ -283,7 +287,3 @@ def upload(self, path: Union[Path, str]) -> None: response = networkManager.formData("model/upload", parameters) if response.hasFailed(): raise NetworkRequestError(response, "Failed to upload model") - - @property - def entityTagType(self) -> int: - return EntityTagType.model diff --git a/coretex/entities/tag.py b/coretex/entities/tag.py index adce39ce..697c0d2e 100644 --- a/coretex/entities/tag.py +++ b/coretex/entities/tag.py @@ -17,13 +17,13 @@ from enum import IntEnum from typing import Optional, Dict, Any -from abc import abstractmethod +from abc import abstractmethod, ABC import re import random import logging -from ..networking import networkManager, NetworkRequestError, NetworkObject +from ..networking import networkManager, NetworkRequestError class EntityTagType(IntEnum): @@ -32,12 +32,10 @@ class EntityTagType(IntEnum): dataset = 2 -class Taggable(NetworkObject): +class Taggable(ABC): - @property - @abstractmethod - def projectId(self) -> int: - pass + id: int + projectId: int @property @abstractmethod From 6c49e66862e4b217205b3b6dd37aab2c2148d8e1 Mon Sep 17 00:00:00 2001 From: Vuk Manojlovic Date: Thu, 18 Jul 2024 10:54:23 +0200 Subject: [PATCH 07/37] CTX-5524: Implemented requested changes --- coretex/entities/tag.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/coretex/entities/tag.py b/coretex/entities/tag.py index 697c0d2e..46a48d41 100644 --- a/coretex/entities/tag.py +++ b/coretex/entities/tag.py @@ -46,18 +46,26 @@ def _getTagId(self, tagName: str) -> Optional[int]: parameters = { "name": tagName, "type": int(self.entityTagType), - "project_id": self.projectId, + "project_id": self.projectId } response = networkManager.get("tag", parameters) if response.hasFailed(): raise NetworkRequestError(response, "Failed to check existing tags") - tags = response.getJson(dict)["data"] + tags = response.getJson(dict).get("data") + if not isinstance(tags, dict): + raise NetworkRequestError(response, f"Field \"data\" from tag response must be dict, but got {type(tags)} instead") + if len(tags) == 0: return None - tagId: int = tags[0]["id"] + if not isinstance(tags[0], dict): + raise NetworkRequestError(response, f"Tag object from response must be dict, but got {type(tags[0])} instead") + + tagId = tags[0].get("id") + if not isinstance(tagId, int): + raise NetworkRequestError(response, f"Tag object from response must have field id of type int, but got {type(tagId)} instead") return tagId From 7581337edd73c75c13fd2a5f79f5c9b02318f549 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Tue, 23 Jul 2024 11:03:58 +0200 Subject: [PATCH 08/37] CTX-6421: Fixed bug with temp folder not being cleaned after run is finished. Remove unused "os" import. --- coretex/folder_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/coretex/folder_manager.py b/coretex/folder_manager.py index 002a4c9a..661c9779 100644 --- a/coretex/folder_manager.py +++ b/coretex/folder_manager.py @@ -19,12 +19,11 @@ from pathlib import Path from contextlib import contextmanager -import os import shutil import uuid from .utils import file as file_utils - +from .configuration import loadConfig """ Used for handling everything related to local storage @@ -59,8 +58,9 @@ def _createFolder(name: str) -> Path: return path - -_root = Path(os.environ["CTX_STORAGE_PATH"]).expanduser() +config = loadConfig() +storagePath = config["storagePath"] # this will never be None since _syncConfigWithEnv() will happen before folder_manager initialization +_root = Path(storagePath).expanduser() samplesFolder = _createFolder("samples") modelsFolder = _createFolder("models") From 80ed667196bbd7c8b441342b2340753474415968 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Tue, 23 Jul 2024 11:05:56 +0200 Subject: [PATCH 09/37] CTX-6421: Updated comment. --- coretex/folder_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coretex/folder_manager.py b/coretex/folder_manager.py index 661c9779..a9cd75da 100644 --- a/coretex/folder_manager.py +++ b/coretex/folder_manager.py @@ -59,7 +59,7 @@ def _createFolder(name: str) -> Path: return path config = loadConfig() -storagePath = config["storagePath"] # this will never be None since _syncConfigWithEnv() will happen before folder_manager initialization +storagePath = config["storagePath"] # value of key storagePath in config dictionary is guaranteed to never be None _root = Path(storagePath).expanduser() samplesFolder = _createFolder("samples") From fced7d8c6bab33a032ef9ba107a50e9d4dbc1192 Mon Sep 17 00:00:00 2001 From: Vuk Manojlovic Date: Tue, 23 Jul 2024 11:39:40 +0200 Subject: [PATCH 10/37] CTX-5524: Fixed removeTag documentation --- coretex/entities/tag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coretex/entities/tag.py b/coretex/entities/tag.py index 46a48d41..eeafa83c 100644 --- a/coretex/entities/tag.py +++ b/coretex/entities/tag.py @@ -123,7 +123,7 @@ def addTag(self, tag: str, color: Optional[str] = None) -> None: def removeTag(self, tag: str) -> None: """ - Remove tag with provided name from the model + Remove tag with provided name from the entity Parameters ---------- From 7a526fd021e3cdbd4f37d747c5efdeab44e6e898 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Tue, 23 Jul 2024 16:08:31 +0200 Subject: [PATCH 11/37] CTX-6421: Implementation of discussed folder_manager module refactor. --- coretex/__init__.py | 2 +- coretex/_folder_manager.py | 138 ++++++++++++ coretex/_logger.py | 2 +- coretex/_task/base_callback.py | 2 +- coretex/_task/initialization.py | 2 +- coretex/_task/worker/utils.py | 2 +- coretex/bioinformatics/ctx_qiime2/utils.py | 2 +- coretex/cache.py | 2 +- coretex/cli/commands/task.py | 8 +- .../synthetic_image_generator.py | 2 +- coretex/entities/dataset/network_dataset.py | 2 +- .../sequence_dataset/sequence_dataset.py | 2 +- coretex/entities/model/model.py | 2 +- coretex/entities/sample/network_sample.py | 2 +- coretex/entities/task_run/artifact.py | 2 +- coretex/entities/task_run/task_run.py | 2 +- coretex/entities/task_run/utils.py | 2 +- coretex/folder_manager.py | 196 ------------------ coretex/utils/inference.py | 2 +- 19 files changed, 159 insertions(+), 215 deletions(-) create mode 100644 coretex/_folder_manager.py delete mode 100644 coretex/folder_manager.py diff --git a/coretex/__init__.py b/coretex/__init__.py index 079d56dd..e76f049f 100644 --- a/coretex/__init__.py +++ b/coretex/__init__.py @@ -24,7 +24,6 @@ from ._logger import _initializeDefaultLogger, _initializeCLILogger from .configuration import isCliRuntime - if isCliRuntime(): _initializeCLILogger() else: @@ -34,3 +33,4 @@ # Use this only from .entities import * from ._task import currentTaskRun, initializeRTask, TaskRunWorker +from ._folder_manager import folder_manager diff --git a/coretex/_folder_manager.py b/coretex/_folder_manager.py new file mode 100644 index 00000000..20f23622 --- /dev/null +++ b/coretex/_folder_manager.py @@ -0,0 +1,138 @@ +# Copyright (C) 2023 Coretex LLC + +# This file is part of Coretex.ai + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from typing import Iterator, Optional, Union +from pathlib import Path +from contextlib import contextmanager + +import os +import shutil +import uuid + +from .utils import file as file_utils + + +class FolderManager: + + """ + Used for handling everything related to local storage + when working with Coretex + + Contains + -------- + samplesFolder : str + folder where samples are stored + modelsFolder : str + folder where models are stored + temp : str + folder where temp files and folders are stored, + this is deleted when the run has finished executing + + datasetsFolder : Path + folder where datasets are stored (samples are symlinked for datasets) + cache : Path + folder where cache module stores items + logs : Path + folder where node and run logs are stored + environments : Path + folder where node stores python environments + """ + + def __init__(self, storagePath: Union[Path, str]): + if isinstance(storagePath, str): + storagePath = Path(storagePath) + + self._root = storagePath.expanduser() + + self.samplesFolder = self._createFolder("samples") + self.modelsFolder = self._createFolder("models") + self.datasetsFolder = self._createFolder("datasets") + self.cache = self._createFolder("cache") + self.logs = self._createFolder("logs") + self.environments = self._createFolder("environments") + self.temp = self._createFolder("temp") + self._artifactsFolder = self._createFolder("artifacts") + + self.runsLogDirectory = self.logs / "runs" + self.runsLogDirectory.mkdir(exist_ok = True) + + self.coretexpylibLogs = self.logs / "coretexpylib" + self.coretexpylibLogs.mkdir(exist_ok = True) + + def _createFolder(self, name: str) -> Path: + path = self._root / name + + if not path.exists(): + path.mkdir(parents = True, exist_ok = True) + + return path + + def createTempFolder(self, name: str) -> Path: + tempFolderPath = self.temp / name + + if tempFolderPath.exists(): + raise FileExistsError + + tempFolderPath.mkdir() + return tempFolderPath + + def getArtifactsFolder(self, taskRunId: int) -> Path: + return self._artifactsFolder / str(taskRunId) + + def clearDirectory(self, path: Path) -> None: + for element in file_utils.walk(path): + if element.is_file(): + element.unlink() + + if element.is_dir(): + shutil.rmtree(element) + + def clearTempFiles(self) -> None: + self.clearDirectory(self.temp) + self.clearDirectory(self._artifactsFolder) + + def getRunLogsDir(self, taskRunId: int) -> Path: + taskRunLogsDir = self.runsLogDirectory / str(taskRunId) + taskRunLogsDir.mkdir(parents = True, exist_ok = True) + + return taskRunLogsDir + + @contextmanager + def tempFile(self, name: Optional[str] = None) -> Iterator[Path]: + if name is None: + name = str(uuid.uuid4()) + + path = self.temp / name + if path.exists(): + raise FileExistsError(path) + try: + yield path + finally: + path.unlink(missing_ok = True) + + +folder_manager = FolderManager(os.environ["CTX_STORAGE_PATH"]) + +@contextmanager +def currentFolderManager(storagePath: Path) -> Iterator[None]: + global folder_manager + originalManager = folder_manager + folder_manager = FolderManager(storagePath) + try: + yield None + finally: + folder_manager = originalManager diff --git a/coretex/_logger.py b/coretex/_logger.py index 13521e92..d5a1b3cb 100644 --- a/coretex/_logger.py +++ b/coretex/_logger.py @@ -17,7 +17,7 @@ from datetime import datetime -from . import folder_manager +from ._folder_manager import folder_manager from .logging import initializeLogger, LogSeverity from .configuration import CONFIG_DIR diff --git a/coretex/_task/base_callback.py b/coretex/_task/base_callback.py index 73dcef2c..4bb61308 100644 --- a/coretex/_task/base_callback.py +++ b/coretex/_task/base_callback.py @@ -23,7 +23,7 @@ import signal from .current_task_run import setCurrentTaskRun -from .. import folder_manager +from .._folder_manager import folder_manager from ..entities import TaskRun diff --git a/coretex/_task/initialization.py b/coretex/_task/initialization.py index 2d1e2152..9bce3d4a 100644 --- a/coretex/_task/initialization.py +++ b/coretex/_task/initialization.py @@ -22,7 +22,7 @@ from .remote import processRemote from .current_task_run import setCurrentTaskRun -from .. import folder_manager +from .._folder_manager import folder_manager from ..entities import TaskRun, TaskRunStatus from ..logging import createFormatter, initializeLogger from ..logging.severity import LogSeverity diff --git a/coretex/_task/worker/utils.py b/coretex/_task/worker/utils.py index bcc287db..e05c376d 100644 --- a/coretex/_task/worker/utils.py +++ b/coretex/_task/worker/utils.py @@ -19,7 +19,7 @@ import logging -from ... import folder_manager +from ..._folder_manager import folder_manager from ...utils import createFileHandler diff --git a/coretex/bioinformatics/ctx_qiime2/utils.py b/coretex/bioinformatics/ctx_qiime2/utils.py index 191c3d2e..4e0a04ca 100644 --- a/coretex/bioinformatics/ctx_qiime2/utils.py +++ b/coretex/bioinformatics/ctx_qiime2/utils.py @@ -23,7 +23,7 @@ import shutil import gzip -from ... import folder_manager +from ..._folder_manager import folder_manager from ...entities import TaskRun, CustomSample, CustomDataset from ...networking import NetworkRequestError diff --git a/coretex/cache.py b/coretex/cache.py index c41282bd..d3845b51 100644 --- a/coretex/cache.py +++ b/coretex/cache.py @@ -25,7 +25,7 @@ import requests -from . import folder_manager +from ._folder_manager import folder_manager class CacheException(Exception): diff --git a/coretex/cli/commands/task.py b/coretex/cli/commands/task.py index 35e6d278..2e83de86 100644 --- a/coretex/cli/commands/task.py +++ b/coretex/cli/commands/task.py @@ -22,7 +22,7 @@ from ..modules.user import initializeUserSession from ..modules.utils import onBeforeCommandExecute from ..modules.project_utils import getProject -from ... import folder_manager +from ..._folder_manager import folder_manager, currentFolderManager from ..._task import TaskRunWorker, executeRunLocally, readTaskConfig, runLogger from ...configuration import loadConfig from ...entities import TaskRun, TaskRunStatus @@ -45,7 +45,8 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo parameters = readTaskConfig() # clearing temporary files in case that node was manually killed before - folder_manager.clearTempFiles() + with currentFolderManager(config["storagePath"]): + folder_manager.clearTempFiles() selectedProject = getProject(project, config) if selectedProject is None: @@ -87,4 +88,5 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo else: taskRun.updateStatus(TaskRunStatus.completedWithSuccess) - folder_manager.clearTempFiles() + with currentFolderManager(config["storagePath"]): + folder_manager.clearTempFiles() diff --git a/coretex/entities/dataset/image_dataset/synthetic_image_generator.py b/coretex/entities/dataset/image_dataset/synthetic_image_generator.py index 4f8766b2..09c367f0 100644 --- a/coretex/entities/dataset/image_dataset/synthetic_image_generator.py +++ b/coretex/entities/dataset/image_dataset/synthetic_image_generator.py @@ -32,7 +32,7 @@ from .base import BaseImageDataset from ...sample import ImageSample, AnnotatedImageSampleData from ...annotation import CoretexSegmentationInstance, CoretexImageAnnotation, BBox -from .... import folder_manager +from ...._folder_manager import folder_manager ANNOTATION_NAME = "annotations.json" diff --git a/coretex/entities/dataset/network_dataset.py b/coretex/entities/dataset/network_dataset.py index 413e3766..a6721342 100644 --- a/coretex/entities/dataset/network_dataset.py +++ b/coretex/entities/dataset/network_dataset.py @@ -30,7 +30,7 @@ from .state import DatasetState from ..sample import NetworkSample from ..utils import isEntityNameValid -from ... import folder_manager +from ..._folder_manager import folder_manager from ...codable import KeyDescriptor from ...networking import NetworkObject, \ fileChunkUpload, networkManager, NetworkRequestError diff --git a/coretex/entities/dataset/sequence_dataset/sequence_dataset.py b/coretex/entities/dataset/sequence_dataset/sequence_dataset.py index b2f83137..db328e51 100644 --- a/coretex/entities/dataset/sequence_dataset/sequence_dataset.py +++ b/coretex/entities/dataset/sequence_dataset/sequence_dataset.py @@ -24,7 +24,7 @@ from .base import BaseSequenceDataset from ..network_dataset import NetworkDataset, _chunkSampleImport, _encryptedSampleImport from ...sample import SequenceSample, CustomSample -from .... import folder_manager +from ...._folder_manager import folder_manager from ....codable import KeyDescriptor from ....cryptography import getProjectKey from ....utils import file as file_utils diff --git a/coretex/entities/model/model.py b/coretex/entities/model/model.py index 9c8cfad2..524e0f91 100644 --- a/coretex/entities/model/model.py +++ b/coretex/entities/model/model.py @@ -25,7 +25,7 @@ import logging from ..utils import isEntityNameValid -from ... import folder_manager +from ..._folder_manager import folder_manager from ...networking import networkManager, NetworkObject, ChunkUploadSession, MAX_CHUNK_SIZE, NetworkRequestError from ...codable import KeyDescriptor diff --git a/coretex/entities/sample/network_sample.py b/coretex/entities/sample/network_sample.py index eda3d9c2..bae1533a 100644 --- a/coretex/entities/sample/network_sample.py +++ b/coretex/entities/sample/network_sample.py @@ -26,7 +26,7 @@ from .sample import Sample from ..project import ProjectType -from ... import folder_manager +from ..._folder_manager import folder_manager from ...codable import KeyDescriptor from ...networking import NetworkObject, networkManager, NetworkRequestError, \ fileChunkUpload, MAX_CHUNK_SIZE, FileData diff --git a/coretex/entities/task_run/artifact.py b/coretex/entities/task_run/artifact.py index 4b3db090..3eeacce3 100644 --- a/coretex/entities/task_run/artifact.py +++ b/coretex/entities/task_run/artifact.py @@ -20,7 +20,7 @@ from enum import IntEnum from pathlib import Path -from ... import folder_manager +from ..._folder_manager import folder_manager from ...codable import Codable, KeyDescriptor from ...networking import networkManager, FileData from ...utils import guessMimeType diff --git a/coretex/entities/task_run/task_run.py b/coretex/entities/task_run/task_run.py index 52bb366e..314bd6f1 100644 --- a/coretex/entities/task_run/task_run.py +++ b/coretex/entities/task_run/task_run.py @@ -35,7 +35,7 @@ from ..dataset import Dataset, LocalDataset, NetworkDataset from ..project import ProjectType from ..model import Model -from ... import folder_manager +from ..._folder_manager import folder_manager from ...codable import KeyDescriptor from ...networking import networkManager, NetworkObject, NetworkRequestError, FileData diff --git a/coretex/entities/task_run/utils.py b/coretex/entities/task_run/utils.py index c1a26da9..ae549669 100644 --- a/coretex/entities/task_run/utils.py +++ b/coretex/entities/task_run/utils.py @@ -26,7 +26,7 @@ from ..dataset import Dataset, LocalDataset, NetworkDataset, LocalCustomDataset, \ CustomDataset, LocalImageDataset, ImageDataset from ..project import ProjectType -from ... import folder_manager +from ..._folder_manager import folder_manager def getDatasetType(type_: ProjectType, isLocal: bool) -> Type[Dataset]: diff --git a/coretex/folder_manager.py b/coretex/folder_manager.py deleted file mode 100644 index a9cd75da..00000000 --- a/coretex/folder_manager.py +++ /dev/null @@ -1,196 +0,0 @@ -# Copyright (C) 2023 Coretex LLC - -# This file is part of Coretex.ai - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from typing import Iterator, Optional -from pathlib import Path -from contextlib import contextmanager - -import shutil -import uuid - -from .utils import file as file_utils -from .configuration import loadConfig - -""" - Used for handling everything related to local storage - when working with Coretex - - Contains - -------- - samplesFolder : str - folder where samples are stored - modelsFolder : str - folder where models are stored - temp : str - folder where temp files and folders are stored, - this is deleted when the run has finished executing - - datasetsFolder : Path - folder where datasets are stored (samples are symlinked for datasets) - cache : Path - folder where cache module stores items - logs : Path - folder where node and run logs are stored - environments : Path - folder where node stores python environments -""" - - -def _createFolder(name: str) -> Path: - path = _root / name - - if not path.exists(): - path.mkdir(parents = True, exist_ok = True) - - return path - -config = loadConfig() -storagePath = config["storagePath"] # value of key storagePath in config dictionary is guaranteed to never be None -_root = Path(storagePath).expanduser() - -samplesFolder = _createFolder("samples") -modelsFolder = _createFolder("models") -datasetsFolder = _createFolder("datasets") -cache = _createFolder("cache") -logs = _createFolder("logs") -environments = _createFolder("environments") -temp = _createFolder("temp") -_artifactsFolder = _createFolder("artifacts") - -runsLogDirectory = logs / "runs" -runsLogDirectory.mkdir(exist_ok = True) - -coretexpylibLogs = logs / "coretexpylib" -coretexpylibLogs.mkdir(exist_ok = True) - - -def createTempFolder(name: str) -> Path: - """ - Creates temp folder which is deleted once - the run has finished executing - - Parameters - ---------- - name : str - name of the folder - - Returns - ------- - Path -> path to the created folder - - Raises - ------ - FileExistsError -> if the temp folder already exists - - Example - ------- - >>> from coretex import folder_manager - \b - >>> dummyFolderPath = folder_manager.createTempFolder("dummyTempFolder") - >>> print(dummyFolderPath) - "/Users/X/.coretex/temp/dummyTempFolder" - """ - - tempFolderPath = temp / name - - if tempFolderPath.exists(): - raise FileExistsError - - tempFolderPath.mkdir() - return tempFolderPath - - -def getArtifactsFolder(taskRunId: int) -> Path: - """ - Retrieves the path to where the artifacts are stored - for the specified TaskRuns - - Parameters - ---------- - taskRunId : int - id of the TaskRun - - Returns - ------- - Path -> path to the TaskRun artifacts local storage - - Example - ------- - >>> from coretex.folder_management import FolderManager - \b - >>> artifactsFolderPath = FolderManager.instance().getArtifactsFolder(1023) - >>> print(artifactsFolderPath) - Path("/Users/bogdanbm/.coretex/artifacts/1023") - - """ - - return _artifactsFolder / str(taskRunId) - - -def clearDirectory(path: Path) -> None: - for element in file_utils.walk(path): - if element.is_file(): - element.unlink() - - if element.is_dir(): - shutil.rmtree(element) - - -def clearTempFiles() -> None: - """ - Deletes all temp files and folders (including artifacts) - """ - - clearDirectory(temp) - clearDirectory(_artifactsFolder) - - -def getRunLogsDir(taskRunId: int) -> Path: - taskRunLogsDir = runsLogDirectory / str(taskRunId) - taskRunLogsDir.mkdir(parents = True, exist_ok = True) - - return taskRunLogsDir - - -@contextmanager -def tempFile(name: Optional[str] = None) -> Iterator[Path]: - """ - Returns a path to temporary file and deletes - it if it exists once the context is exited. - - Parameters - ---------- - name : Optional[str] - Name of the file. If not specified a random uuid4 - will be generated and used as the name - - Returns - ------- - Iterator[Path] -> path to the file - """ - - if name is None: - name = str(uuid.uuid4()) - - path = temp / name - if path.exists(): - raise FileExistsError(path) - - try: - yield path - finally: - path.unlink(missing_ok = True) diff --git a/coretex/utils/inference.py b/coretex/utils/inference.py index 8a6f1058..588bdfec 100644 --- a/coretex/utils/inference.py +++ b/coretex/utils/inference.py @@ -10,7 +10,7 @@ import ezkl import numpy as np -from .. import folder_manager +from .._folder_manager import folder_manager async def genWitness(inputPath: Path, circuit: Path, witnessPath: Path) -> None: From 1ad133cf362a012ba253d4bef4f4f37387ecc6dd Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Tue, 23 Jul 2024 16:09:21 +0200 Subject: [PATCH 12/37] CTX-6421: Revert deleted newline. --- coretex/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/coretex/__init__.py b/coretex/__init__.py index e76f049f..12a51abb 100644 --- a/coretex/__init__.py +++ b/coretex/__init__.py @@ -24,6 +24,7 @@ from ._logger import _initializeDefaultLogger, _initializeCLILogger from .configuration import isCliRuntime + if isCliRuntime(): _initializeCLILogger() else: From 0f7e97a7df5a7b5ccd663880c39de57172e0b3bb Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Tue, 23 Jul 2024 16:10:22 +0200 Subject: [PATCH 13/37] CTX-6421: Added newline below folder_manager declaration. --- coretex/_folder_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/coretex/_folder_manager.py b/coretex/_folder_manager.py index 20f23622..7f57d88a 100644 --- a/coretex/_folder_manager.py +++ b/coretex/_folder_manager.py @@ -127,6 +127,7 @@ def tempFile(self, name: Optional[str] = None) -> Iterator[Path]: folder_manager = FolderManager(os.environ["CTX_STORAGE_PATH"]) + @contextmanager def currentFolderManager(storagePath: Path) -> Iterator[None]: global folder_manager From 829b64921dfe9aac22bda7ee382dcd0423ccbd64 Mon Sep 17 00:00:00 2001 From: coretex-admin Date: Wed, 24 Jul 2024 02:29:02 +0000 Subject: [PATCH 14/37] Bump version to 1.0.163 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bf42eb95..0219eee3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "coretex" -version = "1.0.162" +version = "1.0.163" authors = [ { name="Duško Mirković", email="dmirkovic@coretex.ai" } ] From 8d5c203f88d11462af60512409848d769f968c19 Mon Sep 17 00:00:00 2001 From: Dusko Vesic Date: Wed, 24 Jul 2024 09:14:26 +0200 Subject: [PATCH 15/37] code improvements --- coretex/entities/dataset/network_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coretex/entities/dataset/network_dataset.py b/coretex/entities/dataset/network_dataset.py index 7fad979d..ce17156b 100644 --- a/coretex/entities/dataset/network_dataset.py +++ b/coretex/entities/dataset/network_dataset.py @@ -284,7 +284,7 @@ def createCacheDataset(cls, prefix: str, dependencies: List[str], projectId: int ------- >>> from coretex import NetworkDataset \b - >>> dummyDataset = NetworkDataset.createDataset("dummyDataset", dependencies, 123) + >>> dummyDataset = NetworkDataset.createCacheDataset("dummyDataset", dependencies, 123) """ if not isEntityNameValid(prefix): From cbbadc2028b64e2921142eba860c2313d0c18f22 Mon Sep 17 00:00:00 2001 From: Dusko Vesic Date: Wed, 24 Jul 2024 09:31:14 +0200 Subject: [PATCH 16/37] code improvements --- coretex/entities/dataset/network_dataset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/coretex/entities/dataset/network_dataset.py b/coretex/entities/dataset/network_dataset.py index ce17156b..8635a4ac 100644 --- a/coretex/entities/dataset/network_dataset.py +++ b/coretex/entities/dataset/network_dataset.py @@ -284,6 +284,7 @@ def createCacheDataset(cls, prefix: str, dependencies: List[str], projectId: int ------- >>> from coretex import NetworkDataset \b + >>> dependencies = [str(projectId), str(index), str(parameter)] >>> dummyDataset = NetworkDataset.createCacheDataset("dummyDataset", dependencies, 123) """ From 01e4b620f79526463da914124ecd59e01d423895 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 24 Jul 2024 10:40:11 +0200 Subject: [PATCH 17/37] CTX-6421: Circular import bugfix --- coretex/_folder_manager.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/coretex/_folder_manager.py b/coretex/_folder_manager.py index 7f57d88a..affc1fde 100644 --- a/coretex/_folder_manager.py +++ b/coretex/_folder_manager.py @@ -23,8 +23,6 @@ import shutil import uuid -from .utils import file as file_utils - class FolderManager: @@ -94,12 +92,8 @@ def getArtifactsFolder(self, taskRunId: int) -> Path: return self._artifactsFolder / str(taskRunId) def clearDirectory(self, path: Path) -> None: - for element in file_utils.walk(path): - if element.is_file(): - element.unlink() - - if element.is_dir(): - shutil.rmtree(element) + shutil.rmtree(path) + path.mkdir() def clearTempFiles(self) -> None: self.clearDirectory(self.temp) From bde886330804bfa2aeca9cf2adf0e8dc9e80ded5 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 24 Jul 2024 14:32:04 +0200 Subject: [PATCH 18/37] CTX-6421: Reverted accident function documentation removal. --- coretex/_folder_manager.py | 58 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/coretex/_folder_manager.py b/coretex/_folder_manager.py index affc1fde..d3b3dfb9 100644 --- a/coretex/_folder_manager.py +++ b/coretex/_folder_manager.py @@ -80,6 +80,28 @@ def _createFolder(self, name: str) -> Path: return path def createTempFolder(self, name: str) -> Path: + """ + Creates temp folder which is deleted once + the run has finished executing + Parameters + ---------- + name : str + name of the folder + Returns + ------- + Path -> path to the created folder + Raises + ------ + FileExistsError -> if the temp folder already exists + Example + ------- + >>> from coretex import folder_manager + \b + >>> dummyFolderPath = folder_manager.createTempFolder("dummyTempFolder") + >>> print(dummyFolderPath) + "/Users/X/.coretex/temp/dummyTempFolder" + """ + tempFolderPath = self.temp / name if tempFolderPath.exists(): @@ -89,6 +111,25 @@ def createTempFolder(self, name: str) -> Path: return tempFolderPath def getArtifactsFolder(self, taskRunId: int) -> Path: + """ + Retrieves the path to where the artifacts are stored + for the specified TaskRuns + Parameters + ---------- + taskRunId : int + id of the TaskRun + Returns + ------- + Path -> path to the TaskRun artifacts local storage + Example + ------- + >>> from coretex.folder_management import FolderManager + \b + >>> artifactsFolderPath = FolderManager.instance().getArtifactsFolder(1023) + >>> print(artifactsFolderPath) + Path("/Users/bogdanbm/.coretex/artifacts/1023") + """ + return self._artifactsFolder / str(taskRunId) def clearDirectory(self, path: Path) -> None: @@ -96,6 +137,10 @@ def clearDirectory(self, path: Path) -> None: path.mkdir() def clearTempFiles(self) -> None: + """ + Deletes all temp files and folders (including artifacts) + """ + self.clearDirectory(self.temp) self.clearDirectory(self._artifactsFolder) @@ -107,6 +152,19 @@ def getRunLogsDir(self, taskRunId: int) -> Path: @contextmanager def tempFile(self, name: Optional[str] = None) -> Iterator[Path]: + """ + Returns a path to temporary file and deletes + it if it exists once the context is exited. + Parameters + ---------- + name : Optional[str] + Name of the file. If not specified a random uuid4 + will be generated and used as the name + Returns + ------- + Iterator[Path] -> path to the file + """ + if name is None: name = str(uuid.uuid4()) From c4d6e5be25edbd0e64991ed0c5574706ce7114cb Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 24 Jul 2024 14:49:50 +0200 Subject: [PATCH 19/37] CTX-6421: Added missing newlines. --- coretex/_folder_manager.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/coretex/_folder_manager.py b/coretex/_folder_manager.py index d3b3dfb9..79543928 100644 --- a/coretex/_folder_manager.py +++ b/coretex/_folder_manager.py @@ -83,16 +83,20 @@ def createTempFolder(self, name: str) -> Path: """ Creates temp folder which is deleted once the run has finished executing + Parameters ---------- name : str name of the folder + Returns ------- Path -> path to the created folder + Raises ------ FileExistsError -> if the temp folder already exists + Example ------- >>> from coretex import folder_manager @@ -114,13 +118,16 @@ def getArtifactsFolder(self, taskRunId: int) -> Path: """ Retrieves the path to where the artifacts are stored for the specified TaskRuns + Parameters ---------- taskRunId : int id of the TaskRun + Returns ------- Path -> path to the TaskRun artifacts local storage + Example ------- >>> from coretex.folder_management import FolderManager @@ -155,11 +162,13 @@ def tempFile(self, name: Optional[str] = None) -> Iterator[Path]: """ Returns a path to temporary file and deletes it if it exists once the context is exited. + Parameters ---------- name : Optional[str] Name of the file. If not specified a random uuid4 will be generated and used as the name + Returns ------- Iterator[Path] -> path to the file From 2e4640c83b36addc51c52e8aa0e981a72f88ce7b Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 24 Jul 2024 16:24:11 +0200 Subject: [PATCH 20/37] CTX-6421: Removed contextmanager as it wasn't working... Handled this on simpler way --- coretex/_folder_manager.py | 11 ----------- coretex/cli/commands/task.py | 9 ++++----- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/coretex/_folder_manager.py b/coretex/_folder_manager.py index 79543928..b710be30 100644 --- a/coretex/_folder_manager.py +++ b/coretex/_folder_manager.py @@ -187,14 +187,3 @@ def tempFile(self, name: Optional[str] = None) -> Iterator[Path]: folder_manager = FolderManager(os.environ["CTX_STORAGE_PATH"]) - - -@contextmanager -def currentFolderManager(storagePath: Path) -> Iterator[None]: - global folder_manager - originalManager = folder_manager - folder_manager = FolderManager(storagePath) - try: - yield None - finally: - folder_manager = originalManager diff --git a/coretex/cli/commands/task.py b/coretex/cli/commands/task.py index 2e83de86..64c9161c 100644 --- a/coretex/cli/commands/task.py +++ b/coretex/cli/commands/task.py @@ -22,7 +22,7 @@ from ..modules.user import initializeUserSession from ..modules.utils import onBeforeCommandExecute from ..modules.project_utils import getProject -from ..._folder_manager import folder_manager, currentFolderManager +from ..._folder_manager import FolderManager from ..._task import TaskRunWorker, executeRunLocally, readTaskConfig, runLogger from ...configuration import loadConfig from ...entities import TaskRun, TaskRunStatus @@ -45,8 +45,8 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo parameters = readTaskConfig() # clearing temporary files in case that node was manually killed before - with currentFolderManager(config["storagePath"]): - folder_manager.clearTempFiles() + folderManager = FolderManager(config["storagePath"]) + folderManager.clearTempFiles() selectedProject = getProject(project, config) if selectedProject is None: @@ -88,5 +88,4 @@ def run(path: str, name: Optional[str], description: Optional[str], snapshot: bo else: taskRun.updateStatus(TaskRunStatus.completedWithSuccess) - with currentFolderManager(config["storagePath"]): - folder_manager.clearTempFiles() + folderManager.clearTempFiles() From 0f05b16fb69b86204a7734b174c003d84246035a Mon Sep 17 00:00:00 2001 From: coretex-admin Date: Thu, 25 Jul 2024 02:26:53 +0000 Subject: [PATCH 21/37] Bump version to 1.0.164 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0219eee3..665ad92d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "coretex" -version = "1.0.163" +version = "1.0.164" authors = [ { name="Duško Mirković", email="dmirkovic@coretex.ai" } ] From 92d4750c50f6f68882bf5c2e58d191a680718dbc Mon Sep 17 00:00:00 2001 From: coretex-admin Date: Fri, 26 Jul 2024 02:24:21 +0000 Subject: [PATCH 22/37] Bump version to 1.0.165 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 665ad92d..d1cc6919 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "coretex" -version = "1.0.164" +version = "1.0.165" authors = [ { name="Duško Mirković", email="dmirkovic@coretex.ai" } ] From a630d49cd96ded1fc50a6f1ed83db0840038c440 Mon Sep 17 00:00:00 2001 From: coretex-admin Date: Mon, 29 Jul 2024 02:37:23 +0000 Subject: [PATCH 23/37] Bump version to 1.0.166 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d1cc6919..49ab205a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "coretex" -version = "1.0.165" +version = "1.0.166" authors = [ { name="Duško Mirković", email="dmirkovic@coretex.ai" } ] From 68721abbefa47e8ef41cb9716f08fb753e38916b Mon Sep 17 00:00:00 2001 From: Vuk Manojlovic Date: Mon, 29 Jul 2024 17:27:36 +0200 Subject: [PATCH 24/37] CTX-5524: Implemented discussed changes (EntityTagType as return type) --- coretex/entities/dataset/network_dataset.py | 2 +- coretex/entities/model/model.py | 2 +- coretex/entities/tag.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/coretex/entities/dataset/network_dataset.py b/coretex/entities/dataset/network_dataset.py index 645cd6a6..199cb210 100644 --- a/coretex/entities/dataset/network_dataset.py +++ b/coretex/entities/dataset/network_dataset.py @@ -131,7 +131,7 @@ def path(self) -> Path: return folder_manager.datasetsFolder / str(self.id) @property - def entityTagType(self) -> int: + def entityTagType(self) -> EntityTagType: return EntityTagType.dataset # Codable overrides diff --git a/coretex/entities/model/model.py b/coretex/entities/model/model.py index 5812dbf4..85a2994d 100644 --- a/coretex/entities/model/model.py +++ b/coretex/entities/model/model.py @@ -84,7 +84,7 @@ def zipPath(self) -> Path: return self.path.with_suffix(".zip") @property - def entityTagType(self) -> int: + def entityTagType(self) -> EntityTagType: return EntityTagType.model @classmethod diff --git a/coretex/entities/tag.py b/coretex/entities/tag.py index eeafa83c..ce63c89f 100644 --- a/coretex/entities/tag.py +++ b/coretex/entities/tag.py @@ -39,7 +39,7 @@ class Taggable(ABC): @property @abstractmethod - def entityTagType(self) -> int: + def entityTagType(self) -> EntityTagType: pass def _getTagId(self, tagName: str) -> Optional[int]: @@ -54,7 +54,7 @@ def _getTagId(self, tagName: str) -> Optional[int]: raise NetworkRequestError(response, "Failed to check existing tags") tags = response.getJson(dict).get("data") - if not isinstance(tags, dict): + if not isinstance(tags, list): raise NetworkRequestError(response, f"Field \"data\" from tag response must be dict, but got {type(tags)} instead") if len(tags) == 0: From af859f9962d851a79a1076b8f9da593abb3e7886 Mon Sep 17 00:00:00 2001 From: coretex-admin Date: Tue, 30 Jul 2024 02:32:23 +0000 Subject: [PATCH 25/37] Bump version to 1.0.167 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 49ab205a..08ea329c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "coretex" -version = "1.0.166" +version = "1.0.167" authors = [ { name="Duško Mirković", email="dmirkovic@coretex.ai" } ] From 0d7c3659feddbe39494ad3fd3998b570d2524207 Mon Sep 17 00:00:00 2001 From: Dusko Mirkovic Date: Tue, 30 Jul 2024 09:16:27 +0200 Subject: [PATCH 26/37] Hotfix: stream parameter was unused --- coretex/networking/network_manager_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/coretex/networking/network_manager_base.py b/coretex/networking/network_manager_base.py index 6a003f8f..f64d7b1b 100644 --- a/coretex/networking/network_manager_base.py +++ b/coretex/networking/network_manager_base.py @@ -248,7 +248,8 @@ def request( auth = auth, timeout = timeout, files = files, - headers = headers + headers = headers, + stream = stream ) response = NetworkResponse(rawResponse, endpoint) From 6430f948ec0a3e4425afff266cdb80491d61878e Mon Sep 17 00:00:00 2001 From: Dusko Mirkovic Date: Tue, 30 Jul 2024 10:08:36 +0200 Subject: [PATCH 27/37] Revert "Hotfix: stream parameter was unused" This reverts commit 0d7c3659feddbe39494ad3fd3998b570d2524207. --- coretex/networking/network_manager_base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/coretex/networking/network_manager_base.py b/coretex/networking/network_manager_base.py index f64d7b1b..6a003f8f 100644 --- a/coretex/networking/network_manager_base.py +++ b/coretex/networking/network_manager_base.py @@ -248,8 +248,7 @@ def request( auth = auth, timeout = timeout, files = files, - headers = headers, - stream = stream + headers = headers ) response = NetworkResponse(rawResponse, endpoint) From ee414b3488bada25119212172237797ba3c79343 Mon Sep 17 00:00:00 2001 From: Vuk Manojlovic Date: Tue, 30 Jul 2024 10:22:50 +0200 Subject: [PATCH 28/37] CTX-5524: Implemented requested changes --- coretex/entities/tag.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/coretex/entities/tag.py b/coretex/entities/tag.py index ce63c89f..14a88e79 100644 --- a/coretex/entities/tag.py +++ b/coretex/entities/tag.py @@ -45,7 +45,7 @@ def entityTagType(self) -> EntityTagType: def _getTagId(self, tagName: str) -> Optional[int]: parameters = { "name": tagName, - "type": int(self.entityTagType), + "type": self.entityTagType.value, "project_id": self.projectId } @@ -113,7 +113,7 @@ def addTag(self, tag: str, color: Optional[str] = None) -> None: parameters = { "entity_id": self.id, - "type": self.entityTagType, + "type": self.entityTagType.value, "tags": tags } @@ -144,7 +144,7 @@ def removeTag(self, tag: str) -> None: parameters = { "entity_id": self.id, "tag_id": tagId, - "type": self.entityTagType + "type": self.entityTagType.value } response = networkManager.post("tag/remove", parameters) From d1f418d9639b863d5b7123325fcb7288e68f8f22 Mon Sep 17 00:00:00 2001 From: Vuk Manojlovic Date: Tue, 30 Jul 2024 10:39:29 +0200 Subject: [PATCH 29/37] CTX-5524: Implemented requested changes --- coretex/entities/tag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coretex/entities/tag.py b/coretex/entities/tag.py index 14a88e79..d7329f24 100644 --- a/coretex/entities/tag.py +++ b/coretex/entities/tag.py @@ -97,7 +97,7 @@ def addTag(self, tag: str, color: Optional[str] = None) -> None: if color is None: color = f"#{random.randint(0, 0xFFFFFF):06x}" else: - if re.match(r'^#([A-Fa-f0-9]{3}|[A-Fa-f0-9]{6})$', color) is None: + if re.match(r"^#([A-Fa-f0-9]{3}|[A-Fa-f0-9]{6})$", color) is None: raise ValueError(">> [Coretex] Tag color has to follow hexadecimal color code") tags: Dict[str, Any] = {} From 27299c951f04df80576d0c90e3561a5176cc7eb9 Mon Sep 17 00:00:00 2001 From: coretex-admin Date: Wed, 31 Jul 2024 02:28:03 +0000 Subject: [PATCH 30/37] Bump version to 1.0.168 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 08ea329c..a8d23ba9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "coretex" -version = "1.0.167" +version = "1.0.168" authors = [ { name="Duško Mirković", email="dmirkovic@coretex.ai" } ] From aa8b6ec89481d4a634fede8cad3dfd90f5464aea Mon Sep 17 00:00:00 2001 From: Vuk Manojlovic Date: Wed, 31 Jul 2024 10:51:22 +0200 Subject: [PATCH 31/37] Hotfix: Added bounding box inflation --- coretex/entities/annotation/image/bbox.py | 33 +++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/coretex/entities/annotation/image/bbox.py b/coretex/entities/annotation/image/bbox.py index 65fd4aea..197ece85 100644 --- a/coretex/entities/annotation/image/bbox.py +++ b/coretex/entities/annotation/image/bbox.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Final, List, Dict +from typing import Final, List, Dict, Optional, Tuple from typing_extensions import Self from ....codable import Codable, KeyDescriptor @@ -159,11 +159,13 @@ def iou(self, other: 'BBox') -> float: """ Calculate Intersection over Union (IoU) between two bounding boxes - Parameters: + Parameters + ---------- other : BBox bounding box for which the IoU will be calculated Returns + ------- float -> IoU score """ @@ -176,3 +178,30 @@ def iou(self, other: 'BBox') -> float: unionArea = self.area + other.area - intersectionArea return intersectionArea / unionArea if unionArea > 0 else 0.0 + + + def inflate(self, percentage: int, imageSize: Optional[Tuple[int, int]] = None) -> None: + """ + Increases the size of the bounding box by a percentage + + Parameters + ---------- + percentage : int + the percentage by which the bounding box will be inflated + imageSize : Optional[Tuple[int, int]] + bounding box will not be able to go beyond these dimensions (width, height) + """ + + if imageSize is None: + imageSize = (float("inf"), float("inf")) + + imageWidth, imageHeight = imageSize + + inflateFactor = percentage / 100.0 + inflateWidth = self.width * inflateFactor / 2 + inflateHeight = self.height * inflateFactor / 2 + + self.minX = max(0, self.minX - inflateWidth) + self.minY = max(0, self.minY - inflateHeight) + self.width = min(imageWidth, self.width + inflateWidth * 2) + self.height = min(imageHeight, self.height + inflateHeight * 2) From f8f3e72d38c54b5c2e867308fdda4f9878fa5bb5 Mon Sep 17 00:00:00 2001 From: Vuk Manojlovic Date: Wed, 31 Jul 2024 11:00:01 +0200 Subject: [PATCH 32/37] Hotfix: Fixed linter errors --- coretex/entities/annotation/image/bbox.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/coretex/entities/annotation/image/bbox.py b/coretex/entities/annotation/image/bbox.py index 197ece85..d151a727 100644 --- a/coretex/entities/annotation/image/bbox.py +++ b/coretex/entities/annotation/image/bbox.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Final, List, Dict, Optional, Tuple +from typing import List, Dict, Optional, Tuple, Union from typing_extensions import Self from ....codable import Codable, KeyDescriptor @@ -39,11 +39,11 @@ class BBox(Codable): """ def __init__(self, minX: int = 0, minY: int = 0, width: int = 0, height: int = 0) -> None: - self.minX: Final = minX - self.minY: Final = minY + self.minX: int = minX + self.minY: int = minY - self.width: Final = width - self.height: Final = height + self.width: int = width + self.height: int = height @property def maxX(self) -> int: @@ -180,7 +180,7 @@ def iou(self, other: 'BBox') -> float: return intersectionArea / unionArea if unionArea > 0 else 0.0 - def inflate(self, percentage: int, imageSize: Optional[Tuple[int, int]] = None) -> None: + def inflate(self, percentage: int, imageSize: Optional[Tuple[Union[int, float], Union[int, float]]] = None) -> None: """ Increases the size of the bounding box by a percentage @@ -201,7 +201,7 @@ def inflate(self, percentage: int, imageSize: Optional[Tuple[int, int]] = None) inflateWidth = self.width * inflateFactor / 2 inflateHeight = self.height * inflateFactor / 2 - self.minX = max(0, self.minX - inflateWidth) - self.minY = max(0, self.minY - inflateHeight) - self.width = min(imageWidth, self.width + inflateWidth * 2) - self.height = min(imageHeight, self.height + inflateHeight * 2) + self.minX = int(max(0, self.minX - inflateWidth)) + self.minY = int(max(0, self.minY - inflateHeight)) + self.width = int(min(imageWidth, self.width + inflateWidth * 2)) + self.height = int(min(imageHeight, self.height + inflateHeight * 2)) From 4b8a590723a6fab46ee201153ddddff6d0f47b3f Mon Sep 17 00:00:00 2001 From: Vuk Manojlovic Date: Wed, 31 Jul 2024 11:34:00 +0200 Subject: [PATCH 33/37] Hotfix: Implemeted requested changes --- coretex/entities/annotation/image/bbox.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/coretex/entities/annotation/image/bbox.py b/coretex/entities/annotation/image/bbox.py index d151a727..ab385e89 100644 --- a/coretex/entities/annotation/image/bbox.py +++ b/coretex/entities/annotation/image/bbox.py @@ -39,11 +39,11 @@ class BBox(Codable): """ def __init__(self, minX: int = 0, minY: int = 0, width: int = 0, height: int = 0) -> None: - self.minX: int = minX - self.minY: int = minY + self.minX = minX + self.minY = minY - self.width: int = width - self.height: int = height + self.width = width + self.height = height @property def maxX(self) -> int: @@ -179,7 +179,6 @@ def iou(self, other: 'BBox') -> float: unionArea = self.area + other.area - intersectionArea return intersectionArea / unionArea if unionArea > 0 else 0.0 - def inflate(self, percentage: int, imageSize: Optional[Tuple[Union[int, float], Union[int, float]]] = None) -> None: """ Increases the size of the bounding box by a percentage From a474a87051fa88fadbd1d31865ff6a73c942e430 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Wed, 31 Jul 2024 15:18:25 +0200 Subject: [PATCH 34/37] CTX-6421: Edited functon description according to new changes and discussion on PR. --- coretex/_folder_manager.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/coretex/_folder_manager.py b/coretex/_folder_manager.py index b710be30..a23ac1f9 100644 --- a/coretex/_folder_manager.py +++ b/coretex/_folder_manager.py @@ -32,14 +32,13 @@ class FolderManager: Contains -------- - samplesFolder : str + samplesFolder : Path folder where samples are stored - modelsFolder : str + modelsFolder : Path folder where models are stored - temp : str + temp : Path folder where temp files and folders are stored, this is deleted when the run has finished executing - datasetsFolder : Path folder where datasets are stored (samples are symlinked for datasets) cache : Path From 9e4dd4ae4b4e4b2f416c8189c2e71312ce388589 Mon Sep 17 00:00:00 2001 From: coretex-admin Date: Fri, 2 Aug 2024 02:21:23 +0000 Subject: [PATCH 35/37] Bump version to 1.0.169 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a8d23ba9..cc5722ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "coretex" -version = "1.0.168" +version = "1.0.169" authors = [ { name="Duško Mirković", email="dmirkovic@coretex.ai" } ] From 23b02b6170b7ded6011c4a3299e93f3eeaa95ada Mon Sep 17 00:00:00 2001 From: coretex-admin Date: Mon, 5 Aug 2024 02:34:44 +0000 Subject: [PATCH 36/37] Bump version to 1.0.170 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cc5722ac..40c6ff12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "coretex" -version = "1.0.169" +version = "1.0.170" authors = [ { name="Duško Mirković", email="dmirkovic@coretex.ai" } ] From 67b61d2534534e29636b97431287b100d641d0cf Mon Sep 17 00:00:00 2001 From: coretex-admin Date: Wed, 7 Aug 2024 02:21:12 +0000 Subject: [PATCH 37/37] Bump version to 1.0.171 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 40c6ff12..f3761030 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "coretex" -version = "1.0.170" +version = "1.0.171" authors = [ { name="Duško Mirković", email="dmirkovic@coretex.ai" } ]