Merge pull request #32 from dule1322/CTX-3587

CTX-3587: Redesigned folder manager to always work with pathlib.Path

coretex/_logger.py

@@ -17,11 +17,11 @@
 
 from datetime import datetime
 
-from .folder_management import FolderManager
+from . import folder_manager
 from .logging import LogSeverity, initializeLogger
 from .utils import DATE_FORMAT
 
 
 def _initializeDefaultLogger() -> None:
-    logPath = FolderManager.instance().logs / f"coretexpylib_{datetime.now().strftime(DATE_FORMAT)}.log"
+    logPath = folder_manager.logs / f"coretexpylib_{datetime.now().strftime(DATE_FORMAT)}.log"
     initializeLogger(LogSeverity.info, logPath)

coretex/cache/cache_module.py

@@ -29,7 +29,7 @@
 
 import requests
 
-from ..folder_management import FolderManager
+from .. import folder_manager
 
 
 IGNORED_FILE_TYPES = [".pt"]
@@ -49,34 +49,30 @@ def __hashedKey(key: str) -> str:
 
 
 def __zip(zipPath: Path, filePath: Path, fileName: str) -> None:
-    baseName, fileExtension = os.path.splitext(filePath)
-
-    if fileExtension in IGNORED_FILE_TYPES or not zipfile.is_zipfile(filePath):
-        with ZipFile(zipPath, mode = "w") as archive:
+    if filePath.suffix in IGNORED_FILE_TYPES or not zipfile.is_zipfile(filePath):
+        with ZipFile(zipPath, mode = "w", compression = zipfile.ZIP_DEFLATED) as archive:
             archive.write(filePath, fileName)
 
         filePath.unlink(missing_ok = True)
     else:
-        filePath.rename(FolderManager.instance().cache / filePath.name)
+        filePath.rename(folder_manager.cache / filePath.name)
 
 
 def __downloadFromUrl(url: str, fileName: str) -> Tuple[Path, str]:
-    tempPath = FolderManager.instance().temp
     hashUrl = __hashedKey(url)
-    fileName, fileExtension = os.path.splitext(fileName)
 
     with requests.get(url, stream = True) as r:
         r.raise_for_status()
 
-        resultPath = os.path.join(tempPath, hashUrl)
-        fileName = f"{hashUrl}{fileExtension}"
-        resultPath = os.path.join(tempPath, fileName)
+        _, extension = os.path.splitext(fileName)
+        fileName = f"{hashUrl}{extension}"
 
-        with open(resultPath, 'wb') as f:
-            for chunk in r.iter_content(chunk_size=8192):
+        resultPath = folder_manager.temp / fileName
+        with resultPath.open("wb") as f:
+            for chunk in r.iter_content(chunk_size = 8192):
                 f.write(chunk)
 
-    return Path(resultPath), fileName
+        return resultPath, fileName
 
 
 def storeObject(key: str, object: Any, override: bool = False) -> None:
@@ -105,10 +101,10 @@ def storeObject(key: str, object: Any, override: bool = False) -> None:
     """
 
     hashedKey = __hashedKey(key)
-    zipPath = (FolderManager.instance().cache / hashedKey).with_suffix(".zip")
+    zipPath = (folder_manager.cache / hashedKey).with_suffix(".zip")
 
     pickleName = f"{hashedKey}.pickle"
-    picklePath = FolderManager.instance().cache / pickleName
+    picklePath = folder_manager.cache / pickleName
 
     if override:
         zipPath.unlink(missing_ok = True)
@@ -148,7 +144,7 @@ def storeFile(key: str, filePath: str, override: bool = False) -> None:
         >>> cache.storeFile("dummyFile", filePath)
     """
     hashedKey = __hashedKey(key)
-    cachePath = FolderManager.instance().cache / hashedKey
+    cachePath = folder_manager.cache / hashedKey
 
     cacheZipPath = cachePath.with_suffix(".zip")
 
@@ -194,7 +190,7 @@ def storeUrl(url: str, fileName: str, override: bool = False) -> None:
     """
 
     hashedKey = __hashedKey(url)
-    cacheZipPath = (FolderManager.instance().cache / hashedKey).with_suffix(".zip")
+    cacheZipPath = (folder_manager.cache / hashedKey).with_suffix(".zip")
 
     if override:
         cacheZipPath.unlink(missing_ok = True)
@@ -239,14 +235,14 @@ def load(key: str) -> Any:
     """
 
     hashedKey = __hashedKey(key)
-    cacheZipPath = (FolderManager.instance().cache / hashedKey).with_suffix(".zip")
-    picklePath = (FolderManager.instance().cache / hashedKey).with_suffix(".pickle")
+    cacheZipPath = (folder_manager.cache / hashedKey).with_suffix(".zip")
+    picklePath = (folder_manager.cache / hashedKey).with_suffix(".pickle")
 
     if not cacheZipPath.exists():
         raise CacheException(">> [Coretex] Cache with given key doesn't exist.")
 
     with ZipFile(cacheZipPath, "r") as zipFile:
-        zipFile.extractall(FolderManager.instance().cache)
+        zipFile.extractall(folder_manager.cache)
 
         logging.getLogger("coretexpylib").info(">> [Coretex] Cache with given key exists, loading cache...")
 
@@ -283,7 +279,7 @@ def getPath(key: str) -> Path:
     """
 
     hashedKey = __hashedKey(key)
-    cacheZipPath = (FolderManager.instance().cache / hashedKey).with_suffix(".zip")
+    cacheZipPath = (folder_manager.cache / hashedKey).with_suffix(".zip")
 
     if not cacheZipPath.exists():
         raise CacheException(">> [Coretex] Cache with given key doesn't exist.")
@@ -314,7 +310,7 @@ def exists(key: str) -> bool:
     """
 
     hashedKey = __hashedKey(key)
-    cacheZipPath = (FolderManager.instance().cache / hashedKey).with_suffix(".zip")
+    cacheZipPath = (folder_manager.cache / hashedKey).with_suffix(".zip")
 
     return cacheZipPath.exists()
 
@@ -339,7 +335,7 @@ def remove(key: str) -> None:
     """
 
     hashedKey = __hashedKey(key)
-    cacheZipPath = (FolderManager.instance().cache / hashedKey).with_suffix(".zip")
+    cacheZipPath = (folder_manager.cache / hashedKey).with_suffix(".zip")
 
     if not cacheZipPath.exists():
         raise CacheException(">> [Coretex] Cache with given key doesn't exist.")
@@ -359,4 +355,4 @@ def clear() -> None:
         >>> cache.clear()
     """
 
-    shutil.rmtree(FolderManager.instance().cache)
+    shutil.rmtree(folder_manager.cache)

coretex/coretex/dataset/image_dataset/synthetic_image_generator.py

@@ -32,7 +32,7 @@
 from .base import BaseImageDataset
 from ...sample import ImageSample, AnnotatedImageSampleData
 from ...annotation import CoretexSegmentationInstance, CoretexImageAnnotation, BBox
-from ....folder_management import FolderManager
+from .... import folder_manager
 
 
 ANNOTATION_NAME = "annotations.json"
@@ -41,15 +41,14 @@
 class AugmentedImageSample(ImageSample):
 
     @property
-    def path(self) -> str:
+    def path(self) -> Path:
         """
             Returns
             -------
-            str -> path to new augmented samples directory
+            Path -> path to new augmented samples directory
         """
 
-        tempPath = Path(FolderManager.instance().getTempFolder("temp-augmented-ds"))
-        return str(tempPath / str(self.id))
+        return folder_manager.temp / "temp-augmented-ds" / str(self.id)
 
     @classmethod
     def createFromSample(cls, sample: ImageSample) -> Self:
@@ -242,7 +241,7 @@ def augmentDataset(
             scaling factor
     """
 
-    tempPath = Path(FolderManager.instance().createTempFolder("temp-augmented-ds"))
+    tempPath = folder_manager.createTempFolder("temp-augmented-ds")
     augmentedSamples: List[AugmentedImageSample] = []
 
     for i, background in enumerate(backgroundDataset.samples):

coretex/coretex/dataset/network_dataset.py

@@ -15,7 +15,7 @@
 #     You should have received a copy of the GNU Affero General Public License
 #     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from typing import Optional, Type, TypeVar, Generic, List, Dict, Any
+from typing import Optional, TypeVar, Generic, List, Dict, Any
 from typing_extensions import Self
 from datetime import datetime
 from pathlib import Path
@@ -24,10 +24,10 @@
 
 from .dataset import Dataset
 from ..sample import NetworkSample
+from ... import folder_manager
 from ...codable import KeyDescriptor
 from ...networking import NetworkObject, DEFAULT_PAGE_SIZE
 from ...threading import MultithreadedDataProcessor
-from ...folder_management import FolderManager
 
 
 SampleType = TypeVar("SampleType", bound = "NetworkSample")
@@ -68,7 +68,7 @@ def path(self) -> Path:
             Path -> path of dataset
         """
 
-        return FolderManager.instance().datasetsFolder / str(self.id)
+        return folder_manager.datasetsFolder / str(self.id)
 
     # Codable overrides
 

coretex/coretex/experiment/artifact.py

@@ -15,17 +15,14 @@
 #     You should have received a copy of the GNU Affero General Public License
 #     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from __future__ import annotations
-
+from typing import Optional, Dict, List, Union
+from typing_extensions import Self
 from enum import IntEnum
-from typing import Optional, Dict, List
 from pathlib import Path
 
-import os
-
+from ... import folder_manager
 from ...codable import Codable, KeyDescriptor
 from ...networking import networkManager, RequestType, FileData
-from ...folder_management import FolderManager
 from ...utils import guessMimeType
 
 
@@ -38,25 +35,6 @@ class ArtifactType(IntEnum):
     directory = 1
     file      = 2
 
-    @staticmethod
-    def typeForPath(path: str) -> ArtifactType:
-        """
-            Retrieve ArtifactType based on provided path
-
-            Parameters
-            ----------
-            path : str
-                path to Artifact
-        """
-
-        if os.path.isdir(path):
-            return ArtifactType.directory
-
-        if os.path.isfile(path):
-            return ArtifactType.file
-
-        raise RuntimeError(">> [Coretex] Unreachable")
-
 
 class Artifact(Codable):
 
@@ -97,7 +75,7 @@ def localFilePath(self) -> Path:
             Path -> local path to Artifact
         """
 
-        return FolderManager.instance().getArtifactsFolder(self.experimentId) / self.remoteFilePath
+        return folder_manager.getArtifactsFolder(self.experimentId) / self.remoteFilePath
 
     @property
     def isDirectory(self) -> bool:
@@ -131,7 +109,14 @@ def _keyDescriptors(cls) -> Dict[str, KeyDescriptor]:
         return descriptors
 
     @classmethod
-    def create(cls, experimentId: int, localFilePath: str, remoteFilePath: str, mimeType: Optional[str] = None) -> Optional[Artifact]:
+    def create(
+        cls,
+        experimentId: int,
+        localFilePath: Union[Path, str],
+        remoteFilePath: str,
+        mimeType: Optional[str] = None
+    ) -> Optional[Self]:
+
         if mimeType is None:
             # If guessing fails, fallback to binary type
             try:
@@ -152,7 +137,7 @@ def create(cls, experimentId: int, localFilePath: str, remoteFilePath: str, mime
         if response.hasFailed():
             return None
 
-        artifact = Artifact.decode(response.json)
+        artifact = cls.decode(response.json)
         artifact.experimentId = experimentId
 
         return artifact
@@ -166,17 +151,13 @@ def download(self) -> bool:
             bool -> False if response has failed, True otherwise
         """
 
-        artifactsFolder = FolderManager.instance().getArtifactsFolder(self.experimentId)
-        if not artifactsFolder.exists():
-            artifactsFolder.mkdir(parents = True, exist_ok = True)
-
         return not networkManager.genericDownload(
             f"artifact/download-file?path={self.remoteFilePath}&model_queue_id={self.experimentId}",
             str(self.localFilePath)
         ).hasFailed()
 
     @classmethod
-    def fetchAll(cls, experimentId: int, path: Optional[str] = None, recursive: bool = False) -> List[Artifact]:
+    def fetchAll(cls, experimentId: int, path: Optional[str] = None, recursive: bool = False) -> List[Self]:
         """
             Fetch all Artifacts from Coretex.ai for the specified experiment
 
@@ -206,7 +187,7 @@ def fetchAll(cls, experimentId: int, path: Optional[str] = None, recursive: bool
         if response.hasFailed():
             return []
 
-        artifacts = [Artifact.decode(element) for element in response.json]
+        artifacts = [cls.decode(element) for element in response.json]
 
         for artifact in artifacts:
             artifact.experimentId = experimentId

coretex/coretex/experiment/experiment.py

@@ -15,9 +15,8 @@
 #     You should have received a copy of the GNU Affero General Public License
 #     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from typing import Final, Optional, Any, List, Dict, Union, Tuple, TypeVar, Generic
+from typing import Optional, Any, List, Dict, Union, Tuple, TypeVar, Generic
 from typing_extensions import Self
-from threading import Lock
 from zipfile import ZipFile
 from pathlib import Path
 
@@ -32,9 +31,9 @@
 from .parameters import ExperimentParameter, parseParameters
 from ..dataset import *
 from ..space import SpaceTask
+from ... import folder_manager
 from ...codable import KeyDescriptor
 from ...networking import networkManager, NetworkObject, RequestType, NetworkRequestError
-from ...folder_management import FolderManager
 
 
 DatasetType = TypeVar("DatasetType", bound = Dataset)
@@ -102,14 +101,14 @@ def parameters(self) -> Dict[str, Any]:
         return self.__parameters
 
     @property
-    def projectPath(self) -> str:
+    def projectPath(self) -> Path:
         """
             Returns
             -------
-            str -> Path for Experiment
+            Path -> Path for Experiment
         """
 
-        return FolderManager.instance().getTempFolder(str(self.id))
+        return folder_manager.temp / str(self.id)
 
     @property
     def dataset(self) -> DatasetType:
@@ -338,13 +337,19 @@ def downloadProject(self) -> bool:
 
         return not response.hasFailed()
 
-    def createArtifact(self, localFilePath: str, remoteFilePath: str, mimeType: Optional[str] = None) -> Optional[Artifact]:
+    def createArtifact(
+        self,
+        localFilePath: Union[Path, str],
+        remoteFilePath: str,
+        mimeType: Optional[str] = None
+    ) -> Optional[Artifact]:
+
         """
             Creates Artifact for the current Experiment on Coretex.ai
 
             Parameters
             ----------
-            localFilePath : str
+            localFilePath : Union[Path, str]
                 local path of Artifact file
             remoteFilePath : str
                 path of Artifact file on Coretex
@@ -398,6 +403,7 @@ def run(
         description: Optional[str] = None,
         parameters: Optional[List[Dict[str, Any]]] = None
     ) -> Self:
+
         """
             Starts an Experiment on Coretex.ai with the provided parameters