diff --git a/coretex/coretex/sample/network_sample.py b/coretex/coretex/sample/network_sample.py index 9f4ae9a3..c6f632d2 100644 --- a/coretex/coretex/sample/network_sample.py +++ b/coretex/coretex/sample/network_sample.py @@ -122,12 +122,10 @@ def download(self, ignoreCache: bool = False) -> bool: bool -> False if response is failed, True otherwise """ - if os.path.exists(self.zipPath) and not ignoreCache: - return True - - response = networkManager.genericDownload( + response = networkManager.sampleDownload( endpoint = f"{self.__class__._endpoint()}/export?id={self.id}", - destination = self.zipPath + destination = self.zipPath, + ignoreCache = ignoreCache ) return not response.hasFailed() diff --git a/coretex/networking/network_manager_base.py b/coretex/networking/network_manager_base.py index 9c340db7..46047a64 100644 --- a/coretex/networking/network_manager_base.py +++ b/coretex/networking/network_manager_base.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Optional, Any, Dict, List +from typing import Optional, Any, Dict, List, Union from pathlib import Path from abc import ABC, abstractmethod from importlib.metadata import version as getLibraryVersion @@ -246,6 +246,66 @@ def genericDownload( return response + def sampleDownload( + self, + endpoint: str, + destination: Union[str, Path], + ignoreCache: bool, + parameters: Optional[Dict[str, Any]] = None, + retryCount: int = 0 + ) -> NetworkResponse: + """ + Downloads file to the given destination by chunks + + Parameters + ---------- + endpoint : str + API endpoint + destination : Union[str, Path] + path to save file + ignoreCache : bool + whether to overwrite local cache + parameters : Optional[dict[str, Any]] + request parameters (not required) + retryCount : int + number of function calls if request has failed, used + for internal retry mechanism + + Returns + ------- + NetworkResponse as response content to request + + Example + ------- + >>> from coretex import networkManager + \b + >>> response = networkManager.sampleDownload( + endpoint = "dummyObject/download", + destination = "path/to/destination/folder" + ) + >>> if response.hasFailed(): + print("Failed to download the file") + """ + + headers = self._requestHeader() + + if parameters is None: + parameters = {} + + networkResponse = self._requestManager.streamingDownload( + endpoint, + destination, + ignoreCache, + headers, + parameters + ) + + if self.shouldRetry(retryCount, networkResponse): + print(">> [Coretex] Retry count: {0}".format(retryCount)) + return self.sampleDownload(endpoint, destination, ignoreCache, parameters, retryCount + 1) + + return networkResponse + def genericUpload( self, endpoint: str, diff --git a/coretex/networking/network_response.py b/coretex/networking/network_response.py index c01ad540..a9462a01 100644 --- a/coretex/networking/network_response.py +++ b/coretex/networking/network_response.py @@ -57,7 +57,8 @@ def __init__(self, response: Response, endpoint: str): try: self.json = response.json() - except ValueError: + except (ValueError, RuntimeError): + # RuntimeError is present here to avoid the content_consumed error self.json = {} if not response.ok: diff --git a/coretex/networking/requests_manager.py b/coretex/networking/requests_manager.py index a24bd1f2..898ce89e 100644 --- a/coretex/networking/requests_manager.py +++ b/coretex/networking/requests_manager.py @@ -15,8 +15,9 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Final, Any, Optional, Dict, List +from typing import Final, Any, Optional, Dict, List, Union from contextlib import ExitStack +from pathlib import Path import logging @@ -205,6 +206,62 @@ def post( raise RequestFailedError + def streamingDownload( + self, + endpoint: str, + destinationPath: Union[str, Path], + ignoreCache: bool, + headers: Dict[str, str], + parameters: Dict[str, Any] + ) -> NetworkResponse: + """ + Downloads a file from endpoint to destinationPath in chunks of 8192 bytes + + Parameters + ---------- + endpoint : str + API endpoint + destination : Union[str, Path] + path to save file + ignoreCache : bool + whether to overwrite local cache + headers : Any + headers for get + parameters : int + json for get + + Returns + ------- + NetworkResponse -> NetworkResponse object as response content to request + """ + + with self.__session.get( + self.__url(endpoint), + stream = True, + headers = headers, + json = parameters + ) as response: + + response.raise_for_status() + + if isinstance(destinationPath, str): + destinationPath = Path(destinationPath) + + if destinationPath.is_dir(): + raise RuntimeError(">> [Coretex] Destination is a directory not a file") + + if destinationPath.exists(): + if int(response.headers["Content-Length"]) == destinationPath.stat().st_size and not ignoreCache: + return NetworkResponse(response, endpoint) + + destinationPath.unlink() + + with destinationPath.open("wb") as downloadedFile: + for chunk in response.iter_content(chunk_size = 8192): + downloadedFile.write(chunk) + + return NetworkResponse(response, endpoint) + def upload( self, endpoint: str,