Raise generic BackendError (#87)

* add BackendError class

* DOC: add BackendError

* raise backend error

* fix comment

* clean up

* DOC: add docstring example

* TST: full code coverage

* TST: set os dependent coverage configuration file

* Update .github/workflows/test.yml

Co-authored-by: Hagen Wierstorf <[email protected]>

* TST: try runner.os

* DOC: ls() remove FileNotFoundError

* DOC: latest_version() update raises section

* Update audbackend/core/backend.py

Co-authored-by: Hagen Wierstorf <[email protected]>

* DOC: put_archive update raises

* DOC: put_archive() fix order

* Artifactory._exists(): do not catch errors

* TST: test BackendError with all functions

---------

Co-authored-by: Hagen Wierstorf <[email protected]>

Author: frankenjoe

.coveragerc.Linux

@@ -0,0 +1,4 @@
+[report]
+exclude_lines =
+    pragma: no cover
+    pragma: no Linux cover

.coveragerc.Windows

@@ -0,0 +1,4 @@
+[report]
+exclude_lines =
+    pragma: no cover
+    pragma: no Windows cover

.coveragerc.macOS

@@ -0,0 +1,4 @@
+[report]
+exclude_lines =
+    pragma: no cover
+    pragma: no macOS cover

.github/workflows/test.yml

@@ -43,7 +43,7 @@ jobs:
         ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }}
         ARTIFACTORY_API_KEY: ${{ secrets.ARTIFACTORY_API_KEY }}
       run: |
-        python -m pytest
+        python -m pytest --cov-config=.coveragerc.${{ runner.os }}
 
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3

audbackend/__init__.py

@@ -5,6 +5,7 @@
 )
 from audbackend.core.artifactory import Artifactory
 from audbackend.core.backend import Backend
+from audbackend.core.errors import BackendError
 from audbackend.core.filesystem import FileSystem
 from audbackend.core.repository import Repository
 from audbackend.core.utils import md5

audbackend/core/artifactory.py

@@ -1,11 +1,10 @@
-import dohq_artifactory
 import os
 
-import requests
 import typing
 
 import audfactory
 
+from audbackend.core import utils
 from audbackend.core.backend import Backend
 
 
@@ -43,10 +42,8 @@ def _exists(
     ) -> bool:
         r"""Check if file exists on backend."""
         path = self._path(path, version)
-        try:
-            return audfactory.path(path).exists()
-        except self._non_existing_path_error:
-            return False
+        path = audfactory.path(path)
+        return path.exists()
 
     def _folder(
             self,
@@ -82,18 +79,15 @@ def _ls(
             self,
             folder: str,
     ):
-        r"""List all files under folder.
-
-        Return an empty list if no files match or folder does not exist.
+        r"""List all files under folder."""
 
-        """
         folder = self._folder(folder)
-
         folder = audfactory.path(folder)
-        try:
-            paths = [str(x) for x in folder.glob("**/*") if x.is_file()]
-        except self._non_existing_path_error:  # pragma: nocover
-            paths = []
+
+        if not folder.exists():
+            utils.raise_file_not_found_error(str(folder))
+
+        paths = [str(x) for x in folder.glob("**/*") if x.is_file()]
 
         # <host>/<repository>/<folder>/<name>
         # ->
@@ -172,23 +166,3 @@ def _versions(
         vs = [v for v in vs if self._exists(path, v)]
 
         return vs
-
-    _non_existing_path_error = (
-            RuntimeError,
-            requests.exceptions.HTTPError,
-            dohq_artifactory.exception.ArtifactoryException,
-    )
-    r"""Error expected for non-existing paths.
-
-    If a user has no permission to a given path
-    or the path does not exists
-    :func:`audfactory.path` might return a
-    ``RuntimeError``,
-    ``requests.exceptions.HTTPError``
-    for ``dohq_artifactory<0.8``
-    or
-    ``artifactory.exception.ArtifactoryException``
-    for ``dohq_artifactory>=0.8``.
-    So we better catch all of them.
-
-    """

audbackend/core/backend.py

@@ -55,7 +55,7 @@ def checksum(
             MD5 checksum
 
         Raises:
-            FileNotFoundError: if file does not exist on backend
+            BackendError: if an error is raised on the backend
             ValueError: if ``path`` contains invalid character
 
         Examples:
@@ -64,10 +64,12 @@ def checksum(
 
         """
         utils.check_path_for_allowed_chars(path)
-        if not self._exists(path, version):
-            utils.raise_file_not_found_error(path, version=version)
 
-        return self._checksum(path, version)
+        return utils.call_function_on_backend(
+            self._checksum,
+            path,
+            version,
+        )
 
     def _exists(
             self,
@@ -92,6 +94,7 @@ def exists(
             ``True`` if file exists
 
         Raises:
+            BackendError: if an error is raised on the backend
             ValueError: if ``path`` contains invalid character
 
         Examples:
@@ -101,7 +104,11 @@ def exists(
         """
         utils.check_path_for_allowed_chars(path)
 
-        return self._exists(path, version)
+        return utils.call_function_on_backend(
+            self._exists,
+            path,
+            version,
+        )
 
     def get_archive(
             self,
@@ -129,7 +136,7 @@ def get_archive(
             extracted files
 
         Raises:
-            FileNotFoundError: if archive does not exist on backend
+            BackendError: if an error is raised on the backend
             FileNotFoundError: if ``tmp_root`` does not exist
             PermissionError: if the user lacks write permissions
                 for ``dst_path``
@@ -195,7 +202,7 @@ def get_file(
             full path to local file
 
         Raises:
-            FileNotFoundError: if ``src_path`` does not exist on backend
+            BackendError: if an error is raised on the backend
             PermissionError: if the user lacks write permissions
                 for ``dst_path``
             ValueError: if ``src_path`` contains invalid character
@@ -210,13 +217,25 @@ def get_file(
 
         """
         utils.check_path_for_allowed_chars(src_path)
-        if not self._exists(src_path, version):
-            utils.raise_file_not_found_error(src_path, version=version)
 
-        dst_path = audeer.safe_path(dst_path)
-        audeer.mkdir(os.path.dirname(dst_path))
-
-        self._get_file(src_path, dst_path, version, verbose)
+        dst_path = audeer.path(dst_path)
+        dst_root = os.path.dirname(dst_path)
+
+        audeer.mkdir(dst_root)
+        if (
+                not os.access(dst_root, os.W_OK) or
+                (os.path.exists(dst_path) and not os.access(dst_root, os.W_OK))
+        ):  # pragma: no Windows cover
+            msg = f"Permission denied: '{dst_path}'"
+            raise PermissionError(msg)
+
+        utils.call_function_on_backend(
+            self._get_file,
+            src_path,
+            dst_path,
+            version,
+            verbose,
+        )
 
         return dst_path
 
@@ -264,7 +283,8 @@ def latest_version(
             version string
 
         Raises:
-            RuntimeError: if ``path`` does not exist on backend
+            BackendError: if an error is raised on the backend
+            RuntimeError: if no version is found
             ValueError: if ``path`` contains invalid character
 
         Examples:
@@ -288,11 +308,7 @@ def _ls(
             self,
             folder: str,
     ) -> typing.List[typing.Tuple[str, str, str]]:  # pragma: no cover
-        r"""List all files under folder.
-
-        Return an empty list if no files match or folder does not exist.
-
-        """
+        r"""List all files under folder."""
         raise NotImplementedError()
 
     def ls(
@@ -323,7 +339,7 @@ def ls(
             list of tuples (path, version)
 
         Raises:
-            FileNotFoundError: if ``folder`` does not exist
+            BackendError: if an error is raised on the backend
             ValueError: if ``folder`` contains invalid character
 
         Examples:
@@ -340,16 +356,9 @@ def ls(
         utils.check_path_for_allowed_chars(folder)
         if not folder.endswith('/'):
             folder += '/'
-        paths = self._ls(folder)
+        paths = utils.call_function_on_backend(self._ls, folder)
         paths = sorted(paths)
 
-        if len(paths) == 0:
-            if folder == '/':
-                # special case that there are no files on the backend
-                return []
-            else:
-                utils.raise_file_not_found_error(folder)
-
         if pattern:
             paths = [(p, v) for p, v in paths if fnmatch.fnmatch(p, pattern)]
 
@@ -399,10 +408,12 @@ def put_archive(
             verbose: show debug messages
 
         Raises:
-            FileNotFoundError: if one or more files do not exist
-            FileNotFoundError: if ``tmp_root`` does not exist
-            ValueError: if ``dst_path`` contains invalid character
+            BackendError: if an error is raised on the backend
+            FileNotFoundError: if ``src_root``,
+                ``tmp_root``,
+                or one or more ``files`` do not exist
             RuntimeError: if extension of ``dst_path`` is not supported
+            ValueError: if ``dst_path`` contains invalid character
 
         Examples:
             >>> backend.exists('a.tar.gz', '1.0.0')
@@ -414,16 +425,23 @@ def put_archive(
 
         """
         utils.check_path_for_allowed_chars(dst_path)
-        src_root = audeer.safe_path(src_root)
+        src_root = audeer.path(src_root)
+
+        if not os.path.exists(src_root):
+            utils.raise_file_not_found_error(src_root)
 
-        if isinstance(files, str):
-            files = [files]
+        files = audeer.to_list(files)
 
         for file in files:
             path = os.path.join(src_root, file)
             if not os.path.exists(path):
                 utils.raise_file_not_found_error(path)
 
+        if tmp_root is not None:
+            tmp_root = audeer.path(tmp_root)
+            if not os.path.exists(tmp_root):
+                utils.raise_file_not_found_error(tmp_root)
+
         with tempfile.TemporaryDirectory(dir=tmp_root) as tmp:
 
             archive = audeer.path(tmp, os.path.basename(dst_path))
@@ -475,6 +493,7 @@ def put_file(
             file path on backend
 
         Raises:
+            BackendError: if an error is raised on the backend
             FileNotFoundError: if ``src_path`` does not exist
             ValueError: if ``dst_path`` contains invalid character
 
@@ -493,10 +512,16 @@ def put_file(
 
         # skip if file with same checksum already exists
         if not (
-            self._exists(dst_path, version)
-            and self._checksum(dst_path, version) == utils.md5(src_path)
+            self.exists(dst_path, version)
+            and self.checksum(dst_path, version) == utils.md5(src_path)
         ):
-            self._put_file(src_path, dst_path, version, verbose)
+            utils.call_function_on_backend(
+                self._put_file,
+                src_path,
+                dst_path,
+                version,
+                verbose,
+            )
 
     def _remove_file(
             self,
@@ -518,7 +543,7 @@ def remove_file(
             version: version string
 
         Raises:
-            FileNotFoundError: if ``path`` does not exist on backend
+            BackendError: if an error is raised on the backend
             ValueError: if ``path`` contains invalid character
 
         Examples:
@@ -530,10 +555,12 @@ def remove_file(
 
         """
         utils.check_path_for_allowed_chars(path)
-        if not self._exists(path, version):
-            utils.raise_file_not_found_error(path, version=version)
 
-        path = self._remove_file(path, version)
+        utils.call_function_on_backend(
+            self._remove_file,
+            path,
+            version,
+        )
 
     @property
     def sep(self) -> str:
@@ -587,6 +614,7 @@ def versions(
             list of versions in ascending order
 
         Raises:
+            BackendError: if an error is raised on the backend
             ValueError: if ``path`` contains invalid character
 
         Examples:
@@ -596,6 +624,9 @@ def versions(
         """
         utils.check_path_for_allowed_chars(path)
 
-        vs = self._versions(path)
+        vs = utils.call_function_on_backend(
+            self._versions,
+            path,
+        )
 
         return audeer.sort_versions(vs)

audbackend/core/errors.py

@@ -0,0 +1,25 @@
+class BackendError(Exception):
+    r"""Wrapper for any error raised on the backend.
+
+    Args:
+        exception: exception raised by backend
+
+    Examples:
+        >>> try:
+        ...     backend.checksum('does-not-exist', '1.0.0')
+        ... except BackendError as ex:
+        ...     ex.exception
+        FileNotFoundError(2, 'No such file or directory')
+
+    """
+    def __init__(
+            self,
+            exception: Exception,
+    ):
+        self.exception = exception
+        r"""Exception raised by backend."""
+
+        super().__init__(
+            'An exception was raised by the backend, '
+            'please see stack trace for further information.'
+        )