diff --git a/jgscm/__init__.py b/jgscm/__init__.py index 2fd571c..ae7d328 100644 --- a/jgscm/__init__.py +++ b/jgscm/__init__.py @@ -1,4 +1,5 @@ import base64 +from itertools import islice import os import uuid @@ -125,7 +126,7 @@ def list_checkpoints(self, path): checkpoints = [{ "id": os.path.splitext(file.path)[0][-36:], "last_modified": file.updated, - } for file in it] + } for file in islice(it, self.parent.max_list_size)] except NotFound: return [] checkpoints.sort(key=lambda c: c["last_modified"], reverse=True) @@ -161,7 +162,7 @@ class GoogleStorageContentManager(ContentsManager): "for authorization. If you do not set this parameter, " "gcloud will be OK if the default project exists." ) - max_list_size = Int(1024, config=True, help="list_blobs() limit") + max_list_size = Int(128, config=True, help="list_blobs() limit") cache_buckets = Bool(True, config=True, help="Value indicating whether to cache the bucket " "objects for faster operations.") @@ -340,7 +341,7 @@ def delete_file(self, path): return it = bucket.list_blobs(prefix=bucket_path, delimiter="/", max_results=self.max_list_size) - files = list(it) + files = list(islice(it, self.max_list_size)) folders = it.prefixes bucket.delete_blobs(files) for folder in folders: @@ -367,7 +368,7 @@ def rename_file(self, old_path, new_path): new_bucket_path += "/" it = old_bucket.list_blobs(prefix=old_bucket_path, delimiter="/", max_results=self.max_list_size) - old_blobs = list(it) + old_blobs = list(islice(it, self.max_list_size)) folders = it.prefixes for ob in old_blobs: old_bucket.rename_blob( @@ -388,7 +389,7 @@ def rename_file(self, old_path, new_path): new_bucket_path += "/" it = old_bucket.list_blobs(prefix=old_bucket_path, delimiter="/", max_results=self.max_list_size) - old_blobs = list(it) + old_blobs = list(islice(it, self.max_list_size)) folders = it.prefixes for ob in old_blobs: old_bucket.copy_blob(ob, new_bucket, new_bucket_path + @@ -553,7 +554,7 @@ def _fetch(self, path, content=True): it = bucket.list_blobs(prefix=bucket_path, delimiter="/", max_results=max_list_size) try: - files = list(it) + files = list(islice(it, max_list_size)) except BrokenPipeError: return self._fetch(path, content) except NotFound: diff --git a/setup.py b/setup.py index ef92598..4ffe0af 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="jgscm", description="Jupyter Google Cloud Storage ContentsManager", - version="0.1.4", + version="0.1.5", license="MIT", author="Vadim Markovtsev", author_email="vadim@sourced.tech",