Object Store Get & API Cleanup (#443)

- Add new object store get command
- Fix object store help commands
- Further API cleanup

Author: Martin-Molinero

README.md

@@ -79,6 +79,7 @@ A locally-focused workflow (local development, local execution) with the CLI may
 - [`lean cloud object-store get`](#lean-cloud-object-store-get)
 - [`lean cloud object-store list`](#lean-cloud-object-store-list)
 - [`lean cloud object-store ls`](#lean-cloud-object-store-ls)
+- [`lean cloud object-store properties`](#lean-cloud-object-store-properties)
 - [`lean cloud object-store set`](#lean-cloud-object-store-set)
 - [`lean cloud optimize`](#lean-cloud-optimize)
 - [`lean cloud pull`](#lean-cloud-pull)
@@ -112,6 +113,7 @@ A locally-focused workflow (local development, local execution) with the CLI may
 - [`lean object-store get`](#lean-object-store-get)
 - [`lean object-store list`](#lean-object-store-list)
 - [`lean object-store ls`](#lean-object-store-ls)
+- [`lean object-store properties`](#lean-object-store-properties)
 - [`lean object-store set`](#lean-object-store-set)
 - [`lean optimize`](#lean-optimize)
 - [`lean project-create`](#lean-project-create)
@@ -450,6 +452,8 @@ Usage: lean cloud object-store delete [OPTIONS] KEY
 
   Delete a value from the organization's cloud object store.
 
+  :param key: The desired key name to delete.
+
 Options:
   --verbose  Enable debug logging
   --help     Show this message and exit.
@@ -459,16 +463,20 @@ _See code: [lean/commands/cloud/object_store/delete.py](lean/commands/cloud/obje
 
 ### `lean cloud object-store get`
 
-Get a value from the organization's cloud object store.
+Download an object store value to disk from the organization's cloud object store.
 
 ```
-Usage: lean cloud object-store get [OPTIONS] KEY
+Usage: lean cloud object-store get [OPTIONS] [KEY]...
+
+  Download an object store value to disk from the organization's cloud object store.
 
-  Get a value from the organization's cloud object store.
+  :param key: The desired key to fetch, multiple can be provided.
 
 Options:
-  --verbose  Enable debug logging
-  --help     Show this message and exit.
+  --destination-folder TEXT  The destination folder to download the object store values, if not provided will use to
+                             current directory
+  --verbose                  Enable debug logging
+  --help                     Show this message and exit.
 ```
 
 _See code: [lean/commands/cloud/object_store/get.py](lean/commands/cloud/object_store/get.py)_
@@ -482,6 +490,8 @@ Usage: lean cloud object-store list [OPTIONS] [KEY]
 
   List all values for the given root key in the organization's cloud object store.
 
+  :param key: The desired root key to list.
+
 Options:
   --verbose  Enable debug logging
   --help     Show this message and exit.
@@ -498,13 +508,33 @@ Usage: lean cloud object-store ls [OPTIONS] [KEY]
 
   List all values for the given root key in the organization's cloud object store.
 
+  :param key: The desired root key to list.
+
 Options:
   --verbose  Enable debug logging
   --help     Show this message and exit.
 ```
 
 _See code: [lean/commands/cloud/object_store/ls.py](lean/commands/cloud/object_store/ls.py)_
 
+### `lean cloud object-store properties`
+
+Get a value properties from the organization's cloud object store.
+
+```
+Usage: lean cloud object-store properties [OPTIONS] KEY
+
+  Get a value properties from the organization's cloud object store.
+
+  :param key: The desired key to fetch the properties for.
+
+Options:
+  --verbose  Enable debug logging
+  --help     Show this message and exit.
+```
+
+_See code: [lean/commands/cloud/object_store/properties.py](lean/commands/cloud/object_store/properties.py)_
+
 ### `lean cloud object-store set`
 
 Sets the data to the given key in the organization's cloud object store.
@@ -1427,6 +1457,22 @@ Options:
 
 _See code: [lean/commands/object_store/ls.py](lean/commands/object_store/ls.py)_
 
+### `lean object-store properties`
+
+Opens the local storage directory in the file explorer.
+
+```
+Usage: lean object-store properties [OPTIONS]
+
+  Opens the local storage directory in the file explorer.
+
+Options:
+  --verbose  Enable debug logging
+  --help     Show this message and exit.
+```
+
+_See code: [lean/commands/object_store/properties.py](lean/commands/object_store/properties.py)_
+
 ### `lean object-store set`
 
 Opens the local storage directory in the file explorer.

lean/commands/cloud/object_store/__init__.py

@@ -16,9 +16,10 @@
 from lean.commands.cloud.object_store.set import set
 from lean.commands.cloud.object_store.list import list
 from lean.commands.cloud.object_store.delete import delete
+from lean.commands.cloud.object_store.properties import properties
 
 object_store.add_command(get)
 object_store.add_command(set)
 object_store.add_command(list)
 object_store.add_command(delete)
-
+object_store.add_command(properties)

lean/commands/cloud/object_store/delete.py

@@ -22,8 +22,9 @@
 def delete(key: str) -> str:
     """
     Delete a value from the organization's cloud object store.
-    
+
+    :param key: The desired key name to delete.
     """
     organization_id = container.organization_manager.try_get_working_organization_id()
     api_client = container.api_client
-    api_client.object_store.delete(key, organization_id)
+    api_client.object_store.delete(key, organization_id)

lean/commands/cloud/object_store/get.py

@@ -11,45 +11,50 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from click import command, argument
+from uuid import uuid4
+from os import path, getcwd, unlink, mkdir
+
+from click import command, option, argument
 from lean.click import LeanCommand
 from lean.container import container
 
 
 @command(cls=LeanCommand)
-@argument("key", type=str)
-def get(key: str) -> str:
+@argument("key", type=str, nargs=-1)
+@option("--destination-folder", type=str, default="",
+          help=f"The destination folder to download the object store values,"
+               f" if not provided will use to current directory")
+def get(key: [str], destination_folder: str):
     """
-    Get a value from the organization's cloud object store.
+    Download an object store value to disk from the organization's cloud object store.
 
+    :param key: The desired key to fetch, multiple can be provided.
     """
     organization_id = container.organization_manager.try_get_working_organization_id()
     api_client = container.api_client
     logger = container.logger
-    data = api_client.object_store.get(key, organization_id)
-
-    try:
-        headers = ["size", "modified", "key", "preview"]
-        display_headers = ["Bytes", "Modified", "Filename", "Preview"]
-        data_row = []
-        for header in headers:
-            if header == "preview":
-                value = str(data["metadata"].get(header, "N/A"))
-                data_row.append(_clean_up_preview(value))
-            else:
-                value = str(data["metadata"].get(header, ""))
-                data_row.append(value)
-        all_rows = [display_headers] + [data_row]
-        column_widths = [max(len(row[i]) for row in all_rows) for i in range(len(all_rows[0]))]
-        for row in all_rows:
-            logger.info("  ".join(value.ljust(width) for value, width in zip(row, column_widths)))
-    except KeyError as e:
-        logger.error(f"Key {key} not found.")
-    except Exception as e:
-        logger.error(f"Error: {e}")
-
-
-def _clean_up_preview(preview: str) -> str:
-    return preview.rstrip()[:10]
 
+    logger.info(f"Fetching object store download url")
+    url = api_client.object_store.get(key, organization_id, logger)
+    if not destination_folder:
+        destination_folder = getcwd()
+
+    if not path.exists(destination_folder):
+        mkdir(destination_folder)
+
+    temp_file = path.join(destination_folder, f"{str(uuid4())}.zip")
+
+    with logger.transient_progress() as progress:
+        progress.add_task(f"Start downloading keys into {temp_file}:", total=None)
+        logger.debug(f"Downloading: {url}")
+
+        api_client.data.download_url(url, temp_file, lambda advance: None)
+
+    logger.info(f"Unzipping object store keys values into: '{destination_folder}'")
+    from zipfile import ZipFile
+    with ZipFile(temp_file, 'r') as zip_ref:
+        zip_ref.extractall(destination_folder)
 
+    if path.exists(temp_file):
+        logger.debug(f"Deleting temp file: '{temp_file}'")
+        unlink(temp_file)

lean/commands/cloud/object_store/list.py

@@ -19,9 +19,11 @@
 
 @object_store.command(cls=LeanCommand, name="list", aliases=["ls"])
 @argument("key", type=str, default="/")
-def list(key: str) -> str:
+def list(key: str):
     """
     List all values for the given root key in the organization's cloud object store.
+
+    :param key: The desired root key to list.
     """
     organization_id = container.organization_manager.try_get_working_organization_id()
     api_client = container.api_client
@@ -41,4 +43,4 @@ def list(key: str) -> str:
     except KeyError as e:
         logger.error(f"Key {key} not found.")
     except Exception as e:
-        logger.error(f"Error: {e}")
+        logger.error(f"Error: {e}")

lean/commands/cloud/object_store/properties.py

@@ -0,0 +1,54 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from click import command, argument
+from lean.click import LeanCommand
+from lean.container import container
+
+
+@command(cls=LeanCommand)
+@argument("key", type=str)
+def properties(key: str):
+    """
+    Get a value properties from the organization's cloud object store.
+
+    :param key: The desired key to fetch the properties for.
+    """
+    organization_id = container.organization_manager.try_get_working_organization_id()
+    api_client = container.api_client
+    logger = container.logger
+    data = api_client.object_store.properties(key, organization_id)
+
+    try:
+        headers = ["size", "modified", "key", "preview"]
+        display_headers = ["Bytes", "Modified", "Filename", "Preview"]
+        data_row = []
+        for header in headers:
+            if header == "preview":
+                value = str(data["metadata"].get(header, "N/A"))
+                data_row.append(_clean_up_preview(value))
+            else:
+                value = str(data["metadata"].get(header, ""))
+                data_row.append(value)
+        all_rows = [display_headers] + [data_row]
+        column_widths = [max(len(row[i]) for row in all_rows) for i in range(len(all_rows[0]))]
+        for row in all_rows:
+            logger.info("  ".join(value.ljust(width) for value, width in zip(row, column_widths)))
+    except KeyError as e:
+        logger.error(f"Key {key} not found.")
+    except Exception as e:
+        logger.error(f"Error: {e}")
+
+
+def _clean_up_preview(preview: str) -> str:
+    return preview.rstrip()[:10]

lean/commands/cloud/object_store/set.py

@@ -31,4 +31,4 @@ def set(key: str, path: Path) -> None:
     api_client = container.api_client
     with open(path, "rb") as file:
         bytes_data: bytes = file.read()
-    api_client.object_store.set(key, bytes_data, organization_id)
+    api_client.object_store.set(key, bytes_data, organization_id)

lean/commands/object_store/__init__.py

@@ -16,9 +16,10 @@
 from lean.commands.object_store.set import set
 from lean.commands.object_store.list import list
 from lean.commands.object_store.delete import delete
+from lean.commands.object_store.properties import properties
 
 object_store.add_command(get)
 object_store.add_command(set)
 object_store.add_command(list)
 object_store.add_command(delete)
-
+object_store.add_command(properties)

lean/commands/object_store/properties.py

@@ -0,0 +1,26 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from click import command
+from lean.click import LeanCommand
+from lean.container import container
+from lean.components.util.object_store_helper import open_storage_directory_in_explorer
+
+
+@command(cls=LeanCommand)
+def properties() -> str:
+    """
+    Opens the local storage directory in the file explorer.
+    """
+    open_storage_directory_in_explorer(container.lean_config_manager)