Lazy loading of inspector tree exposes JSON briefly

[jdbcode]

When expanding a node for the first time, the JSON blob is briefly exposed before parsed to tree, and if long, extends the panel across the whole map.

The tree building and opening handler are a lot of code and maybe more complicated than needed. Is the lazy loading beneficial when there are 1000's of elements vs rendering the entire tree? A foundation for building the entire tree follows. It still needs to be tested against objects with 1000's of elements to see how fast it will render and respond. If it is performant, we should maybe consider using it, to simplify and avoid exposed JSON. If it is slow to render and not very responsive, we need to figure out how to not expose the JSON in the current implementation.

!pip install -q ipytree
from ipytree import Tree, Node
from google.colab import output
output.enable_custom_widget_manager()

def create_tree(data, name=None):
    if name is None:
        name = type(data).__name__
    node = Node(f'{name}: ({len(data)} elements)', show_icon=False, opened=False)
    if isinstance(data, dict):
        for key, value in data.items():
            if isinstance(value, (dict, list)):
                node.add_node(create_tree(value, key))
            else:
                child_node = Node(f'{key}: {value}', show_icon=False)
                node.add_node(child_node)
    elif isinstance(data, list):
        for index, item in enumerate(data):
            if isinstance(item, (dict, list)):
                node.add_node(create_tree(item, f'{index}'))
            else:
                child_node = Node(f'{index}: {item}', show_icon=False)
                node.add_node(child_node)
    return node

data = {
  "type": "Image",
  "bands": [
    {
      "id": "B1",
      "data_type": {
        "type": "PixelType",
        "precision": "float"
      },
      "dimensions": [
        9161,
        9161
      ],
      "crs": "EPSG:32628",
      "crs_transform": [
        30,
        0,
        341085,
        0,
        -30,
        8808015
      ]
    },
    {
      "id": "B2",
      "data_type": {
        "type": "PixelType",
        "precision": "float"
      },
      "dimensions": [
        9161,
        9161
      ],
      "crs": "EPSG:32628",
      "crs_transform": [
        30,
        0,
        341085,
        0,
        -30,
        8808015
      ]
    },
  ]
}

tree = Tree(animation=0)
tree.add_node(create_tree(data))
tree

[jdbcode]

The proposed function above (that renders the entire tree) won't work. It's slow to load a single Landsat 9 TOA image's metadata and, at least in free Colab, won't even render the tree with two images in the collection. Test these data with above function - collection limit set at 2:

data = ee.ImageCollection("LANDSAT/LC09/C02/T1_TOA").limit(2).getInfo()

tree = Tree(animation=0)
tree.add_node(create_tree(data))
tree

[jdbcode]

Maybe in the current implementation the JSON data can be stored as an added attribute, like add a json parameter to the Node class ~e.g.:

class CustomNode(Node):
    def __init__(self, name=None, children=None, json=None):
        super().__init__(name=name, children=children)
        self.json = json

Instead of storing the JSON data as the name attribute, store it in the new json attribute and use an empty string or <em>Loading</em> as the name attribute. When a person opens a node, fetch the JSON and update the name.

[jdbcode]

Here is a potential fix. It adds a Node subclass with an ee_data attribute for storing the JSON data (MyNode is not a good name, need to change).

class MyNode(ipytree.Node):
    def __init__(self, *args, ee_data=None, **kwargs):
        super().__init__(*args, **kwargs)
        self.ee_data = ee_data

The StructureTree class is modified so that the name of "JSON" nodes is Loading and JSON data is stored in the ee_data attribute.

class StructureTree(ipytree.Tree):

    JSON_PREFIX = 'JSON: '

    def __init__(self, data, **kwargs):
        self.data = data
        super().__init__(StructureTree.__ipytreeify(data))

    @staticmethod
    def __ipytreeify(data) -> tuple:
        """Return a sequence of nodes"""

        def is_json(in_str):
            '''Determines if a string is JSON.'''
            return bool(in_str) and in_str.startswith(StructureTree.JSON_PREFIX)

        def inline_style(x, color='#af00db'):
            '''Wrap a string with inline HTML styling.'''
            return f'<B><SPAN style="color:{color}">{x}</SPAN></B>'

        def handle_node_open(change):
            if change['new']:
                nodes_unpacked = []
                for node in change['owner'].nodes:
                    # If there no subnodes, try to populate the subnodes.
                    if len(node.nodes) == 0:
                        if is_json(node.ee_data):
                          unpacked_json = json.loads(node.ee_data[len(StructureTree.JSON_PREFIX):])
                          if isinstance(unpacked_json, list):
                              nodes_unpacked = StructureTree.__ipytreeify(unpacked_json)
                          elif isinstance(unpacked_json, dict):
                              nodes_unpacked = StructureTree.__ipytreeify(unpacked_json)
                          else:
                              raise
                        else: # ~is_json(node.name)
                          nodes_unpacked.append(node)
                        change['owner'].nodes = nodes_unpacked

        if isinstance(data, list):
            node_list = []
            for count, el in enumerate(data):
                if isinstance(el, list):
                    subnode = MyNode(
                        name=f'{inline_style("List")} ({len(el)} elements)',
                        nodes=[MyNode('<em>Loading</em>', ee_data=f'{StructureTree.JSON_PREFIX}{json.dumps(el)}', show_icon=False)],
                        opened=False,
                        show_icon=False)
                    subnode.observe(handle_node_open, names='opened')
                elif isinstance(el, dict):
                    subnode = MyNode(
                        name=f'{inline_style("Object")} ({len(el)} elements)',
                        nodes=[MyNode('<em>Loading</em>', ee_data=f'{StructureTree.JSON_PREFIX}{json.dumps(el)}', show_icon=False)],
                        opened=False,
                        show_icon=False)
                    subnode.observe(handle_node_open, names='opened')
                else:
                    subnode = MyNode(f'{el}', show_icon=False)
                node_list.append(subnode)
            return node_list
        elif isinstance(data, dict):
            node_list = []
            for key, value in data.items():
                if isinstance(value, list):
                    subnode = MyNode(
                        name=f'{inline_style(key)}: List ({len(value)} elements)',
                        nodes=[MyNode('<em>Loading</em>', ee_data=f'{StructureTree.JSON_PREFIX}{json.dumps(value)}', show_icon=False)],
                        opened=False,
                        show_icon=False)
                    subnode.observe(handle_node_open, names='opened')
                elif isinstance(value, dict):
                    subnode = MyNode(
                        name=f'{inline_style(key)}: Object ({len(value)} elements)',
                        nodes=[MyNode('<em>Loading</em>', ee_data=f'{StructureTree.JSON_PREFIX}{json.dumps(value)}', show_icon=False)],
                        opened=False,
                        show_icon=False)
                    subnode.observe(handle_node_open, names='opened')
                else:
                    subnode = MyNode(f'{inline_style(key)}: {value}', show_icon=False)
                node_list.append(subnode)
            return node_list
        else:
            return (data, )

Since ee_data default is None, is_json function needed to add a check for None so that the startswith string method does not bonk. I tested it with the same Landsat data from above and it works great. It also works for the map inspector demo in the 01_ipyleaflet.ipynb (some other bug seems to crop up when implemented in the map panel - also happens with current implementation, I'll file a separate issue).

[jdbcode]

Here is a refactored version of StructureTree from previous comment (still uses badly named MyNode class).

class MyNode(ipytree.Node):
    def __init__(self, *args, ee_data=None, **kwargs):
        super().__init__(*args, **kwargs)
        self.ee_data = ee_data

class StructureTree(ipytree.Tree):
    JSON_PREFIX = 'JSON: '

    def __init__(self, data, **kwargs):
        self.data = data
        super().__init__(StructureTree.__ipytreeify(data))

    @staticmethod
    def __ipytreeify(data) -> tuple:
        """Return a sequence of nodes"""

        def is_json(in_str):
            '''Determines if a string is JSON.'''
            return bool(in_str) and in_str.startswith(StructureTree.JSON_PREFIX)

        def inline_style(x, color='#af00db'):
            '''Wrap a string with inline HTML styling.'''
            return f'<B><SPAN style="color:{color}">{x}</SPAN></B>'

        def handle_node_open(change):
            if change['new']:
                nodes_unpacked = []
                for node in change['owner'].nodes:
                    if not node.nodes:
                        if is_json(node.ee_data):
                            unpacked_json = json.loads(node.ee_data[len(
                                StructureTree.JSON_PREFIX):])
                            nodes_unpacked = StructureTree.__ipytreeify(unpacked_json)
                        else:
                            nodes_unpacked.append(node)
                        change['owner'].nodes = nodes_unpacked

        NODE_PARAMS = {
            'show_icon': False,
            'opened': False,
            'observe': handle_node_open,
        }

        def create_loading_node(data):
            return MyNode('<em>Loading</em>',
                          ee_data=f'{StructureTree.JSON_PREFIX}{json.dumps(data)}',
                          **NODE_PARAMS)

        def create_node_name(label, modifer, data):
            return f'{inline_style(label)}{modifer} ({len(data)} elements)'

        node_list = []
        if isinstance(data, list):
            for count, el in enumerate(data):
                if isinstance(el, (list, dict)):
                    node_type = 'List' if isinstance(el, list) else 'Object'
                    subnode = MyNode(
                        name=create_node_name(node_type, '', el),
                        nodes=[create_loading_node(el)],
                        **NODE_PARAMS)
                    subnode.observe(handle_node_open, names='opened')
                else:
                    subnode = MyNode(f'{el}', **NODE_PARAMS)
                node_list.append(subnode)
        elif isinstance(data, dict):
            for key, value in data.items():
                if isinstance(value, (list, dict)):
                    node_type = ': List' if isinstance(value, list) else ': Object'
                    subnode = MyNode(
                        name=create_node_name(key, node_type, value),
                        nodes=[create_loading_node(value)],
                        **NODE_PARAMS)
                    subnode.observe(handle_node_open, names='opened')
                else:
                    subnode = MyNode(f'{inline_style(key)}: {value}', **NODE_PARAMS)
                node_list.append(subnode)
        else:
            node_list.append(MyNode(str(data), show_icon=False))

        return tuple(node_list)