diff --git a/metaflow/plugins/cards/card_cli.py b/metaflow/plugins/cards/card_cli.py index 2c0c8017744..021bb94f014 100644 --- a/metaflow/plugins/cards/card_cli.py +++ b/metaflow/plugins/cards/card_cli.py @@ -1,12 +1,19 @@ from metaflow.client import Task from metaflow import JSONType, namespace -from metaflow.exception import CommandException +from metaflow.util import resolve_identity +from metaflow.exception import ( + CommandException, + MetaflowNotFound, + MetaflowNamespaceMismatch, +) import webbrowser import re from metaflow._vendor import click import os import json +import uuid import signal +import inspect import random from contextlib import contextmanager from functools import wraps @@ -375,14 +382,45 @@ def wrapper(*args, **kwargs): return wrapper -def render_card(mf_card, task, timeout_value=None): - rendered_info = None +def update_card(mf_card, mode, task, data, timeout_value=None): + def _reload_token(): + if data["render_seq"] == "final": + # final data update should always trigger a card reload to show + # the final card, hence a different token for the final update + return "final" + elif mf_card.RELOAD_POLICY == mf_card.RELOAD_POLICY_ALWAYS: + return "render-seq-%s" % data["render_seq"] + elif mf_card.RELOAD_POLICY == mf_card.RELOAD_POLICY_NEVER: + return "never" + elif mf_card.RELOAD_POLICY == mf_card.RELOAD_POLICY_ONCHANGE: + return mf_card.reload_content_token(task, data) + + def _add_token_html(html): + if html is not None: + return html.replace(mf_card.RELOAD_POLICY_TOKEN, _reload_token()) + + def _add_token_json(json_msg): + if json_msg is not None: + return {"reload_token": _reload_token(), "data": json_msg} + + def _call(): + # compatibility with old render()-method that doesn't accept the data arg + new_render = "data" in inspect.getfullargspec(mf_card.render).args + if mode == "render": + if new_render: + return _add_token_html(mf_card.render(task, data)) + else: + return _add_token_html(mf_card.render(task)) + elif mode == "render_runtime": + return _add_token_html(mf_card.render_runtime(task, data)) + elif mode == "refresh": + return _add_token_json(mf_card.refresh(task, data)) + if timeout_value is None or timeout_value < 0: - rendered_info = mf_card.render(task) + return _call() else: with timeout(timeout_value): - rendered_info = mf_card.render(task) - return rendered_info + return _call() @card.command(help="create a HTML card") @@ -414,29 +452,61 @@ def render_card(mf_card, task, timeout_value=None): is_flag=True, help="Upon failing to render a card, render a card holding the stack trace", ) +@click.option( + "--id", + default=None, + show_default=True, + type=str, + help="ID of the card", +) @click.option( "--component-file", default=None, show_default=True, type=str, - help="JSON File with Pre-rendered components.(internal)", + help="JSON File with Pre-rendered components. (internal)", ) @click.option( - "--id", + "--mode", + default="render", + show_default=True, + type=str, + help="Rendering mode. (internal)", +) +@click.option( + "--data-file", default=None, show_default=True, type=str, - help="ID of the card", + help="JSON file containing data to be updated. (internal)", +) +@click.option( + "--card-uuid", + default=None, + show_default=True, + type=str, + help="Card UUID. (internal)", +) +@click.option( + "--delete-input-files", + default=False, + is_flag=True, + show_default=True, + help="Delete data-file and compontent-file after reading. (internal)", ) @click.pass_context def create( ctx, pathspec, + mode=None, type=None, options=None, timeout=None, component_file=None, + data_file=None, render_error_card=False, + card_uuid=None, + delete_input_files=None, id=None, ): card_id = id @@ -452,11 +522,26 @@ def create( graph_dict, _ = ctx.obj.graph.output_steps() + if card_uuid is None: + card_uuid = str(uuid.uuid4()).replace("-", "") + # Components are rendered in a Step and added via `current.card.append` are added here. component_arr = [] if component_file is not None: with open(component_file, "r") as f: component_arr = json.load(f) + # data is passed in as temporary files which can be deleted after use + if delete_input_files: + os.remove(component_file) + + # Load data to be refreshed for runtime cards + data = {} + if data_file is not None: + with open(data_file, "r") as f: + data = json.load(f) + # data is passed in as temporary files which can be deleted after use + if delete_input_files: + os.remove(data_file) task = Task(full_pathspec) from metaflow.plugins import CARDS @@ -500,7 +585,9 @@ def create( if mf_card: try: - rendered_info = render_card(mf_card, task, timeout_value=timeout) + rendered_info = update_card( + mf_card, mode, task, data, timeout_value=timeout + ) except: if render_error_card: error_stack_trace = str(UnrenderableCardException(type, options)) @@ -508,10 +595,10 @@ def create( raise UnrenderableCardException(type, options) # - if error_stack_trace is not None: + if error_stack_trace is not None and mode != "refresh": rendered_info = error_card().render(task, stack_trace=error_stack_trace) - if rendered_info is None and render_error_card: + if rendered_info is None and render_error_card and mode != "refresh": rendered_info = error_card().render( task, stack_trace="No information rendered From card of type %s" % type ) @@ -532,12 +619,20 @@ def create( card_id = None if rendered_info is not None: - card_info = card_datastore.save_card(save_type, rendered_info, card_id=card_id) - ctx.obj.echo( - "Card created with type: %s and hash: %s" - % (card_info.type, card_info.hash[:NUM_SHORT_HASH_CHARS]), - fg="green", - ) + if mode == "refresh": + card_datastore.save_data( + card_uuid, save_type, rendered_info, card_id=card_id + ) + ctx.obj.echo("Data updated", fg="green") + else: + card_info = card_datastore.save_card( + card_uuid, save_type, rendered_info, card_id=card_id + ) + ctx.obj.echo( + "Card created with type: %s and hash: %s" + % (card_info.type, card_info.hash[:NUM_SHORT_HASH_CHARS]), + fg="green", + ) @card.command() @@ -655,7 +750,6 @@ def list( as_json=False, file=None, ): - card_id = id if pathspec is None: list_many_cards( @@ -687,3 +781,89 @@ def list( show_list_as_json=as_json, file=file, ) + + +@card.command(help="Run local card viewer server") +@click.option( + "--run-id", + default=None, + show_default=True, + type=str, + help="Run ID of the flow", +) +@click.option( + "--port", + default=8324, + show_default=True, + type=int, + help="Port on which Metaflow card server will run", +) +@click.option( + "--namespace", + "user_namespace", + default=None, + show_default=True, + type=str, + help="Namespace of the flow", +) +@click.option( + "--max-cards", + default=30, + show_default=True, + type=int, + help="Maximum number of cards to be shown at any time by the server", +) +@click.pass_context +def server(ctx, run_id, port, user_namespace, max_cards): + from .card_server import create_card_server, CardServerOptions + user_namespace = resolve_identity() if user_namespace is None else user_namespace + run = _get_run_object(ctx.obj, run_id, user_namespace) + options = CardServerOptions( + run_object=run, + only_running=False, + follow_resumed=False, + flow_datastore=ctx.obj.flow_datastore, + max_cards=max_cards, + ) + create_card_server(options, port, ctx.obj) + + +def _get_run_object(obj, run_id, user_namespace): + from metaflow import Flow, Run, Task + + flow_name = obj.flow.name + try: + if run_id is not None: + namespace(None) + else: + _msg = "Searching for runs in namespace: %s" % user_namespace + obj.echo(_msg, fg="blue", bold=False) + namespace(user_namespace) + flow = Flow(pathspec=flow_name) + except MetaflowNotFound: + raise CommandException("No run found for *%s*." % flow_name) + + except MetaflowNamespaceMismatch: + raise CommandException( + "No run found for *%s* in namespace *%s*. You can switch the namespace using --namespace" + % (flow_name, user_namespace) + ) + + if run_id is None: + run_id = flow.latest_run.pathspec + + else: + assert len(run_id.split("/")) == 1, "run_id should be of the form " + run_id = "/".join([flow_name, run_id]) + + try: + run = Run(run_id) + except MetaflowNotFound: + raise CommandException("No run found for runid: *%s*." % run_id) + except MetaflowNamespaceMismatch: + raise CommandException( + "No run found for runid: *%s* in namespace *%s*. You can switch the namespace using --namespace" + % (run_id, user_namespace) + ) + obj.echo("Using run-id %s" % run_id, fg="blue", bold=False) + return run diff --git a/metaflow/plugins/cards/card_client.py b/metaflow/plugins/cards/card_client.py index 07800a347f3..f6babea3e1e 100644 --- a/metaflow/plugins/cards/card_client.py +++ b/metaflow/plugins/cards/card_client.py @@ -2,7 +2,7 @@ from metaflow.datastore import FlowDataStore from metaflow.metaflow_config import CARD_SUFFIX from .card_resolver import resolve_paths_from_task, resumed_info -from .card_datastore import CardDatastore +from .card_datastore import CardDatastore, CardNameSuffix from .exception import ( UnresolvableDatastoreException, IncorrectArguementException, @@ -57,6 +57,15 @@ def __init__( # Tempfile to open stuff in browser self._temp_file = None + def _get_data(self) -> Optional[dict]: + # currently an internal method to retrieve a card's data. + data_paths = self._card_ds.extract_data_paths( + card_type=self.type, card_hash=self.hash, card_id=self._card_id + ) + if len(data_paths) == 0: + return None + return self._card_ds.get_card_data(data_paths[0]) + def get(self) -> str: """ Retrieves the HTML contents of the card from the @@ -172,7 +181,7 @@ def _get_card(self, index): if index >= self._high: raise IndexError path = self._card_paths[index] - card_info = self._card_ds.card_info_from_path(path) + card_info = self._card_ds.card_info_from_path(path, suffix=CardNameSuffix.CARD) # todo : find card creation date and put it in client. return Card( self._card_ds, @@ -252,6 +261,7 @@ def get_cards( # Exception that the task argument should be of form `Task` or `str` raise IncorrectArguementException(_TYPE(task)) + origin_taskpathspec = None if follow_resumed: origin_taskpathspec = resumed_info(task) if origin_taskpathspec: diff --git a/metaflow/plugins/cards/card_creator.py b/metaflow/plugins/cards/card_creator.py new file mode 100644 index 00000000000..1a4ab3e5908 --- /dev/null +++ b/metaflow/plugins/cards/card_creator.py @@ -0,0 +1,236 @@ +import time +import subprocess +import tempfile +import json +import sys +import os +from metaflow import current + +ASYNC_TIMEOUT = 30 + + +def warning_message(message, logger=None, ts=False): + msg = "[@card WARNING] %s" % message + if logger: + logger(msg, timestamp=ts, bad=True) + + +class CardProcessManager: + """ + This class is responsible for managing the card creation processes. + + """ + + async_card_processes = { + # "carduuid": { + # "proc": subprocess.Popen, + # "started": time.time() + # } + } + + @classmethod + def _register_card_process(cls, carduuid, proc): + cls.async_card_processes[carduuid] = { + "proc": proc, + "started": time.time(), + } + + @classmethod + def _get_card_process(cls, carduuid): + proc_dict = cls.async_card_processes.get(carduuid, None) + if proc_dict is not None: + return proc_dict["proc"], proc_dict["started"] + return None, None + + @classmethod + def _remove_card_process(cls, carduuid): + if carduuid in cls.async_card_processes: + cls.async_card_processes[carduuid]["proc"].kill() + del cls.async_card_processes[carduuid] + + +class CardCreator: + + state_manager = None # of type `CardStateManager` + + def __init__( + self, + base_command=None, + pathspec=None, + # TODO Add a pathspec somewhere here so that it can instantiate correctly. + ): + self._base_command = base_command + self._pathspec = pathspec + + @property + def pathspec(self): + return self._pathspec + + def _dump_state_to_dict(self): + return { + "base_command": self._base_command, + "pathspec": self._pathspec, + } + + def create( + self, + card_uuid=None, + user_set_card_id=None, + runtime_card=False, + decorator_attributes=None, + card_options=None, + logger=None, + mode="render", + final=False, + component_serialzer=None, + fetch_latest_data=None, + ): + # warning_message("calling proc for uuid %s" % self._card_uuid, self._logger) + if mode != "render" and not runtime_card: + # silently ignore runtime updates for cards that don't support them + return + elif mode == "refresh": + # don't serialize components, which can be a somewhat expensive operation, + # if we are just updating data + component_strings = [] + else: + component_strings = component_serialzer() + data = fetch_latest_data(final=final) + runspec = "/".join(self._pathspec.split("/")[1:]) + self._run_cards_subprocess( + card_uuid, + user_set_card_id, + mode, + runspec, + decorator_attributes, + card_options, + component_strings, + logger, + data, + ) + + def _run_cards_subprocess( + self, + card_uuid, + user_set_card_id, + mode, + runspec, + decorator_attributes, + card_options, + component_strings, + logger, + data=None, + ): + components_file = data_file = None + wait = mode == "render" + + if len(component_strings) > 0: + # note that we can't delete temporary files here when calling the subprocess + # async due to a race condition. The subprocess must delete them + components_file = tempfile.NamedTemporaryFile( + "w", suffix=".json", delete=False + ) + json.dump(component_strings, components_file) + components_file.seek(0) + if data is not None: + data_file = tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) + json.dump(data, data_file) + data_file.seek(0) + + if self.state_manager is not None: + self.state_manager._save_card_state( + card_uuid, components=component_strings, data=data + ) + + cmd = [] + cmd += self._base_command + [ + "card", + "create", + runspec, + "--delete-input-files", + "--card-uuid", + card_uuid, + "--mode", + mode, + "--type", + decorator_attributes["type"], + # Add the options relating to card arguments. + # todo : add scope as a CLI arg for the create method. + ] + if card_options is not None and len(card_options) > 0: + cmd += ["--options", json.dumps(card_options)] + # set the id argument. + + if decorator_attributes["timeout"] is not None: + cmd += ["--timeout", str(decorator_attributes["timeout"])] + + if user_set_card_id is not None: + cmd += ["--id", str(user_set_card_id)] + + if decorator_attributes["save_errors"]: + cmd += ["--render-error-card"] + + if components_file is not None: + cmd += ["--component-file", components_file.name] + + if data_file is not None: + cmd += ["--data-file", data_file.name] + + response, fail = self._run_command( + cmd, + card_uuid, + os.environ, + timeout=decorator_attributes["timeout"], + wait=wait, + ) + # warning_message(str(cmd), logger) + if fail: + resp = "" if response is None else response.decode("utf-8") + logger( + "Card render failed with error : \n\n %s" % resp, + timestamp=False, + bad=True, + ) + + def _run_command(self, cmd, card_uuid, env, wait=True, timeout=None): + fail = False + timeout_args = {} + async_timeout = ASYNC_TIMEOUT + if timeout is not None: + async_timeout = int(timeout) + 10 + timeout_args = dict(timeout=int(timeout) + 10) + + if wait: + try: + rep = subprocess.check_output( + cmd, env=env, stderr=subprocess.STDOUT, **timeout_args + ) + except subprocess.CalledProcessError as e: + rep = e.output + fail = True + except subprocess.TimeoutExpired as e: + rep = e.output + fail = True + return rep, fail + else: + _async_proc, _async_started = CardProcessManager._get_card_process( + card_uuid + ) + if _async_proc and _async_proc.poll() is None: + if time.time() - _async_started > async_timeout: + CardProcessManager._remove_card_process(card_uuid) + else: + # silently refuse to run an async process if a previous one is still running + # and timeout hasn't been reached + return "".encode(), False + else: + CardProcessManager._register_card_process( + card_uuid, + subprocess.Popen( + cmd, + env=env, + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + ), + ) + return "".encode(), False diff --git a/metaflow/plugins/cards/card_datastore.py b/metaflow/plugins/cards/card_datastore.py index 59931592878..d8e5ff799f1 100644 --- a/metaflow/plugins/cards/card_datastore.py +++ b/metaflow/plugins/cards/card_datastore.py @@ -6,6 +6,7 @@ from hashlib import sha1 from io import BytesIO import os +import json import shutil from metaflow.plugins.datastores.local_storage import LocalStorage @@ -28,6 +29,16 @@ CardInfo = namedtuple("CardInfo", ["type", "hash", "id", "filename"]) +class CardNameSuffix: + DATA = "data.json" + CARD = "html" + + +class CardPathSuffix: + DATA = "runtime" + CARD = "cards" + + def path_spec_resolver(pathspec): splits = pathspec.split("/") splits.extend([None] * (4 - len(splits))) @@ -85,18 +96,24 @@ def __init__(self, flow_datastore, pathspec=None): self._run_id = run_id self._step_name = step_name self._pathspec = pathspec - self._temp_card_save_path = self._get_write_path(base_pth=TEMP_DIR_NAME) + self._temp_card_save_path = self._get_write_path( + base_pth=TEMP_DIR_NAME, suffix=CardPathSuffix.CARD + ) @classmethod - def get_card_location(cls, base_path, card_name, card_html, card_id=None): - chash = sha1(bytes(card_html, "utf-8")).hexdigest() + def get_card_location( + cls, base_path, card_name, uuid, card_id=None, suffix=CardNameSuffix.CARD + ): + chash = uuid if card_id is None: - card_file_name = "%s-%s.html" % (card_name, chash) + card_file_name = "%s-%s.%s" % (card_name, chash, suffix) else: - card_file_name = "%s-%s-%s.html" % (card_name, card_id, chash) + card_file_name = "%s-%s-%s.%s" % (card_name, card_id, chash, suffix) return os.path.join(base_path, card_file_name) - def _make_path(self, base_pth, pathspec=None, with_steps=False): + def _make_path( + self, base_pth, pathspec=None, with_steps=False, suffix=CardPathSuffix.CARD + ): sysroot = base_pth if pathspec is not None: # since most cards are at a task level there will always be 4 non-none values returned @@ -121,7 +138,7 @@ def _make_path(self, base_pth, pathspec=None, with_steps=False): step_name, "tasks", task_id, - "cards", + suffix, ] else: pth_arr = [ @@ -131,20 +148,33 @@ def _make_path(self, base_pth, pathspec=None, with_steps=False): run_id, "tasks", task_id, - "cards", + suffix, ] if sysroot == "" or sysroot is None: pth_arr.pop(0) return os.path.join(*pth_arr) - def _get_write_path(self, base_pth=""): - return self._make_path(base_pth, pathspec=self._pathspec, with_steps=True) + def _get_write_path(self, base_pth="", suffix=CardPathSuffix.CARD): + return self._make_path( + base_pth, pathspec=self._pathspec, with_steps=True, suffix=suffix + ) + + def _get_read_path(self, base_pth="", with_steps=False, suffix=CardPathSuffix.CARD): + # Data paths will always be under the path with steps + if suffix == CardPathSuffix.DATA: + return self._make_path( + base_pth=base_pth, + pathspec=self._pathspec, + with_steps=True, + suffix=suffix, + ) - def _get_read_path(self, base_pth="", with_steps=False): - return self._make_path(base_pth, pathspec=self._pathspec, with_steps=with_steps) + return self._make_path( + base_pth, pathspec=self._pathspec, with_steps=with_steps, suffix=suffix + ) @staticmethod - def card_info_from_path(path): + def card_info_from_path(path, suffix=CardNameSuffix.CARD): """ Args: path (str): The path to the card @@ -160,8 +190,8 @@ def card_info_from_path(path): if len(file_split) not in [2, 3]: raise Exception( - "Invalid card file name %s. Card file names should be of form TYPE-HASH.html or TYPE-ID-HASH.html" - % card_file_name + "Invalid file name %s. Card/Data file names should be of form TYPE-HASH.%s or TYPE-ID-HASH.%s" + % (card_file_name, suffix, suffix) ) card_type, card_hash, card_id = None, None, None @@ -170,10 +200,23 @@ def card_info_from_path(path): else: card_type, card_id, card_hash = file_split - card_hash = card_hash.split(".html")[0] + card_hash = card_hash.split("." + suffix)[0] return CardInfo(card_type, card_hash, card_id, card_file_name) - def save_card(self, card_type, card_html, card_id=None, overwrite=True): + def save_data(self, uuid, card_type, json_data, card_id=None): + card_file_name = card_type + loc = self.get_card_location( + self._get_write_path(suffix=CardPathSuffix.DATA), + card_file_name, + uuid, + card_id=card_id, + suffix=CardNameSuffix.DATA, + ) + self._backend.save_bytes( + [(loc, BytesIO(json.dumps(json_data).encode("utf-8")))], overwrite=True + ) + + def save_card(self, uuid, card_type, card_html, card_id=None, overwrite=True): card_file_name = card_type # TEMPORARY_WORKAROUND: FIXME (LATER) : Fix the duplication of below block in a few months. # Check file blame to understand the age of this temporary workaround. @@ -193,7 +236,11 @@ def save_card(self, card_type, card_html, card_id=None, overwrite=True): # It will also easily end up breaking the metaflow-ui (which maybe using a client from an older version). # Hence, we are writing cards to both paths so that we can introduce breaking changes later in the future. card_path_with_steps = self.get_card_location( - self._get_write_path(), card_file_name, card_html, card_id=card_id + self._get_write_path(suffix=CardPathSuffix.CARD), + card_file_name, + uuid, + card_id=card_id, + suffix=CardNameSuffix.CARD, ) if SKIP_CARD_DUALWRITE: self._backend.save_bytes( @@ -202,28 +249,31 @@ def save_card(self, card_type, card_html, card_id=None, overwrite=True): ) else: card_path_without_steps = self.get_card_location( - self._get_read_path(with_steps=False), + self._get_read_path(with_steps=False, suffix=CardPathSuffix.CARD), card_file_name, - card_html, + uuid, card_id=card_id, + suffix=CardNameSuffix.CARD, ) for cp in [card_path_with_steps, card_path_without_steps]: self._backend.save_bytes( [(cp, BytesIO(bytes(card_html, "utf-8")))], overwrite=overwrite ) - return self.card_info_from_path(card_path_with_steps) + return self.card_info_from_path( + card_path_with_steps, suffix=CardNameSuffix.CARD + ) def _list_card_paths(self, card_type=None, card_hash=None, card_id=None): # Check for new cards first card_paths = [] card_paths_with_steps = self._backend.list_content( - [self._get_read_path(with_steps=True)] + [self._get_read_path(with_steps=True, suffix=CardPathSuffix.CARD)] ) if len(card_paths_with_steps) == 0: card_paths_without_steps = self._backend.list_content( - [self._get_read_path(with_steps=False)] + [self._get_read_path(with_steps=False, suffix=CardPathSuffix.CARD)] ) if len(card_paths_without_steps) == 0: # If there are no files found on the Path then raise an error of @@ -240,7 +290,7 @@ def _list_card_paths(self, card_type=None, card_hash=None, card_id=None): cards_found = [] for task_card_path in card_paths: card_path = task_card_path.path - card_info = self.card_info_from_path(card_path) + card_info = self.card_info_from_path(card_path, suffix=CardNameSuffix.CARD) if card_type is not None and card_info.type != card_type: continue elif card_hash is not None: @@ -254,11 +304,35 @@ def _list_card_paths(self, card_type=None, card_hash=None, card_id=None): return cards_found + def _list_card_data(self, card_type=None, card_hash=None, card_id=None): + card_data_paths = self._backend.list_content( + [self._get_read_path(suffix=CardPathSuffix.DATA)] + ) + data_found = [] + + for data_path in card_data_paths: + _pth = data_path.path + card_info = self.card_info_from_path(_pth, suffix=CardNameSuffix.DATA) + if card_type is not None and card_info.type != card_type: + continue + elif card_hash is not None: + if not card_info.hash.startswith(card_hash): + continue + elif card_id is not None and card_info.id != card_id: + continue + if data_path.is_file: + data_found.append(_pth) + + return data_found + def create_full_path(self, card_path): return os.path.join(self._backend.datastore_root, card_path) def get_card_names(self, card_paths): - return [self.card_info_from_path(path) for path in card_paths] + return [ + self.card_info_from_path(path, suffix=CardNameSuffix.CARD) + for path in card_paths + ] def get_card_html(self, path): with self._backend.load_bytes([path]) as get_results: @@ -267,6 +341,13 @@ def get_card_html(self, path): with open(path, "r") as f: return f.read() + def get_card_data(self, path): + with self._backend.load_bytes([path]) as get_results: + for _, path, _ in get_results: + if path is not None: + with open(path, "r") as f: + return json.loads(f.read()) + def cache_locally(self, path, save_path=None): """ Saves the data present in the `path` the `metaflow_card_cache` directory or to the `save_path`. @@ -292,6 +373,15 @@ def cache_locally(self, path, save_path=None): shutil.copy(path, main_path) return main_path + def extract_data_paths(self, card_type=None, card_hash=None, card_id=None): + return self._list_card_data( + # card_hash is the unique identifier to the card. + # Its no longer the actual hash! + card_type=card_type, + card_hash=card_hash, + card_id=card_id, + ) + def extract_card_paths(self, card_type=None, card_hash=None, card_id=None): return self._list_card_paths( card_type=card_type, card_hash=card_hash, card_id=card_id diff --git a/metaflow/plugins/cards/card_decorator.py b/metaflow/plugins/cards/card_decorator.py index efa13a1ec90..29bddf798ba 100644 --- a/metaflow/plugins/cards/card_decorator.py +++ b/metaflow/plugins/cards/card_decorator.py @@ -2,6 +2,8 @@ import os import tempfile import sys +from functools import partial +import time import json from typing import Dict, Any @@ -9,7 +11,10 @@ from metaflow.decorators import StepDecorator, flow_decorators from metaflow.current import current from metaflow.util import to_unicode +from metaflow.metaflow_config import DATASTORE_LOCAL_DIR from .component_serializer import CardComponentCollector, get_card_class +from .card_creator import CardCreator +from .runtime_collector_state import CardStateManager # from metaflow import get_metadata @@ -17,6 +22,8 @@ from .exception import CARD_ID_PATTERN, TYPE_CHECK_REGEX +ASYNC_TIMEOUT = 30 + def warning_message(message, logger=None, ts=False): msg = "[@card WARNING] %s" % message @@ -60,6 +67,10 @@ class CardDecorator(StepDecorator): _called_once = {} + card_creator = None + + _collector_state_dump_file = None + def __init__(self, *args, **kwargs): super(CardDecorator, self).__init__(*args, **kwargs) self._task_datastore = None @@ -69,6 +80,12 @@ def __init__(self, *args, **kwargs): self._is_editable = False self._card_uuid = None self._user_set_card_id = None + self._current_step = None + self._state_manager = None # This is set on a per TASK basis + + @classmethod + def _set_card_creator(cls, card_creator): + cls.card_creator = card_creator def _is_event_registered(self, evt_name): return evt_name in self._called_once @@ -89,7 +106,6 @@ def _increment_step_counter(cls): def step_init( self, flow, graph, step_name, decorators, environment, flow_datastore, logger ): - self._flow_datastore = flow_datastore self._environment = environment self._logger = logger @@ -126,11 +142,19 @@ def task_pre_step( ubf_context, inputs, ): + + self._task_datastore = task_datastore + self._metadata = metadata + card_type = self.attributes["type"] card_class = get_card_class(card_type) + + self._is_runtime_card = False if card_class is not None: # Card type was not found if card_class.ALLOW_USER_COMPONENTS: self._is_editable = True + self._is_runtime_card = card_class.IS_RUNTIME_CARD + # We have a step counter to ensure that on calling the final card decorator's `task_pre_step` # we call a `finalize` function in the `CardComponentCollector`. # This can help ensure the behaviour of the `current.card` object is according to specification. @@ -151,11 +175,21 @@ def task_pre_step( warning_message(wrn_msg, self._logger) self._user_set_card_id = None + pathspec = "/".join([flow.name, str(run_id), step_name, str(task_id)]) # As we have multiple decorators, # we need to ensure that `current.card` has `CardComponentCollector` instantiated only once. if not self._is_event_registered("pre-step"): self._register_event("pre-step") - current._update_env({"card": CardComponentCollector(self._logger)}) + self._set_card_creator( + CardCreator( + base_command=self._create_base_command(), + pathspec=pathspec, + ) + ) + + current._update_env( + {"card": CardComponentCollector(self._logger, self.card_creator)} + ) # this line happens because of decospecs parsing. customize = False @@ -165,8 +199,11 @@ def task_pre_step( card_metadata = current.card._add_card( self.attributes["type"], self._user_set_card_id, - self._is_editable, - customize, + self.attributes, + self.card_options, + editable=self._is_editable, + customize=customize, + runtime_card=self._is_runtime_card, ) self._card_uuid = card_metadata["uuid"] @@ -175,18 +212,44 @@ def task_pre_step( # This will set up the `current.card` object for usage inside `@step` code. if self.step_counter == self.total_decos_on_step[step_name]: current.card._finalize() + self._dump_state_of_collector_to_file(pathspec) + # Set env var here so that any process spawned + # by this process can access the card state. + os.environ["METAFLOW_CARD_PATHSPEC"] = pathspec - self._task_datastore = task_datastore - self._metadata = metadata + def task_exception( + self, exception, step_name, flow, graph, retry_count, max_user_code_retries + ): + if self._state_manager is not None: + self._state_manager.done() + + def _dump_state_of_collector_to_file(self, pathspec): + # Create a directory in the DATASTORE_LOCAL_DIR like `DATASTORE_LOCAL_DIR/` + self._state_manager = CardStateManager(pathspec) + self.card_creator.state_manager = self._state_manager + self._state_manager.component_collector.save(current.card._dump_state_to_dict()) def task_finished( self, step_name, flow, graph, is_task_ok, retry_count, max_user_code_retries ): - if not is_task_ok: - return - component_strings = current.card._serialize_components(self._card_uuid) - runspec = "/".join([current.run_id, current.step_name, current.task_id]) - self._run_cards_subprocess(runspec, component_strings) + create_options = dict( + card_uuid=self._card_uuid, + user_set_card_id=self._user_set_card_id, + runtime_card=self._is_runtime_card, + decorator_attributes=self.attributes, + card_options=self.card_options, + logger=self._logger, + fetch_latest_data=partial(current.card._get_latest_data, self._card_uuid), + component_serialzer=partial( + current.card._serialize_components, self._card_uuid + ), + ) + if is_task_ok: + self.card_creator.create(mode="render", **create_options) + self.card_creator.create(mode="refresh", final=True, **create_options) + current.card._finished() + if self._state_manager is not None: + self._state_manager.done() @staticmethod def _options(mapping): @@ -199,8 +262,7 @@ def _options(mapping): if not isinstance(value, bool): yield to_unicode(value) - def _create_top_level_args(self): - + def _create_base_command(self): top_level_options = { "quiet": True, "metadata": self._metadata.TYPE, @@ -213,68 +275,6 @@ def _create_top_level_args(self): # We don't provide --with as all execution is taking place in # the context of the main process } - return list(self._options(top_level_options)) - - def _run_cards_subprocess(self, runspec, component_strings): - temp_file = None - if len(component_strings) > 0: - temp_file = tempfile.NamedTemporaryFile("w", suffix=".json") - json.dump(component_strings, temp_file) - temp_file.seek(0) executable = sys.executable - cmd = [ - executable, - sys.argv[0], - ] - cmd += self._create_top_level_args() + [ - "card", - "create", - runspec, - "--type", - self.attributes["type"], - # Add the options relating to card arguments. - # todo : add scope as a CLI arg for the create method. - ] - if self.card_options is not None and len(self.card_options) > 0: - cmd += ["--options", json.dumps(self.card_options)] - # set the id argument. - - if self.attributes["timeout"] is not None: - cmd += ["--timeout", str(self.attributes["timeout"])] - - if self._user_set_card_id is not None: - cmd += ["--id", str(self._user_set_card_id)] - - if self.attributes["save_errors"]: - cmd += ["--render-error-card"] - - if temp_file is not None: - cmd += ["--component-file", temp_file.name] - - response, fail = self._run_command( - cmd, os.environ, timeout=self.attributes["timeout"] - ) - if fail: - resp = "" if response is None else response.decode("utf-8") - self._logger( - "Card render failed with error : \n\n %s" % resp, - timestamp=False, - bad=True, - ) - - def _run_command(self, cmd, env, timeout=None): - fail = False - timeout_args = {} - if timeout is not None: - timeout_args = dict(timeout=int(timeout) + 10) - try: - rep = subprocess.check_output( - cmd, env=env, stderr=subprocess.STDOUT, **timeout_args - ) - except subprocess.CalledProcessError as e: - rep = e.output - fail = True - except subprocess.TimeoutExpired as e: - rep = e.output - fail = True - return rep, fail + cmd = [executable, sys.argv[0]] + return cmd + list(self._options(top_level_options)) diff --git a/metaflow/plugins/cards/card_locks.py b/metaflow/plugins/cards/card_locks.py new file mode 100644 index 00000000000..202fa3e03ce --- /dev/null +++ b/metaflow/plugins/cards/card_locks.py @@ -0,0 +1,223 @@ +from .runtime_collector_state import _get_card_state_directory +import fcntl +import os +import functools + + +class CardLock: + + POSSIBLE_STATES = ["locked_by_me", "unlocked", "locked_by_other"] + + lock_base_path = None + + def __init__(self, carduuid): + self._carduuid = carduuid + self._lock_path = os.path.join(self.lock_base_path, self._carduuid + ".lock") + self._lock_file = None + self._current_state = "unlocked" + + def _try_to_acquire_lock(self): + try: + lock_file = open(self._lock_path, "w") + fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) + return lock_file + except BlockingIOError: + return False + + @property + def owned_by_others(self): + return self._current_state == "locked_by_other" + + @property + def owns_the_lock(self): + return self._current_state == "locked_by_me" + + @property + def seems_unlocked(self): + return self._current_state == "unlocked" + + def test_lockability(self): + if self.owns_the_lock: + return True + elif self.seems_unlocked: + _lock_file = self._try_to_acquire_lock() + if _lock_file: + _lock_file.close() + return True + self._current_state = "locked_by_other" + return False + else: + return False + + def lock(self): + if self.owns_the_lock: + return True + elif self.seems_unlocked: + self._lock_file = self._try_to_acquire_lock() + if self._lock_file: + self._current_state = "locked_by_me" + return True + else: + self._current_state = "locked_by_other" + return False + else: + return False + + def unlock(self): + if self.owns_the_lock: + fcntl.flock(self._lock_file, fcntl.LOCK_UN) + self._lock_file.close() + self._current_state = "unlocked" + return True + else: + return False + + def __del__(self): + if self.owns_the_lock: + self.unlock() + + +class LockableCardDict: + """ + This is a class that will help us ensure that when `current.card` interface is called in an outside subprocess, + then we are able to call the `current.card` interface in outside subprocesses without any issues. + + Some constraints which the `LockableCardDict` aims to fulfill: + + > No two processes can lock the same card at the same time. Once a card is locked by a process, no other process can lock it. + (Meaning the first process to lock the card locks it for-ever. No other process can lock it after that). + + The life-cycle of cards is as follows: + 1. Pre-task execution : + - @card decorator instantiates a CardComponentCollector with a LockableCardDict set to `pre_task_startup=True` and empty dictionary for it's `card_component_store` + - This ensures that new CardComponentManagers can be trivially added by the main process. + - Once main process is done adding all cards, we run `finalize` method : + - it will set `pre_task_startup=False` and set post_task_startup=True + - it will instantiate the locks. Won't check if they are already possessed by others since the code is being run in the main process and + task execution has not started yet; (All lock alteration happens once user code starts executing). + - Upon finishing finalize `current.card` will also dump the state of `current.card` for being available to any outside subprocesses. + 2. Task Execution: + - Main process : + - Which ever cards the main thread update's will be locked by the main thread. + - The cards which are not accessed / updated by the main thread, can be locked by other processes. + - Locked cards will provide the same type of object back to the user, but the `card_proc` method will be replaced with a + method that will warn the user that the card is locked. + - Subprocesses: + - calls the `get_runtime_card` method to get the `current.card` object + - the method will seek out the `current.card`'s state location and load the state. + - the state of the card specifies what ever task spec's component collector's state + - will be able to call render_runtime and refresh from the cli + - `refresh` will also dump the state of the components to disk. + - [todo] when finished will call `current.card.done()` which will dump the state of the components/data-updates to disk + - [todo] no other processes will also be able to lock "done" cards + - [todo] If a subprocess is gracefully shutdown, then: + - [todo] the card manager will call `current.card.done()` which will avoid writes from other processes + 3. Post Task Execution: + - sub processes: + - Should have **ideally** ended and released all locks and ran the finalize method. + - If not then what ever is the last component state dumped in the subprocess will be the components loaded for the card. + - Main process: + - For each card, + - call the render method. + - [todo] If the card was locked by an outside process then seek it's component state and load it. + - if the card was not locked by an outside process then use the in-memory component state of the card. + """ + + def __init__( + self, + card_component_store, + lock_base_path, + pre_task_startup=False, + post_task_startup=True, + ): + self._card_component_store = card_component_store + self._card_locks = {} + self._lock_base_path = lock_base_path + self._pre_task_startup = pre_task_startup + self._post_task_startup = post_task_startup + self._init_locks() + + def _init_locks(self): + if self._pre_task_startup: + return + + CardLock.lock_base_path = self._lock_base_path + for carduuid in self._card_component_store: + self._card_locks[carduuid] = CardLock(carduuid) + + if not self._post_task_startup: + return + + # Since locks are initializing here we can test lockability here. + # As this code may be instantiated in a subprocess outside the main process, + # we need to ensure that when then LockableCardDict initializes, it can keeps only + # unlocked cards in the card_component_store. + # At this point when the code is running, the owning process of this object will + # not have any locks that it owns since all objects are being initialized. + for carduuid in self._card_component_store: + self._card_locks[carduuid].test_lockability() + if not self._card_locks[carduuid].seems_unlocked: + self._disable_rendering_for_locked_card(carduuid) + + def finalize(self): + # This method will only be called once by the CardComponent collector in scope of the + # @card decorator + self._pre_task_startup = False + self._init_locks() + self._post_task_startup = True + + def _disable_rendering_for_locked_card(self, carduuid): + def _wrap_method(instance, method_name): + def wrapper(*args, **kwargs): + instance._warning( + "Card is locked by another process. " + "This card will not be rendered/refreshed." + ) + return None + + # Bind the new method to the instance + setattr(instance, method_name, wrapper) + + _wrap_method(self._card_component_store[carduuid], "_card_proc") + + def _lock_check_for_getter(self, key): + # Here the key is uuid, so it's an internal things. + # Users will never have access to this object + assert key in self._card_locks, "Card not found in card store." + if self._post_task_startup: + if self._card_locks[key].seems_unlocked: + if not self._card_locks[key].lock(): + # If a card was locked then the in-memory status + # here will also flip to locked. + self._disable_rendering_for_locked_card(key) + + def _lock_check_for_setter(self, key): + if self._post_task_startup: + if self._card_locks[key].seems_unlocked: + if not self._card_locks[key].lock(): + return False + elif self._card_locks[key].owned_by_others: + return False + return True + + def unlock_everything(self): + for key in self._card_locks: + self._card_locks[key].unlock() + + def __contains__(self, key): + return key in self._card_component_store + + def __getitem__(self, key): + self._lock_check_for_getter(key) + return self._card_component_store[key] + + def __setitem__(self, key, value): + self._card_component_store[key] = value + if not self._lock_check_for_setter(key): + self._disable_rendering_for_locked_card(key) + + def items(self): + return self._card_component_store.items() + + def __len__(self): + return len(self._card_component_store) diff --git a/metaflow/plugins/cards/card_modules/base.html b/metaflow/plugins/cards/card_modules/base.html index 38a066179ea..91c05ca1f4b 100644 --- a/metaflow/plugins/cards/card_modules/base.html +++ b/metaflow/plugins/cards/card_modules/base.html @@ -18,6 +18,7 @@
+ + + + +
+ +
+ + + \ No newline at end of file diff --git a/metaflow/plugins/cards/component_serializer.py b/metaflow/plugins/cards/component_serializer.py index 9451cd67406..7c868c7a287 100644 --- a/metaflow/plugins/cards/component_serializer.py +++ b/metaflow/plugins/cards/component_serializer.py @@ -1,11 +1,19 @@ from .card_modules import MetaflowCardComponent from .card_modules.basic import ErrorComponent, SectionComponent from .card_modules.components import UserComponent +from .card_locks import LockableCardDict +from .runtime_collector_state import _get_card_state_directory +from functools import partial import uuid import json +import time _TYPE = type +# TODO move these to config +RUNTIME_CARD_MIN_REFRESH_INTERVAL = 5 +RUNTIME_CARD_RENDER_INTERVAL = 60 + def get_card_class(card_type): from metaflow.plugins import CARDS @@ -16,11 +24,302 @@ def get_card_class(card_type): return filtered_cards[0] +def _component_is_valid(component): + """ + Validates if the component is of the correct class. + """ + if not issubclass(type(component), MetaflowCardComponent): + return False + return True + + +def warning_message(message, logger=None, ts=False): + msg = "[@card WARNING] %s" % message + if logger: + logger(msg, timestamp=ts, bad=True) + + class WarningComponent(ErrorComponent): def __init__(self, warning_message): super().__init__("@card WARNING", warning_message) +class ComponentStore: + """ + The `ComponentStore` class handles the in-memory storage of the components for a single card. + This class has combination of a array/dictionary like interface to access the components. + + It exposes the `append` /`extend` methods like an array to add components. + It also exposes the `__getitem__`/`__setitem__` methods like a dictionary to access the components by thier Ids. + + The reason this has dual behavior is because components cannot be stored entirely as a map/dictionary because + the order of the components matter. The order of the components will visually affect the order of the components seen on the browser. + + """ + + def __init__(self, logger, components=None): + self._component_map = {} + self._components = [] + self._logger = logger + if components is not None: + for c in list(components): + self._store_component(c, component_id=None) + + def _realtime_updateable_components(self): + for c in self._components: + if c.REALTIME_UPDATABLE: + yield c + + def _create_component_id(self, component): + uuid_bit = "".join(uuid.uuid4().hex.split("-"))[:6] + return type(component).__name__.lower() + "_" + uuid_bit + + def _store_component(self, component, component_id=None): + if not _component_is_valid(component): + warning_message( + "Component (%s) is not a valid MetaflowCardComponent. It will not be stored." + % str(component), + self._logger, + ) + return + if component_id is not None: + component.id = component_id + elif component.id is None: + component.id = self._create_component_id(component) + self._components.append(component) + self._component_map[component.id] = self._components[-1] + + def _remove_component(self, component_id): + self._components.remove(self._component_map[component_id]) + del self._component_map[component_id] + + def __iter__(self): + return iter(self._components) + + def __setitem__(self, key, value): + if self._component_map.get(key) is not None: + # FIXME: what happens to this codepath + # Component Exists in the store + # What happens we relace a realtime component with a non realtime one. + # We have to ensure that layout change has take place so that the card should get re-rendered. + pass + else: + self._store_component(value, component_id=key) + + def __getitem__(self, key): + if key not in self._component_map: + raise KeyError( + "MetaflowCardComponent with id `%s` not found. Available components for the cards include : %s" + % (key, ", ".join(self.keys())) + ) + return self._component_map[key] + + def __delitem__(self, key): + if key not in self._component_map: + raise KeyError( + "MetaflowCardComponent with id `%s` not found. Available components for the cards include : %s" + % (key, ", ".join(self.keys())) + ) + self._remove_component(key) + + def __contains__(self, key): + return key in self._component_map + + def append(self, component, id=None): + self._store_component(component, component_id=id) + + def extend(self, components): + for c in components: + self._store_component(c, component_id=None) + + def clear(self): + self._components.clear() + self._component_map.clear() + + def keys(self): + return list(self._component_map.keys()) + + def values(self): + return self._components + + def __str__(self): + return "Card components present in the card: `%s` " % ("`, `".join(self.keys())) + + def __len__(self): + return len(self._components) + + +class CardComponentManager: + """ + This class manages the components for a single card. + It uses the `ComponentStore` to manage the storage of the components + and exposes methods to add, remove and access the components. + + It also exposes a `refresh` method that will allow refreshing a card with new data + for realtime(ish) updates. + + The `CardComponentCollector` class helps manage interaction with individual cards. The `CardComponentManager` + class helps manage the components for a single card. + The `CardComponentCollector` class uses this class to manage the components for a single card. + + The `CardComponentCollector` exposes convinience methods similar to this class for a default editable card. + `CardComponentCollector` resolves the default editable card at the time of task initialization and + exposes component manipulation methods for that card. These methods include : + + - `append` + - `extend` + - `clear` + - `refresh` + - `components` + - `__iter__` + + Under the hood these common methods will call the corresponding methods on the `CardComponentManager`. + `CardComponentManager` leverages the `ComponentStore` under the hood to actually add/update/remove components + under the hood. + + ## Usage Patterns : + + ```python + current.card["mycardid"].append(component, id="comp123") + current.card["mycardid"].extend([component]) + current.card["mycardid"].refresh(data) # refreshes the card with new data + current.card["mycardid"].components["comp123"] # returns the component with id "comp123" + current.card["mycardid"].components["comp123"].update() + current.card["mycardid"].components.clear() # Wipe all the components + del current.card["mycardid"].components["mycomponentid"] # Delete a component + current.card["mycardid"].components["mynewcomponent"] = Markdown("## New Component") # Set a new component + ``` + """ + + def __init__( + self, + card_uuid=None, + decorator_attributes=None, + card_creator=None, + components=None, + logger=None, + no_warnings=False, + user_set_card_id=None, + runtime_card=False, + card_options=None, + ): + self._card_creator_args = dict( + card_uuid=card_uuid, + user_set_card_id=user_set_card_id, + runtime_card=runtime_card, + decorator_attributes=decorator_attributes, + card_options=card_options, + ) + self._card_creator = card_creator + self._latest_user_data = None + self._last_refresh = 0 + self._last_render = 0 + self._render_seq = 0 + self._logger = logger + self._no_warnings = no_warnings + self._warn_once = { + "update": {}, + "not_implemented": {}, + } + if components is None: + self._components = ComponentStore(logger=self._logger, components=None) + else: + self._components = ComponentStore( + logger=self._logger, components=list(components) + ) + + def _dump_state_to_dict(self): + return {**self._card_creator_args, "no_warnings": self._no_warnings} + + @classmethod + def _load_from_dict(cls, card_dict, card_creator, logger=None): + return cls(**card_dict, card_creator=card_creator, logger=logger) + + def append(self, component, id=None): + self._components.append(component, id=id) + + def extend(self, components): + self._components.extend(components) + + def clear(self): + self._components.clear() + + def _card_proc(self, mode): + self._card_creator.create( + **self._card_creator_args, + fetch_latest_data=self._get_latest_data, + component_serialzer=self._serialize_components, + logger=self._logger, + mode=mode, + ) + + def refresh( + self, + data=None, + force=False, + ): + # todo make this a configurable variable + self._latest_user_data = data + nu = time.time() + + if nu - self._last_refresh < RUNTIME_CARD_MIN_REFRESH_INTERVAL: + # rate limit refreshes: silently ignore requests that + # happen too frequently + return + self._last_refresh = nu + # FIXME force render if components have changed + if force or nu - self._last_render > RUNTIME_CARD_RENDER_INTERVAL: + self._render_seq += 1 + self._last_render = nu + self._card_proc("render_runtime") + else: + self._card_proc("refresh") + + @property + def components(self): + return self._components + + def _warning(self, message): + msg = "[@card WARNING] %s" % message + self._logger(msg, timestamp=False, bad=True) + + def _get_latest_data(self, final=False): + seq = "final" if final else self._render_seq + component_dict = {} + for component in self._components._realtime_updateable_components(): + rendered_comp = _render_card_component(component) + if rendered_comp is not None: + component_dict.update({component.id: rendered_comp}) + # FIXME: Verify _latest_user_data is json serializable + return { + "user": self._latest_user_data, + "components": component_dict, + "render_seq": seq, + } + + def _serialize_components(self): + serialized_components = [] + has_user_components = any( + [ + issubclass(type(component), UserComponent) + for component in self._components + ] + ) + for component in self._components: + rendered_obj = _render_card_component(component) + if rendered_obj is None: + continue + serialized_components.append(rendered_obj) + if has_user_components and len(serialized_components) > 0: + serialized_components = [ + SectionComponent(contents=serialized_components).render() + ] + return serialized_components + + def __iter__(self): + return iter(self._components) + + class CardComponentCollector: """ This class helps collect `MetaflowCardComponent`s during runtime execution @@ -42,25 +341,95 @@ class CardComponentCollector: - [x] by looking it up by its type, e.g. `current.card.get(type='pytorch')`. """ - def __init__(self, logger=None): + def _dump_state_to_dict(self): + """ + Dump the following object's state to file: + - `self._card_component_store` + - `self._cards_meta` + - `self._card_id_map` + - `self._default_editable_card` + - `self._card_creator` + + """ + + def _serialize_store(card_manager_dict): + serialized_store = {} + for card_uuid, card_manager in card_manager_dict.items(): + serialized_store[card_uuid] = card_manager._dump_state_to_dict() + return serialized_store + + state_object = { + "card_component_store": _serialize_store(self._card_component_store), + "cards_meta": self._cards_meta, + "card_id_map": self._card_id_map, + "default_editable_card": self._default_editable_card, # UUId of defautl editable card + "card_creator": self._card_creator._dump_state_to_dict(), + "no_warnings": self._no_warnings, + } + return state_object + + @classmethod + def _load_state(cls, state_dict, logger, state_manager): + from .card_creator import CardCreator + + def _load_stores(card_manager_dict): + loaded_store = {} + for card_uuid, card_manager in card_manager_dict.items(): + loaded_store[card_uuid] = CardComponentManager._load_from_dict( + card_manager, card_creator, logger=logger + ) + return loaded_store + + card_creator = CardCreator(**state_dict["card_creator"]) + card_creator.state_manager = state_manager + collector_object = cls( + logger=logger, card_creator=card_creator, inside_main_process=False + ) + collector_object._card_component_store = LockableCardDict( + _load_stores(state_dict["card_component_store"]), + _get_card_state_directory(card_creator.pathspec), + pre_task_startup=False, + post_task_startup=True, + ) + collector_object._cards_meta = state_dict["cards_meta"] + collector_object._card_id_map = state_dict["card_id_map"] + collector_object._default_editable_card = state_dict["default_editable_card"] + collector_object._no_warnings = state_dict["no_warnings"] + return collector_object + + def __init__(self, logger=None, card_creator=None, inside_main_process=True): from metaflow.metaflow_config import CARD_NO_WARNING - self._cards_components = ( - {} - ) # a dict with key as uuid and value as a list of MetaflowCardComponent. + self._prefinalize_cards_dict = {} + self._card_component_store = { + # Each key in the dictionary is the UUID of an individual card. + # value is of type `CardComponentManager`, holding a list of MetaflowCardComponents for that particular card + # When the @card decorator instantiates the CardComponentCollector, it will first have a {} for this variable. + # Once the decorator adds all the cards to the collector and calls `_finalize`, this variable will turn into a lockable dictionary. + # This lockable dictionary will help ensure that this same object can be instantiated outside metaflow + # and we can dis-allow concurrent access to the same card. + } self._cards_meta = ( {} ) # a `dict` of (card_uuid, `dict)` holding all metadata about all @card decorators on the `current` @step. self._card_id_map = {} # card_id to uuid map for all cards with ids self._logger = logger + self._card_creator = card_creator # `self._default_editable_card` holds the uuid of the card that is default editable. This card has access to `append`/`extend` methods of `self` self._default_editable_card = None - self._warned_once = {"__getitem__": {}, "append": False, "extend": False} + self._warned_once = { + "__getitem__": {}, + "append": False, + "extend": False, + "update": False, + "update_no_id": False, + } self._no_warnings = True if CARD_NO_WARNING else False + self._inside_main_process = inside_main_process @staticmethod def create_uuid(): - return str(uuid.uuid4()) + return str(uuid.uuid4()).replace("-", "") def _log(self, *args, **kwargs): if self._logger: @@ -70,9 +439,12 @@ def _add_card( self, card_type, card_id, + decorator_attributes, + card_options, editable=False, customize=False, suppress_warnings=False, + runtime_card=False, ): """ This function helps collect cards from all the card decorators. @@ -95,9 +467,22 @@ def _add_card( editable=editable, customize=customize, suppress_warnings=suppress_warnings, + runtime_card=runtime_card, + decorator_attributes=decorator_attributes, + card_options=card_options, ) self._cards_meta[card_uuid] = card_metadata - self._cards_components[card_uuid] = [] + self._prefinalize_cards_dict[card_uuid] = CardComponentManager( + card_uuid=card_uuid, + decorator_attributes=decorator_attributes, + card_creator=self._card_creator, + components=None, + logger=self._logger, + no_warnings=self._no_warnings, + user_set_card_id=card_id, + runtime_card=runtime_card, + card_options=card_options, + ) return card_metadata def _warning(self, message): @@ -105,11 +490,13 @@ def _warning(self, message): self._log(msg, timestamp=False, bad=True) def _add_warning_to_cards(self, warn_msg): + # FIXME: This should behave based on the type of card it is updating. + # All cards shouldnt get a warning component. if self._no_warnings: return - for card_id in self._cards_components: + for card_id in self._card_component_store: if not self._cards_meta[card_id]["suppress_warnings"]: - self._cards_components[card_id].append(WarningComponent(warn_msg)) + self._card_component_store[card_id].append(WarningComponent(warn_msg)) def get(self, type=None): """`get` @@ -128,7 +515,7 @@ def get(self, type=None): for card_meta in self._cards_meta.values() if card_meta["type"] == card_type ] - return [self._cards_components[uuid] for uuid in card_uuids] + return [self._card_component_store[uuid] for uuid in card_uuids] def _finalize(self): """ @@ -138,6 +525,7 @@ def _finalize(self): - The `self._default_editable_card` holds the uuid of the card that will have access to the `append`/`extend` methods. 2. Resolving edge cases where @card `id` argument may be `None` or have a duplicate `id` when there are more than one editable cards. 3. Resolving the `self._default_editable_card` to the card with the`customize=True` argument. + 4. Finalizing all the locks in the `self._card_component_store` """ all_card_meta = list(self._cards_meta.values()) for c in all_card_meta: @@ -150,7 +538,7 @@ def _finalize(self): editable_cards_meta = [c for c in all_card_meta if c["editable"]] if len(editable_cards_meta) == 0: - return + return self._init_lockable_card_store() # Create the `self._card_id_map` lookup table which maps card `id` to `uuid`. # This table has access to all cards with `id`s set to them. @@ -163,7 +551,7 @@ def _finalize(self): # If there is only one editable card then this card becomes `self._default_editable_card` if len(editable_cards_meta) == 1: self._default_editable_card = editable_cards_meta[0]["uuid"] - return + return self._init_lockable_card_store() # Segregate cards which have id as none and those which don't. not_none_id_cards = [c for c in editable_cards_meta if c["card_id"] is not None] @@ -211,6 +599,17 @@ def _finalize(self): # since `editable_cards_meta` hold only `editable=True` by default we can just set this card here. self._default_editable_card = customize_cards[0]["uuid"] + return self._init_lockable_card_store() + + def _init_lockable_card_store(self): + self._card_component_store = LockableCardDict( + self._prefinalize_cards_dict, + _get_card_state_directory(self._card_creator.pathspec), + pre_task_startup=True, + post_task_startup=False, + ) + self._card_component_store.finalize() + def __getitem__(self, key): """ Choose a specific card for manipulation. @@ -229,13 +628,13 @@ def __getitem__(self, key): Returns ------- - CardComponentCollector + CardComponentManager An object with `append` and `extend` calls which allow you to add components to the chosen card. """ if key in self._card_id_map: card_uuid = self._card_id_map[key] - return self._cards_components[card_uuid] + return self._card_component_store[card_uuid] if key not in self._warned_once["__getitem__"]: _warn_msg = [ "`current.card['%s']` is not present. Please set the `id` argument in @card to '%s' to access `current.card['%s']`." @@ -246,6 +645,7 @@ def __getitem__(self, key): self._warning(" ".join(_warn_msg)) self._add_warning_to_cards("\n".join(_warn_msg)) self._warned_once["__getitem__"][key] = True + return [] def __setitem__(self, key, value): @@ -263,7 +663,7 @@ def __setitem__(self, key, value): key: str Card ID. - value: List[CardComponent] + value: List[MetaflowCardComponent] List of card components to assign to this card. """ if key in self._card_id_map: @@ -275,7 +675,19 @@ def __setitem__(self, key, value): ) self._warning(_warning_msg) return - self._cards_components[card_uuid] = value + self._card_component_store[card_uuid] = CardComponentManager( + card_uuid=card_uuid, + decorator_attributes=self._cards_meta[card_uuid][ + "decorator_attributes" + ], + card_creator=self._card_creator, + components=value, + logger=self._logger, + no_warnings=self._no_warnings, + user_set_card_id=key, + card_options=self._cards_meta[card_uuid]["card_options"], + runtime_card=self._cards_meta[card_uuid]["runtime_card"], + ) return self._warning( @@ -283,18 +695,18 @@ def __setitem__(self, key, value): % (key, key, key) ) - def append(self, component): + def append(self, component, id=None): """ Appends a component to the current card. Parameters ---------- - component : CardComponent + component : MetaflowCardComponent Card component to add to this card. """ if self._default_editable_card is None: if ( - len(self._cards_components) == 1 + len(self._card_component_store) == 1 ): # if there is one card which is not the _default_editable_card then the card is not editable card_type = list(self._cards_meta.values())[0]["type"] if list(self._cards_meta.values())[0]["exists"]: @@ -324,7 +736,7 @@ def append(self, component): self._warned_once["append"] = True return - self._cards_components[self._default_editable_card].append(component) + self._card_component_store[self._default_editable_card].append(component, id=id) def extend(self, components): """ @@ -332,12 +744,12 @@ def extend(self, components): Parameters ---------- - component : Iterator[CardComponent] + component : Iterator[MetaflowCardComponent] Card components to add to this card. """ if self._default_editable_card is None: # if there is one card which is not the _default_editable_card then the card is not editable - if len(self._cards_components) == 1: + if len(self._card_component_store) == 1: card_type = list(self._cards_meta.values())[0]["type"] _warning_msg = [ "Card of type `%s` is not an editable card." % card_type, @@ -357,7 +769,50 @@ def extend(self, components): return - self._cards_components[self._default_editable_card].extend(components) + self._card_component_store[self._default_editable_card].extend(components) + + @property + def components(self): + # FIXME: document + if self._default_editable_card is None: + if len(self._card_component_store) == 1: + card_type = list(self._cards_meta.values())[0]["type"] + _warning_msg = [ + "Card of type `%s` is not an editable card." % card_type, + "Components list will not be updated and `current.card.components` will not work for any call during this runtime execution.", + "Please use an editable card", # todo : link to documentation + ] + else: + _warning_msg = [ + "`current.card.components` cannot disambiguate between multiple @card decorators.", + "Components list will not be accessible and `current.card.components` will not work for any call during this runtime execution.", + "To fix this set the `id` argument in all @card when using multiple @card decorators over a single @step and reference `current.card[ID].components`", # todo : Add Link to documentation + "to update/access the appropriate card component.", + ] + if not self._warned_once["components"]: + self._warning(" ".join(_warning_msg)) + self._warned_once["components"] = True + return + + return self._card_component_store[self._default_editable_card].components + + def clear(self): + # FIXME: document + if self._default_editable_card is not None: + self._card_component_store[self._default_editable_card].clear() + + def refresh(self, *args, **kwargs): + # FIXME: document + if self._default_editable_card is not None: + self._card_component_store[self._default_editable_card].refresh( + *args, **kwargs + ) + + def _get_latest_data(self, card_uuid, final=False): + """ + Returns latest data so it can be used in the final render() call + """ + return self._card_component_store[card_uuid]._get_latest_data(final=final) def _serialize_components(self, card_uuid): """ @@ -365,37 +820,31 @@ def _serialize_components(self, card_uuid): Components exposed by metaflow ensure that they render safely. If components don't render safely then we don't add them to the final list of serialized functions """ - serialized_components = [] - if card_uuid not in self._cards_components: + if card_uuid not in self._card_component_store: return [] - has_user_components = any( - [ - issubclass(type(component), UserComponent) - for component in self._cards_components[card_uuid] - ] - ) - for component in self._cards_components[card_uuid]: - if not issubclass(type(component), MetaflowCardComponent): - continue - try: - rendered_obj = component.render() - except: - continue - else: - if not (type(rendered_obj) == str or type(rendered_obj) == dict): - continue - else: - # Since `UserComponent`s are safely_rendered using render_tools.py - # we don't need to check JSON serialization as @render_tools.render_safely - # decorator ensures this check so there is no need to re-serialize - if not issubclass(type(component), UserComponent): - try: # check if rendered object is json serializable. - json.dumps(rendered_obj) - except (TypeError, OverflowError) as e: - continue - serialized_components.append(rendered_obj) - if has_user_components and len(serialized_components) > 0: - serialized_components = [ - SectionComponent(contents=serialized_components).render() - ] - return serialized_components + return self._card_component_store[card_uuid]._serialize_components() + + def _finished(self): + self._card_component_store.unlock_everything() + + +def _render_card_component(component): + if not _component_is_valid(component): + return None + try: + rendered_obj = component.render() + except: + return None + else: + if not (type(rendered_obj) == str or type(rendered_obj) == dict): + return None + else: + # Since `UserComponent`s are safely_rendered using render_tools.py + # we don't need to check JSON serialization as @render_tools.render_safely + # decorator ensures this check so there is no need to re-serialize + if not issubclass(type(component), UserComponent): + try: # check if rendered object is json serializable. + json.dumps(rendered_obj) + except (TypeError, OverflowError) as e: + return None + return rendered_obj diff --git a/metaflow/plugins/cards/runtime_collector_state.py b/metaflow/plugins/cards/runtime_collector_state.py new file mode 100644 index 00000000000..1bbfb5fc9d1 --- /dev/null +++ b/metaflow/plugins/cards/runtime_collector_state.py @@ -0,0 +1,112 @@ +from metaflow.metaflow_config import DATASTORE_LOCAL_DIR +import os +import hashlib +import shutil +import tempfile +import json + +CARD_STATE_ROOT_DIR = os.path.join(DATASTORE_LOCAL_DIR, "mf.card_state") + + +class StateContainer: + def __init__( + self, + file_name, + ): + os.makedirs(os.path.dirname(file_name), exist_ok=True) + self._file_name = file_name + self._file_created = os.path.exists(self._file_name) + + @property + def is_present(self): + return os.path.exists(self._file_name) + + def load(self): + with open(self._file_name, "r") as _file: + state = json.load(_file) + return state + + def save(self, state): + with open(self._file_name, "w") as _file: + json.dump(state, _file) + self._file_created = True + + def done(self): + if self._file_created: + os.remove(self._file_name) + + +class CardStateManager: + + card_state = {} + + def __init__(self, pathspec) -> None: + self._pathspec = pathspec + self._card_state_directory = _get_card_state_directory(pathspec) + self._component_collector_state = StateContainer( + _get_card_collector_state_file_path(pathspec) + ) + self._state_file_created = False + + @property + def component_collector(self): + return self._component_collector_state + + def done(self): + if self._state_file_created: + self._component_collector_state.done() + shutil.rmtree(self._card_state_directory) + + def _save_card_state(self, uuid, components=None, data=None): + if uuid not in self.card_state: + self.card_state[uuid] = { + "components": StateContainer( + os.path.join(self._card_state_directory, f"{uuid}.components.json") + ), + "data": StateContainer( + os.path.join(self._card_state_directory, f"{uuid}.data.json") + ), + } + if components is not None: + self.card_state[uuid]["components"].save(components) + if data is not None: + self.card_state[uuid]["data"].save(data) + + +def _get_card_collector_state_file_path(pathspec): + return os.path.join(_get_card_state_directory(pathspec), "card_state.json") + + +def _get_card_state_directory(pathspec): + pathspec_hash = hashlib.md5(pathspec.encode()).hexdigest() + return os.path.join(CARD_STATE_ROOT_DIR, pathspec_hash) + + +def get_realtime_cards( + pathspec=None, +): + # FIXME : What is a way we can gaurentee that `CardStateManager` + # Loads the correct path based on what ever it's CWD is set in the subprocess + # calling this. + from .component_serializer import CardComponentCollector + from metaflow import current + from metaflow.cli import logger + + if pathspec is None: + # if getattr(current, "card", None): + # return current.card + # else: + if "METAFLOW_CARD_PATHSPEC" not in os.environ: + raise ValueError( + "It seems like `get_realtime_cards` is being called from outside a Metaflow step process. " + "In such cases a `pathspec` argument is required or `METAFLOW_CARD_PATHSPEC` needs to be set " + "as an environment variable." + ) + pathspec = os.environ["METAFLOW_CARD_PATHSPEC"] + + card_state_manager = CardStateManager(pathspec) + return CardComponentCollector._load_state( + state_dict=card_state_manager.component_collector.load(), + logger=logger, + state_manager=card_state_manager, + ) diff --git a/metaflow/plugins/cards/ui/.gitignore b/metaflow/plugins/cards/ui/.gitignore new file mode 100644 index 00000000000..31d93b23fc2 --- /dev/null +++ b/metaflow/plugins/cards/ui/.gitignore @@ -0,0 +1,581 @@ +.yarn + +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.tlog +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Tes# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencit files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*.json +coverage*.xml +coverage*.info + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio 6 auto-generated project file (contains which files were open etc.) +*.vbp + +# Visual Studio 6 workspace and project file (working project files containing files to include in project) +*.dsw +*.dsp + +# Visual Studio 6 technical files +*.ncb +*.aps + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# Visual Studio History (VSHistory) files +.vshistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +# VS Code files for those working on multiple tools +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +# Windows Installer files from build outputs +*.cab +*.msi +*.msix +*.msm +*.msp + +# JetBrains Rider +*.sln.iml +es or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/metaflow/plugins/cards/ui/.prettierrc.json b/metaflow/plugins/cards/ui/.prettierrc.json new file mode 100644 index 00000000000..bf357fbbc08 --- /dev/null +++ b/metaflow/plugins/cards/ui/.prettierrc.json @@ -0,0 +1,3 @@ +{ + "trailingComma": "all" +} diff --git a/metaflow/plugins/cards/ui/.yarnrc.yml b/metaflow/plugins/cards/ui/.yarnrc.yml new file mode 100644 index 00000000000..3186f3f0795 --- /dev/null +++ b/metaflow/plugins/cards/ui/.yarnrc.yml @@ -0,0 +1 @@ +nodeLinker: node-modules diff --git a/metaflow/plugins/cards/ui/public/card-example.json b/metaflow/plugins/cards/ui/public/card-example.json index 91c7bdc609a..613a22f9962 100644 --- a/metaflow/plugins/cards/ui/public/card-example.json +++ b/metaflow/plugins/cards/ui/public/card-example.json @@ -17,6 +17,20 @@ "type": "page", "title": "Run", "contents": [ + { + "type": "section", + "title": "progress bar", + "contents": [ + { + "type": "progressBar", + "id": "progress1", + "max": 100, + "value": 55, + "label": "custom bar", + "unit": "%" + } + ] + }, { "type": "section", "title": "Vertical Table", @@ -52,120 +66,120 @@ "contents": [ { "type": "artifacts", - + "data": [ { - "name":null, - "type":"tuple", - "data":"(1,2,3)" + "name": null, + "type": "tuple", + "data": "(1,2,3)" }, { - "name":"file_param", + "name": "file_param", "type": "NoneType", "data": "None" }, { - "name":"py_set", + "name": "py_set", "type": "set", "data": "{1, 2, 3}" }, { - "name":"img_jpg", + "name": "img_jpg", "type": "bytes", "data": "b'\\xff\\xd8\\xff\\xe0\\x00\\x10JFIF\\x00\\x01\\x02\\x01\\x...\\x80\\x80\\x80\\x80\\x80\\x80\\x80\\x80\\x80\\x83\\xff\\xd9'" }, { - "name":"py_frozenset", + "name": "py_frozenset", "type": "frozenset", "data": "frozenset({4, 5, 6})" }, { - "name":"py_bytearray", + "name": "py_bytearray", "type": "bytearray", "data": "bytearray(b'\\xf0\\xf1\\xf2')" }, { - "name":"custom_class", + "name": "custom_class", "type": "__main__.CustomClass", "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa...aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, { - "name":"py_dict", + "name": "py_dict", "type": "dict", "data": "{'a': 1, 'null': None, True: False}" }, { - "name":"py_float", + "name": "py_float", "type": "float", "data": "3.141592653589793" }, { - "name":"large_dict_large_val", + "name": "large_dict_large_val", "type": "dict", "data": "{'constitution': 'Provided by USConstitution.net\\n---------------...ion of Representatives shall\\nhave intervened.\\n'}" }, { - "name":"name", + "name": "name", "type": "str", "data": "'DefaultCardFlow'" }, { - "name":"large_dict", + "name": "large_dict", "type": "dict", "data": "{'clubs': ['ace', 2, 3, 4, 5, 6, 7, 8, 9, 'jack', 'queen', 'king'], 'diamonds': ['ace', 2, 3, 4, 5, 6, 7, 8, 9, 'jack', 'queen', 'king'], 'hearts': ['ace', 2, 3, 4, 5, 6, 7, 8, 9, 'jack', 'queen', 'king'], 'spades': ['ace', 2, 3, 4, 5, 6, 7, 8, 9, 'jack', 'queen', 'king']}" }, { - "name":"np_array", + "name": "np_array", "type": "numpy.ndarray", "data": "array([ 0, 1, 2, ..., 9999997, 9999998, 9999999],\n dtype=uint64)" }, { - "name":"large_dict_many_keys", + "name": "large_dict_many_keys", "type": "dict", "data": "{'00001cdb-f3cf-4a80-9d50-a5685697a9c3': 1636352431.135456, '00007105-19d0-4c25-bd18-c4b9f0f72326': 1636352431.09861, '00007fe7-87f5-499a-84a0-95bc7829350f': 1636352430.95102, '00009603-ec57-42ac-8ae1-d190d75ec8fd': 1636352431.879671, ...}" }, { - "name":"py_str", + "name": "py_str", "type": "str", "data": "'刺身は美味しい'" }, { - "name":"float_param", + "name": "float_param", "type": "float", "data": "3.141592653589793" }, { - "name":"py_complex", + "name": "py_complex", "type": "complex", "data": "(1+2j)" }, { - "name":"str_param", + "name": "str_param", "type": "str", "data": "'刺身は美味しい'" }, { - "name":"json_param", + "name": "json_param", "type": "str", "data": "'{\"states\": {[{\"CA\", 0}, {\"NY\", 1}]}'" }, { - "name":"large_int", + "name": "large_int", "type": "int", "data": "36893488147419103232" }, { - "name":"large_str", + "name": "large_str", "type": "str", "data": "'Provided by USConstitution.net\\n---------------...ion of Representatives shall\\nhave intervened.\\n'" }, { - "name":"exception", + "name": "exception", "type": "Exception", "data": "Exception('This is an exception!')" }, { - "name":"py_list", + "name": "py_list", "type": "list", "data": "[1, 2, 3]" } diff --git a/metaflow/plugins/cards/ui/public/index.html b/metaflow/plugins/cards/ui/public/index.html index 9b18320f76f..ee15ccee285 100644 --- a/metaflow/plugins/cards/ui/public/index.html +++ b/metaflow/plugins/cards/ui/public/index.html @@ -12,5 +12,27 @@
+ + diff --git a/metaflow/plugins/cards/ui/rollup.config.js b/metaflow/plugins/cards/ui/rollup.config.js index 5d4b7502aef..c3054cd1756 100644 --- a/metaflow/plugins/cards/ui/rollup.config.js +++ b/metaflow/plugins/cards/ui/rollup.config.js @@ -43,7 +43,7 @@ export default { input: "src/main.ts", output: { dir: process.env.OUTPUT_DIR ?? "public/build", - sourcemap: true, + sourcemap: !production, format: "iife", name: "app", }, diff --git a/metaflow/plugins/cards/ui/src/App.svelte b/metaflow/plugins/cards/ui/src/App.svelte index bbb390ebbca..881d3e0234e 100644 --- a/metaflow/plugins/cards/ui/src/App.svelte +++ b/metaflow/plugins/cards/ui/src/App.svelte @@ -3,7 +3,7 @@ import "./prism"; import "./global.css"; import "./prism.css"; - import "./app.css" + import "./app.css"; import { cardData, setCardData, modal } from "./store"; import * as utils from "./utils"; import Aside from "./components/aside.svelte"; @@ -16,11 +16,11 @@ // Get the data from the element in `windows.__MF_DATA__` corresponding to `cardDataId`. This allows multiple sets of // data to exist on a single page - setCardData(cardDataId) + setCardData(cardDataId); // Set the `embed` class to hide the `aside` if specified in the URL const urlParams = new URLSearchParams(window?.location.search); - let embed = Boolean(urlParams.get('embed')) + let embed = Boolean(urlParams.get("embed"));
diff --git a/metaflow/plugins/cards/ui/src/components/artifacts.svelte b/metaflow/plugins/cards/ui/src/components/artifacts.svelte index 72309d41225..3e4de16f06c 100644 --- a/metaflow/plugins/cards/ui/src/components/artifacts.svelte +++ b/metaflow/plugins/cards/ui/src/components/artifacts.svelte @@ -4,10 +4,9 @@ import ArtifactRow from "./artifact-row.svelte"; export let componentData: types.ArtifactsComponent; - const { data } = componentData; // we can't guarantee the data is sorted from the source, so we sort before render - const sortedData = data.sort((a, b) => { + const sortedData = componentData?.data.sort((a, b) => { // nulls first if (a.name && b.name) { if (a.name > b.name) { @@ -24,7 +23,7 @@ {#each sortedData as artifact} - + {/each}
diff --git a/metaflow/plugins/cards/ui/src/components/bar-chart.svelte b/metaflow/plugins/cards/ui/src/components/bar-chart.svelte index 7db0febe9a7..0d88591fcff 100644 --- a/metaflow/plugins/cards/ui/src/components/bar-chart.svelte +++ b/metaflow/plugins/cards/ui/src/components/bar-chart.svelte @@ -17,11 +17,11 @@ BarController, LinearScale, CategoryScale, - PointElement + PointElement, ); export let componentData: types.BarChartComponent; - const { config, data, labels } = componentData; + $: ({ config, data, labels } = componentData); let el: HTMLCanvasElement; diff --git a/metaflow/plugins/cards/ui/src/components/card-component-renderer.svelte b/metaflow/plugins/cards/ui/src/components/card-component-renderer.svelte index 6d9e05f9a14..3965d5ce183 100644 --- a/metaflow/plugins/cards/ui/src/components/card-component-renderer.svelte +++ b/metaflow/plugins/cards/ui/src/components/card-component-renderer.svelte @@ -10,6 +10,7 @@ import Log from "./log.svelte"; import Markdown from "./markdown.svelte"; import Page from "./page.svelte"; + import ProgressBar from "./progress-bar.svelte"; import Section from "./section.svelte"; import Subtitle from "./subtitle.svelte"; import Table from "./table.svelte"; @@ -29,6 +30,7 @@ log: Log, markdown: Markdown, page: Page, + progressBar: ProgressBar, section: Section, subtitle: Subtitle, table: Table, @@ -36,11 +38,12 @@ title: Title, }; - let component = typesMap?.[componentData.type] + let component = typesMap?.[componentData.type]; if (!component) { - console.error("Unknown component type: ", componentData.type) + console.error("Unknown component type: ", componentData.type); } + {#if component} {#if (componentData.type === "page" || componentData.type === "section") && componentData?.contents} diff --git a/metaflow/plugins/cards/ui/src/components/heading.svelte b/metaflow/plugins/cards/ui/src/components/heading.svelte index 03983575467..b41cd0bed21 100644 --- a/metaflow/plugins/cards/ui/src/components/heading.svelte +++ b/metaflow/plugins/cards/ui/src/components/heading.svelte @@ -5,7 +5,7 @@ import Subtitle from "./subtitle.svelte"; export let componentData: types.HeadingComponent; - const { title, subtitle } = componentData; + $: ({ title, subtitle } = componentData);
diff --git a/metaflow/plugins/cards/ui/src/components/image.svelte b/metaflow/plugins/cards/ui/src/components/image.svelte index 3a247455907..e28dfac31a8 100644 --- a/metaflow/plugins/cards/ui/src/components/image.svelte +++ b/metaflow/plugins/cards/ui/src/components/image.svelte @@ -4,9 +4,10 @@ import { modal } from "../store"; export let componentData: types.ImageComponent; - const { src, label, description } = componentData; + $: ({ src, label, description } = componentData); +
modal.set(componentData)} data-component="image">
{label diff --git a/metaflow/plugins/cards/ui/src/components/line-chart.svelte b/metaflow/plugins/cards/ui/src/components/line-chart.svelte index 1defb02e457..1f5173309b9 100644 --- a/metaflow/plugins/cards/ui/src/components/line-chart.svelte +++ b/metaflow/plugins/cards/ui/src/components/line-chart.svelte @@ -17,11 +17,11 @@ LinearScale, LineController, CategoryScale, - PointElement + PointElement, ); export let componentData: types.LineChartComponent; - const { config, data, labels } = componentData; + $: ({ config, data, labels } = componentData); let el: HTMLCanvasElement; diff --git a/metaflow/plugins/cards/ui/src/components/modal.svelte b/metaflow/plugins/cards/ui/src/components/modal.svelte index 3dff91f914d..a9adf1bbbea 100644 --- a/metaflow/plugins/cards/ui/src/components/modal.svelte +++ b/metaflow/plugins/cards/ui/src/components/modal.svelte @@ -20,6 +20,7 @@ {#if componentData && $modal} +
diff --git a/metaflow/plugins/cards/ui/src/components/page.svelte b/metaflow/plugins/cards/ui/src/components/page.svelte index 8f2736c9a39..0eb96f25ae8 100644 --- a/metaflow/plugins/cards/ui/src/components/page.svelte +++ b/metaflow/plugins/cards/ui/src/components/page.svelte @@ -2,10 +2,13 @@ -
+
diff --git a/metaflow/plugins/cards/ui/src/components/progress-bar.svelte b/metaflow/plugins/cards/ui/src/components/progress-bar.svelte new file mode 100644 index 00000000000..51b8354c41a --- /dev/null +++ b/metaflow/plugins/cards/ui/src/components/progress-bar.svelte @@ -0,0 +1,14 @@ + + + +
+ {#if label} + + {/if} + {value}{unit || ""} +
diff --git a/metaflow/plugins/cards/ui/src/components/section.svelte b/metaflow/plugins/cards/ui/src/components/section.svelte index e43a6f34d3f..8377e19cdab 100644 --- a/metaflow/plugins/cards/ui/src/components/section.svelte +++ b/metaflow/plugins/cards/ui/src/components/section.svelte @@ -2,7 +2,7 @@ -

{componentData.text}

+

{componentData?.text || ""}