Skip to content

Commit

Permalink
Use Fsp abstration layer through engine and UI
Browse files Browse the repository at this point in the history
Instead of referencing the remote processing funcs by a `str` name start
embracing the new `@fsp`/`Fsp` API such that wrapped processing
functions are first class APIs.

Summary of the changeset:
- move and load the fsp built-in set in the new `.fsp._api` module
- handle processors ("fsps") which want to yield multiple keyed-values
  (interleaved in time) by expecting both history that is keyed and
  assigned to the appropriate struct-array field, *and* real-time
  `yield`ed value in tuples of the form `tuple[str, float]` such that
  any one (async) processing function can deliver multiple outputs from
  the same base calculation.
- drop `maybe_mk_fsp_shm()` from UI module
- expect and manage `Fsp` instances (`@fsp` decorated funcs) throughout
  the UI code, particularly the `FspAdmin` layer.
  • Loading branch information
goodboy committed Jan 27, 2022
1 parent 72f4474 commit cc53903
Show file tree
Hide file tree
Showing 2 changed files with 117 additions and 103 deletions.
58 changes: 33 additions & 25 deletions piker/fsp/_engine.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# piker: trading gear for hackers
# Copyright (C) Tyler Goodlet (in stewardship of piker0)
# Copyright (C) Tyler Goodlet (in stewardship of pikers)

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
Expand Down Expand Up @@ -27,29 +27,19 @@
import trio
from trio_typing import TaskStatus
import tractor
from tractor._portal import NamespacePath

from ..log import get_logger, get_console_log
from .. import data
from ..data import attach_shm_array
from ..data.feed import Feed
from ..data._sharedmem import ShmArray
from ._momo import _rsi, _wma
from ._volume import _tina_vwap, dolla_vlm
# from ._momo import _rsi, _wma
# from ._volume import _tina_vwap, dolla_vlm
from ._api import _load_builtins

log = get_logger(__name__)

_fsp_builtins = {
'rsi': _rsi,
'wma': _wma,
'vwap': _tina_vwap,
'dolla_vlm': dolla_vlm,
}

# TODO: things to figure the heck out:
# - how to handle non-plottable values (pyqtgraph has facility for this
# now in `arrayToQPath()`)
# - composition of fsps / implicit chaining syntax (we need an issue)


@dataclass
class TaskTracker:
Expand Down Expand Up @@ -88,7 +78,7 @@ async def fsp_compute(
src: ShmArray,
dst: ShmArray,

func_name: str,
# func_name: str,
func: Callable,

attach_stream: bool = False,
Expand All @@ -115,15 +105,27 @@ async def fsp_compute(
# and get historical output
history_output = await out_stream.__anext__()

func_name = func.__name__
profiler(f'{func_name} generated history')

# build a struct array which includes an 'index' field to push
# as history
# build struct array with an 'index' field to push as history
history = np.array(
np.arange(len(history_output)),
dtype=dst.array.dtype
)
history[func_name] = history_output

# TODO: push using a[['f0', 'f1', .., 'fn']] = .. syntax no?
# if the output array is multi-field then push
# each respective field.
fields = getattr(history.dtype, 'fields', None)
if fields:
for key in fields.keys():
if key in history.dtype.fields:
history[func_name] = history_output

# single-key output stream
else:
history[func_name] = history_output

# TODO: XXX:
# THERE'S A BIG BUG HERE WITH THE `index` field since we're
Expand Down Expand Up @@ -164,8 +166,9 @@ async def fsp_compute(
async for processed in out_stream:

log.debug(f"{func_name}: {processed}")
key, output = processed
index = src.index
dst.array[-1][func_name] = processed
dst.array[-1][key] = output

# NOTE: for now we aren't streaming this to the consumer
# stream latest array index entry which basically just acts
Expand Down Expand Up @@ -194,7 +197,7 @@ async def cascade(
src_shm_token: dict,
dst_shm_token: tuple[str, np.dtype],

func_name: str,
ns_path: NamespacePath,

zero_on_step: bool = False,
loglevel: Optional[str] = None,
Expand All @@ -213,10 +216,14 @@ async def cascade(
src = attach_shm_array(token=src_shm_token)
dst = attach_shm_array(readonly=False, token=dst_shm_token)

func: Callable = _fsp_builtins.get(func_name)
# func: Callable = _fsp_builtins.get(tuple(ns_path))
func: Fsp = _load_builtins().get(
NamespacePath(ns_path)
)

if not func:
# TODO: assume it's a func target path
raise ValueError('Unknown fsp target: {func_name}')
raise ValueError(f'Unknown fsp target: {ns_path}')

# open a data feed stream with requested broker
async with data.feed.maybe_open_feed(
Expand All @@ -231,11 +238,12 @@ async def cascade(

) as (feed, quote_stream):

profiler(f'{func_name}: feed up')
profiler(f'{func}: feed up')

assert src.token == feed.shm.token
# last_len = new_len = len(src.array)

func_name = func.__name__
async with (
trio.open_nursery() as n,
):
Expand All @@ -252,7 +260,7 @@ async def cascade(
src=src,
dst=dst,

func_name=func_name,
# func_name=func_name,
func=func
)

Expand Down
Loading

0 comments on commit cc53903

Please sign in to comment.