Merge pull request #284 from pikers/ahabs_dick

Ahab's dick

piker/__init__.py

@@ -18,10 +18,3 @@
 piker: trading gear for hackers.
 
 """
-import msgpack  # noqa
-
-# TODO: remove this now right?
-import msgpack_numpy
-
-# patch msgpack for numpy arrays
-msgpack_numpy.patch()

piker/cli/__init__.py

@@ -16,20 +16,6 @@
 log = get_logger('cli')
 DEFAULT_BROKER = 'questrade'
 
-_config_dir = click.get_app_dir('piker')
-_watchlists_data_path = os.path.join(_config_dir, 'watchlists.json')
-_context_defaults = dict(
-    default_map={
-        # Questrade specific quote poll rates
-        'monitor': {
-            'rate': 3,
-        },
-        'optschain': {
-            'rate': 1,
-        },
-    }
-)
-
 
 @click.command()
 @click.option('--loglevel', '-l', default='warning', help='Logging level')
@@ -58,7 +44,7 @@ async def main():
     trio.run(main)
 
 
-@click.group(context_settings=_context_defaults)
+@click.group(context_settings=config._context_defaults)
 @click.option(
     '--brokers', '-b',
     default=[DEFAULT_BROKER],
@@ -87,8 +73,8 @@ def cli(ctx, brokers, loglevel, tl, configdir):
         'loglevel': loglevel,
         'tractorloglevel': None,
         'log': get_console_log(loglevel),
-        'confdir': _config_dir,
-        'wl_path': _watchlists_data_path,
+        'confdir': config._config_dir,
+        'wl_path': config._watchlists_data_path,
     })
 
     # allow enabling same loglevel in ``tractor`` machinery

piker/config.py

@@ -17,21 +17,109 @@
 """
 Broker configuration mgmt.
 """
+import platform
+import sys
 import os
 from os.path import dirname
 import shutil
 from typing import Optional
 
 from bidict import bidict
 import toml
-import click
 
 from .log import get_logger
 
 log = get_logger('broker-config')
 
-_config_dir = click.get_app_dir('piker')
+
+# taken from ``click`` since apparently they have some
+# super weirdness with sigint and sudo..no clue
+def get_app_dir(app_name, roaming=True, force_posix=False):
+    r"""Returns the config folder for the application.  The default behavior
+    is to return whatever is most appropriate for the operating system.
+
+    To give you an idea, for an app called ``"Foo Bar"``, something like
+    the following folders could be returned:
+
+    Mac OS X:
+      ``~/Library/Application Support/Foo Bar``
+    Mac OS X (POSIX):
+      ``~/.foo-bar``
+    Unix:
+      ``~/.config/foo-bar``
+    Unix (POSIX):
+      ``~/.foo-bar``
+    Win XP (roaming):
+      ``C:\Documents and Settings\<user>\Local Settings\Application Data\Foo Bar``
+    Win XP (not roaming):
+      ``C:\Documents and Settings\<user>\Application Data\Foo Bar``
+    Win 7 (roaming):
+      ``C:\Users\<user>\AppData\Roaming\Foo Bar``
+    Win 7 (not roaming):
+      ``C:\Users\<user>\AppData\Local\Foo Bar``
+
+    .. versionadded:: 2.0
+
+    :param app_name: the application name.  This should be properly capitalized
+                     and can contain whitespace.
+    :param roaming: controls if the folder should be roaming or not on Windows.
+                    Has no affect otherwise.
+    :param force_posix: if this is set to `True` then on any POSIX system the
+                        folder will be stored in the home folder with a leading
+                        dot instead of the XDG config home or darwin's
+                        application support folder.
+    """
+
+    def _posixify(name):
+        return "-".join(name.split()).lower()
+
+    # if WIN:
+    if platform.system() == 'Windows':
+        key = "APPDATA" if roaming else "LOCALAPPDATA"
+        folder = os.environ.get(key)
+        if folder is None:
+            folder = os.path.expanduser("~")
+        return os.path.join(folder, app_name)
+    if force_posix:
+        return os.path.join(os.path.expanduser("~/.{}".format(_posixify(app_name))))
+    if sys.platform == "darwin":
+        return os.path.join(
+            os.path.expanduser("~/Library/Application Support"), app_name
+        )
+    return os.path.join(
+        os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")),
+        _posixify(app_name),
+    )
+
+
+_config_dir = _click_config_dir = get_app_dir('piker')
+_parent_user = os.environ.get('SUDO_USER')
+
+if _parent_user:
+    non_root_user_dir = os.path.expanduser(
+        f'~{_parent_user}'
+    )
+    root = 'root'
+    _config_dir = (
+        non_root_user_dir +
+        _click_config_dir[
+            _click_config_dir.rfind(root) + len(root):
+        ]
+    )
+
 _file_name = 'brokers.toml'
+_watchlists_data_path = os.path.join(_config_dir, 'watchlists.json')
+_context_defaults = dict(
+    default_map={
+        # Questrade specific quote poll rates
+        'monitor': {
+            'rate': 3,
+        },
+        'optschain': {
+            'rate': 1,
+        },
+    }
+)
 
 
 def _override_config_dir(

piker/data/_ahab.py

@@ -0,0 +1,222 @@
+# piker: trading gear for hackers
+# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+Supervisor for docker with included specific-image service helpers.
+
+'''
+from typing import (
+    Optional,
+    # Any,
+)
+from contextlib import asynccontextmanager as acm
+# import time
+
+import trio
+import tractor
+import docker
+import json
+from docker.models.containers import Container
+
+from ..log import get_logger  # , get_console_log
+from ..config import _config_dir
+
+log = get_logger(__name__)
+
+
+_config = '''
+# piker's ``marketstore`` config.
+
+# mount this config using:
+# sudo docker run --mount \
+# type=bind,source="$HOME/.config/piker/",target="/etc" -i -p \
+# 5993:5993 alpacamarkets/marketstore:latest
+
+root_directory: data
+listen_port: 5993
+grpc_listen_port: 5995
+log_level: info
+queryable: true
+stop_grace_period: 0
+wal_rotate_interval: 5
+stale_threshold: 5
+enable_add: true
+enable_remove: false
+
+triggers:
+  - module: ondiskagg.so
+    on: "*/1Sec/OHLCV"
+    config:
+        # filter: "nasdaq"
+        destinations:
+            - 1Min
+            - 5Min
+            - 15Min
+            - 1H
+            - 1D
+
+  - module: stream.so
+    on: '*/*/*'
+    # config:
+    #     filter: "nasdaq"
+
+'''
+
+
+@acm
+async def open_docker(
+    url: Optional[str] = None,
+    **kwargs,
+
+) -> docker.DockerClient:
+
+    client = docker.DockerClient(
+        base_url=url,
+        **kwargs
+    ) if url else docker.from_env(**kwargs)
+
+    try:
+        yield client
+    finally:
+        # for c in client.containers.list():
+        #     c.kill()
+        client.close()
+        # client.api._custom_adapter.close()
+
+
+# async def waitfor(
+#     cntr: Container,
+#     attr_path: tuple[str],
+#     expect=None,
+#     timeout: float = 0.5,
+
+# ) -> Any:
+#     '''
+#     Wait for a container's attr value to be set. If ``expect`` is
+#     provided wait for the value to be set to that value.
+
+#     This is an async version of the helper from our ``pytest-dockerctl``
+#     plugin.
+
+#     '''
+#     def get(val, path):
+#         for key in path:
+#             val = val[key]
+#         return val
+
+#     start = time.time()
+#     while time.time() - start < timeout:
+#         cntr.reload()
+#         val = get(cntr.attrs, attr_path)
+#         if expect is None and val:
+#             return val
+#         elif val == expect:
+#             return val
+#     else:
+#         raise TimeoutError("{} failed to be {}, value: \"{}\"".format(
+#             attr_path, expect if expect else 'not None', val))
+
+
+@tractor.context
+async def open_marketstore_container(
+    ctx: tractor.Context,
+    **kwargs,
+
+) -> None:
+    '''
+    Start and supervise a marketstore instance with its config bind-mounted
+    in from the piker config directory on the system.
+
+    The equivalent cli cmd to this code is:
+
+        sudo docker run --mount \
+        type=bind,source="$HOME/.config/piker/",target="/etc" -i -p \
+        5993:5993 alpacamarkets/marketstore:latest
+
+    '''
+    # log = get_console_log('info', name=__name__)
+
+    async with open_docker() as client:
+        # create a mount from user's local piker config dir into container
+        config_dir_mnt = docker.types.Mount(
+            target='/etc',
+            source=_config_dir,
+            type='bind',
+        )
+
+        cntr: Container = client.containers.run(
+            'alpacamarkets/marketstore:latest',
+            # do we need this for cmds?
+            # '-i',
+
+            # '-p 5993:5993',
+            ports={'5993/tcp': 5993},
+            mounts=[config_dir_mnt],
+            detach=True,
+            # stop_signal='SIGINT',
+            # init=True,
+            # remove=True,
+        )
+        try:
+            started: bool = False
+            logs = cntr.logs(stream=True)
+
+            with trio.move_on_after(0.5):
+                for entry in logs:
+                    entry = entry.decode()
+                    try:
+                        record = json.loads(entry.strip())
+                    except json.JSONDecodeError:
+                        if 'Error' in entry:
+                            raise RuntimeError(entry)
+                    msg = record['msg']
+
+                    if "launching tcp listener for all services..." in msg:
+                        started = True
+                        break
+
+                    await trio.sleep(0)
+
+            if not started and cntr not in client.containers.list():
+                raise RuntimeError(
+                    'Failed to start `marketstore` check logs output for deats'
+                )
+
+            await ctx.started()
+            await trio.sleep_forever()
+
+        finally:
+            cntr.stop()
+
+
+async def main():
+
+    async with tractor.open_nursery(
+        loglevel='runtime',
+    ) as tn:
+        async with (
+            (
+                await tn.start_actor('ahab', enable_modules=[__name__])
+            ).open_context(
+                open_marketstore_container
+            ) as (ctx, first),
+        ):
+            assert not first
+            await trio.sleep_forever()
+
+
+if __name__ == '__main__':
+    trio.run(main)