initial commit

.gitignore

@@ -0,0 +1,4 @@
+__pycache__
+
+pgdata/*
+!pgdata/.mkdir

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.pythonPath": "/usr/local/bin/python3"
+}

Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3
+
+ADD . /app
+WORKDIR /app
+
+RUN pip install .
+CMD ["python3", "main.py"]

README.md

@@ -0,0 +1,7 @@
+# Binance DB
+
+A local cache for binance data, stored in Postgress
+
+## Setup
+
+TODO

binance_db/candle.py

@@ -0,0 +1,62 @@
+from datetime import datetime
+import binance_db.util.constants.ws as ws
+import binance_db.util.constants.rest as rest
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import Column, String, Integer, Float, DateTime
+
+Base = declarative_base()
+
+class Candle(Base):
+    __tablename__ = 'candles'
+
+    pair = Column(String, primary_key=True)
+    open_time = Column(DateTime, primary_key=True)
+    close_time = Column(DateTime)
+    open_price = Column(Float)
+    close_price = Column(Float)
+    high = Column(Float)
+    low = Column(Float)
+    volume = Column(Float)
+    qav = Column(Float)
+    trades = Column(Integer)
+    tbbav = Column(Float)
+    tbqav = Column(Float)
+
+    def __init__(self, pair, kline):
+        self.pair = pair
+        self.open_time = self.to_date(kline[rest.OPEN_TIME])
+        self.close_time = self.to_date(kline[rest.CLOSE_TIME])
+        self.open_price = float(kline[rest.OPEN_PRICE])
+        self.close_price = float(kline[rest.CLOSE_PRICE])
+        self.high = float(kline[rest.HIGH])
+        self.low = float(kline[rest.LOW])
+        self.volume = float(kline[rest.VOLUME])
+        self.qav = float(kline[rest.QAV])
+        self.trades = kline[rest.TRADES]
+        self.tbbav = float(kline[rest.TBBAV])
+        self.tbqav = float(kline[rest.TBQAV])
+
+    def __repr__(self):
+        date = self.open_time.strftime('%Y-%m-%d %H:%M:%S')
+        return "<Candle(pair={}, open_time={}, open={}, close={})>".format(
+                self.pair, date, self.open_price, self.close_price)
+
+    @staticmethod
+    def to_date(timestamp):
+        return datetime.utcfromtimestamp(timestamp / 1000)
+
+class WSCandle(Candle):
+    def __init__(self, ws_event):
+        self.pair = ws_event[ws.SYMBOL]
+        self.open_time = self.to_date(ws_event[ws.KLINE_DATA][ws.OPEN_TIME])
+        self.close_time = self.to_date(ws_event[ws.KLINE_DATA][ws.CLOSE_TIME])
+        self.open_price = float(ws_event[ws.KLINE_DATA][ws.OPEN_PRICE])
+        self.close_price = float(ws_event[ws.KLINE_DATA][ws.CLOSE_PRICE])
+        self.high = float(ws_event[ws.KLINE_DATA][ws.HIGH_PRICE])
+        self.low = float(ws_event[ws.KLINE_DATA][ws.LOW_PRICE])
+        self.volume = float(ws_event[ws.KLINE_DATA][ws.VOLUME])
+        self.qav = float(ws_event[ws.KLINE_DATA][ws.QAV])
+        self.trades = ws_event[ws.KLINE_DATA][ws.TRADES]
+        self.tbbav = float(ws_event[ws.KLINE_DATA][ws.TBBAV])
+        self.tbqav = float(ws_event[ws.KLINE_DATA][ws.TBQAV])
+        self.closed = ws_event[ws.KLINE_DATA][ws.IS_CLOSED]

binance_db/db.py

@@ -0,0 +1,13 @@
+from binance_db.candle import Candle
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+class BinanceDB():
+    def __init__(self, connstr, echo=False):
+        self.engine = create_engine(connstr, echo=echo)
+        Candle.metadata.create_all(self.engine)
+
+    def get_session(self):
+        Session = sessionmaker()
+        Session.configure(bind=self.engine)
+        return Session()

binance_db/util/constants/rest.py

@@ -0,0 +1,31 @@
+'''
+[
+  [
+    1499040000000,      // Open time
+    "0.01634790",       // Open
+    "0.80000000",       // High
+    "0.01575800",       // Low
+    "0.01577100",       // Close
+    "148976.11427815",  // Volume
+    1499644799999,      // Close time
+    "2434.19055334",    // Quote asset volume
+    308,                // Number of trades
+    "1756.87402397",    // Taker buy base asset volume
+    "28.46694368",      // Taker buy quote asset volume
+    "17928899.62484339" // Ignore
+  ]
+]
+'''
+
+OPEN_TIME = 0
+OPEN_PRICE = 1
+HIGH = 2
+LOW = 3
+CLOSE_PRICE = 4
+VOLUME = 5
+CLOSE_TIME = 6
+QAV = 7 # QUOTE_ASSET_VOLUME
+TRADES = 8 # NUMBER_OF_TRADES
+TBBAV = 9 # TAKER_BUY_BASE_ASSET_VOLUME
+TBQAV = 10 # TAKER_BUY_QUOTE_ASSET_VOLUME
+IGNORE = 11

binance_db/util/constants/ws.py

@@ -0,0 +1,51 @@
+'''
+{
+    "e": "kline",     // Event type
+    "E": 123456789,   // Event time
+    "s": "BNBBTC",    // Symbol
+    "k": {
+        "t": 123400000, // Kline start time
+        "T": 123460000, // Kline close time
+        "s": "BNBBTC",  // Symbol
+            "i": "1m",      // Interval
+            "f": 100,       // First trade ID
+            "L": 200,       // Last trade ID
+            "o": "0.0010",  // Open price
+            "c": "0.0020",  // Close price
+            "h": "0.0025",  // High price
+            "l": "0.0015",  // Low price
+            "v": "1000",    // Base asset volume
+            "n": 100,       // Number of trades
+            "x": false,     // Is this kline closed?
+            "q": "1.0000",  // Quote asset volume
+            "V": "500",     // Taker buy base asset volume
+            "Q": "0.500",   // Taker buy quote asset volume
+            "B": "123456"   // Ignore
+    }
+}'''
+
+KLINE_EVENT = "kline"
+ERROR_EVENT = "error"
+
+EVENT_TYPE = 'e'
+EVENT_TIME = 'E'
+SYMBOL = 's'
+KLINE_DATA = 'k'
+
+OPEN_TIME = 't'
+CLOSE_TIME = 'T'
+SYMBOL = 's'
+INTERVAL = 'i'
+FIRST_TRADE_ID = 'f'
+LAST_TRADE_ID = 'L'
+OPEN_PRICE = 'o'
+CLOSE_PRICE = 'c'
+HIGH_PRICE = 'h'
+LOW_PRICE = 'l'
+VOLUME = 'v'
+TRADES = 'n' # NUMBER_OF_TRADES
+IS_CLOSED = 'x'
+QAV = 'q' # QUOTE_ASSET_VOLUME
+TBBAV = 'V' # TAKER_BUY_BASE_ASSET_VOLUME
+TBQAV = 'Q' # TAKER_BUY_QUOTE_ASSET_VOLUME
+IGNORE = 'B'

binance_db/util/logger.py

@@ -0,0 +1,34 @@
+import logging
+
+class Singleton(type):
+    _instances = {}
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+        return cls._instances[cls]
+
+class Logger(metaclass=Singleton):
+    def __init__(self):
+        logger = logging.getLogger()
+        handler = logging.StreamHandler()
+
+        log_format = "[%(asctime)s] [%(process)d] [%(levelname)s] %(message)s"
+        time_format = "%Y-%m-%d %H:%M:%S %z"
+        formatter = logging.Formatter(log_format, time_format)
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+        logger.setLevel(logging.INFO)
+        self.logger = logger
+
+    # TODO: telegram messages
+    def info(self, msg):
+        self.logger.info(msg)
+
+    def debug(self, msg):
+        self.logger.debug(msg)
+
+    def warn(self, msg):
+        self.logger.warn(msg)
+
+    def error(self, msg):
+        self.logger.error(msg)

docker-compose.yml

@@ -0,0 +1,17 @@
+version: '3'
+services:
+  binance-db:
+    image: stoicperlman/binance-db
+    build:
+      context: .
+    restart: always
+    environment:
+      BDB_POSTGRESS_URL: postgress
+      BDB_POSTGRESS_USER: binancedb
+      BDB_POSTGRESS_PASS: ${BDB_POSTGRESS_PASS}
+  posgress:
+    image: postgres:alpine
+    restart: always
+    environment:
+      POSTGRESS_USER: binancedb
+      POSTGRESS_PASS: ${BDB_POSTGRESS_PASS}

main.py

@@ -0,0 +1,52 @@
+from binance_db.db import BinanceDB
+from binance_db.util.logger import Logger
+from binance_db.candle import Candle, WSCandle
+import binance_db.util.constants.ws as ws
+
+from binance.client import Client
+from binance.websockets import BinanceSocketManager
+
+logger = Logger()
+
+bdb = BinanceDB('sqlite:///:memory:', echo=True)
+db = bdb.get_session()
+
+client = Client(api_key='', api_secret='')
+bm = BinanceSocketManager(client)
+
+PAIR = 'BTCUSDT'
+INTERVAL = '1m'
+
+def main():
+    pws = lambda x: process_ws(x, db)
+    bm.start_kline_socket(PAIR, pws, interval=INTERVAL)
+    bm.start()
+    load_historical()
+
+def process_ws(msg, db):
+    if msg[ws.EVENT_TYPE] == ws.ERROR_EVENT:
+        logger.error(msg)
+        exit(1)
+
+    candle = WSCandle(msg)
+
+    if candle.closed:
+        logger.debug(f'New candle: {candle}')
+        db.add(candle)
+        db.commit()
+
+def load_historical():
+    klines = client.get_historical_klines('BTCUSDT', '1m', '10 minutes ago UTC')
+
+    # last isn't closed and will be added by ws
+    klines = klines[:-1]
+    candles = []
+
+    for kline in klines:
+        candles.append(Candle(PAIR, kline))
+
+    db.add_all(candles)
+    db.commit()
+
+if __name__ == '__main__':
+    main()

setup.py

@@ -0,0 +1,29 @@
+from setuptools import setup
+from os import path
+
+here = path.abspath(path.dirname(__file__))
+with open(path.join(here, 'README.md'), encoding='utf-8') as f:
+    long_description = f.read()
+
+setup(
+    name='binance-db',
+    version='0.0.1',
+    description='Binance data cache',
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    url='https://github.com/StoicPerlman/binance-db',
+    author='Sam Kleiner',
+    author_email='[email protected]',
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: MIT License',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+    ],
+    keywords='binance data cache',
+    packages=['binance_db'],
+    install_requires=['sqlalchemy', 'python-binance']
+)