From f0623f72d7721ff07109bb7f85a54074d341b256 Mon Sep 17 00:00:00 2001 From: Sam Kleiner Date: Sat, 16 Jun 2018 15:01:35 -0400 Subject: [PATCH] initial commit --- .gitignore | 4 ++ .vscode/settings.json | 3 ++ Dockerfile | 7 +++ README.md | 7 +++ binance_db/__init__.py | 0 binance_db/candle.py | 62 +++++++++++++++++++++++++++ binance_db/db.py | 13 ++++++ binance_db/util/__init__.py | 0 binance_db/util/constants/__init__.py | 0 binance_db/util/constants/rest.py | 31 ++++++++++++++ binance_db/util/constants/ws.py | 51 ++++++++++++++++++++++ binance_db/util/logger.py | 34 +++++++++++++++ docker-compose.yml | 17 ++++++++ main.py | 52 ++++++++++++++++++++++ pgdata/.mkdir | 0 setup.py | 29 +++++++++++++ 16 files changed, 310 insertions(+) create mode 100644 .gitignore create mode 100644 .vscode/settings.json create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 binance_db/__init__.py create mode 100644 binance_db/candle.py create mode 100644 binance_db/db.py create mode 100644 binance_db/util/__init__.py create mode 100644 binance_db/util/constants/__init__.py create mode 100644 binance_db/util/constants/rest.py create mode 100644 binance_db/util/constants/ws.py create mode 100644 binance_db/util/logger.py create mode 100644 docker-compose.yml create mode 100644 main.py create mode 100644 pgdata/.mkdir create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cbaf0a7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__ + +pgdata/* +!pgdata/.mkdir diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..988937c --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "/usr/local/bin/python3" +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e644f59 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3 + +ADD . /app +WORKDIR /app + +RUN pip install . +CMD ["python3", "main.py"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..04a3d6b --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +# Binance DB + +A local cache for binance data, stored in Postgress + +## Setup + +TODO diff --git a/binance_db/__init__.py b/binance_db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/binance_db/candle.py b/binance_db/candle.py new file mode 100644 index 0000000..bb19b31 --- /dev/null +++ b/binance_db/candle.py @@ -0,0 +1,62 @@ +from datetime import datetime +import binance_db.util.constants.ws as ws +import binance_db.util.constants.rest as rest +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Column, String, Integer, Float, DateTime + +Base = declarative_base() + +class Candle(Base): + __tablename__ = 'candles' + + pair = Column(String, primary_key=True) + open_time = Column(DateTime, primary_key=True) + close_time = Column(DateTime) + open_price = Column(Float) + close_price = Column(Float) + high = Column(Float) + low = Column(Float) + volume = Column(Float) + qav = Column(Float) + trades = Column(Integer) + tbbav = Column(Float) + tbqav = Column(Float) + + def __init__(self, pair, kline): + self.pair = pair + self.open_time = self.to_date(kline[rest.OPEN_TIME]) + self.close_time = self.to_date(kline[rest.CLOSE_TIME]) + self.open_price = float(kline[rest.OPEN_PRICE]) + self.close_price = float(kline[rest.CLOSE_PRICE]) + self.high = float(kline[rest.HIGH]) + self.low = float(kline[rest.LOW]) + self.volume = float(kline[rest.VOLUME]) + self.qav = float(kline[rest.QAV]) + self.trades = kline[rest.TRADES] + self.tbbav = float(kline[rest.TBBAV]) + self.tbqav = float(kline[rest.TBQAV]) + + def __repr__(self): + date = self.open_time.strftime('%Y-%m-%d %H:%M:%S') + return "".format( + self.pair, date, self.open_price, self.close_price) + + @staticmethod + def to_date(timestamp): + return datetime.utcfromtimestamp(timestamp / 1000) + +class WSCandle(Candle): + def __init__(self, ws_event): + self.pair = ws_event[ws.SYMBOL] + self.open_time = self.to_date(ws_event[ws.KLINE_DATA][ws.OPEN_TIME]) + self.close_time = self.to_date(ws_event[ws.KLINE_DATA][ws.CLOSE_TIME]) + self.open_price = float(ws_event[ws.KLINE_DATA][ws.OPEN_PRICE]) + self.close_price = float(ws_event[ws.KLINE_DATA][ws.CLOSE_PRICE]) + self.high = float(ws_event[ws.KLINE_DATA][ws.HIGH_PRICE]) + self.low = float(ws_event[ws.KLINE_DATA][ws.LOW_PRICE]) + self.volume = float(ws_event[ws.KLINE_DATA][ws.VOLUME]) + self.qav = float(ws_event[ws.KLINE_DATA][ws.QAV]) + self.trades = ws_event[ws.KLINE_DATA][ws.TRADES] + self.tbbav = float(ws_event[ws.KLINE_DATA][ws.TBBAV]) + self.tbqav = float(ws_event[ws.KLINE_DATA][ws.TBQAV]) + self.closed = ws_event[ws.KLINE_DATA][ws.IS_CLOSED] diff --git a/binance_db/db.py b/binance_db/db.py new file mode 100644 index 0000000..c4748ba --- /dev/null +++ b/binance_db/db.py @@ -0,0 +1,13 @@ +from binance_db.candle import Candle +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +class BinanceDB(): + def __init__(self, connstr, echo=False): + self.engine = create_engine(connstr, echo=echo) + Candle.metadata.create_all(self.engine) + + def get_session(self): + Session = sessionmaker() + Session.configure(bind=self.engine) + return Session() diff --git a/binance_db/util/__init__.py b/binance_db/util/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/binance_db/util/constants/__init__.py b/binance_db/util/constants/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/binance_db/util/constants/rest.py b/binance_db/util/constants/rest.py new file mode 100644 index 0000000..4d04882 --- /dev/null +++ b/binance_db/util/constants/rest.py @@ -0,0 +1,31 @@ +''' +[ + [ + 1499040000000, // Open time + "0.01634790", // Open + "0.80000000", // High + "0.01575800", // Low + "0.01577100", // Close + "148976.11427815", // Volume + 1499644799999, // Close time + "2434.19055334", // Quote asset volume + 308, // Number of trades + "1756.87402397", // Taker buy base asset volume + "28.46694368", // Taker buy quote asset volume + "17928899.62484339" // Ignore + ] +] +''' + +OPEN_TIME = 0 +OPEN_PRICE = 1 +HIGH = 2 +LOW = 3 +CLOSE_PRICE = 4 +VOLUME = 5 +CLOSE_TIME = 6 +QAV = 7 # QUOTE_ASSET_VOLUME +TRADES = 8 # NUMBER_OF_TRADES +TBBAV = 9 # TAKER_BUY_BASE_ASSET_VOLUME +TBQAV = 10 # TAKER_BUY_QUOTE_ASSET_VOLUME +IGNORE = 11 \ No newline at end of file diff --git a/binance_db/util/constants/ws.py b/binance_db/util/constants/ws.py new file mode 100644 index 0000000..1c93631 --- /dev/null +++ b/binance_db/util/constants/ws.py @@ -0,0 +1,51 @@ +''' +{ + "e": "kline", // Event type + "E": 123456789, // Event time + "s": "BNBBTC", // Symbol + "k": { + "t": 123400000, // Kline start time + "T": 123460000, // Kline close time + "s": "BNBBTC", // Symbol + "i": "1m", // Interval + "f": 100, // First trade ID + "L": 200, // Last trade ID + "o": "0.0010", // Open price + "c": "0.0020", // Close price + "h": "0.0025", // High price + "l": "0.0015", // Low price + "v": "1000", // Base asset volume + "n": 100, // Number of trades + "x": false, // Is this kline closed? + "q": "1.0000", // Quote asset volume + "V": "500", // Taker buy base asset volume + "Q": "0.500", // Taker buy quote asset volume + "B": "123456" // Ignore + } +}''' + +KLINE_EVENT = "kline" +ERROR_EVENT = "error" + +EVENT_TYPE = 'e' +EVENT_TIME = 'E' +SYMBOL = 's' +KLINE_DATA = 'k' + +OPEN_TIME = 't' +CLOSE_TIME = 'T' +SYMBOL = 's' +INTERVAL = 'i' +FIRST_TRADE_ID = 'f' +LAST_TRADE_ID = 'L' +OPEN_PRICE = 'o' +CLOSE_PRICE = 'c' +HIGH_PRICE = 'h' +LOW_PRICE = 'l' +VOLUME = 'v' +TRADES = 'n' # NUMBER_OF_TRADES +IS_CLOSED = 'x' +QAV = 'q' # QUOTE_ASSET_VOLUME +TBBAV = 'V' # TAKER_BUY_BASE_ASSET_VOLUME +TBQAV = 'Q' # TAKER_BUY_QUOTE_ASSET_VOLUME +IGNORE = 'B' \ No newline at end of file diff --git a/binance_db/util/logger.py b/binance_db/util/logger.py new file mode 100644 index 0000000..650bfc8 --- /dev/null +++ b/binance_db/util/logger.py @@ -0,0 +1,34 @@ +import logging + +class Singleton(type): + _instances = {} + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + return cls._instances[cls] + +class Logger(metaclass=Singleton): + def __init__(self): + logger = logging.getLogger() + handler = logging.StreamHandler() + + log_format = "[%(asctime)s] [%(process)d] [%(levelname)s] %(message)s" + time_format = "%Y-%m-%d %H:%M:%S %z" + formatter = logging.Formatter(log_format, time_format) + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.setLevel(logging.INFO) + self.logger = logger + + # TODO: telegram messages + def info(self, msg): + self.logger.info(msg) + + def debug(self, msg): + self.logger.debug(msg) + + def warn(self, msg): + self.logger.warn(msg) + + def error(self, msg): + self.logger.error(msg) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..7a3be13 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,17 @@ +version: '3' +services: + binance-db: + image: stoicperlman/binance-db + build: + context: . + restart: always + environment: + BDB_POSTGRESS_URL: postgress + BDB_POSTGRESS_USER: binancedb + BDB_POSTGRESS_PASS: ${BDB_POSTGRESS_PASS} + posgress: + image: postgres:alpine + restart: always + environment: + POSTGRESS_USER: binancedb + POSTGRESS_PASS: ${BDB_POSTGRESS_PASS} \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..b6a67aa --- /dev/null +++ b/main.py @@ -0,0 +1,52 @@ +from binance_db.db import BinanceDB +from binance_db.util.logger import Logger +from binance_db.candle import Candle, WSCandle +import binance_db.util.constants.ws as ws + +from binance.client import Client +from binance.websockets import BinanceSocketManager + +logger = Logger() + +bdb = BinanceDB('sqlite:///:memory:', echo=True) +db = bdb.get_session() + +client = Client(api_key='', api_secret='') +bm = BinanceSocketManager(client) + +PAIR = 'BTCUSDT' +INTERVAL = '1m' + +def main(): + pws = lambda x: process_ws(x, db) + bm.start_kline_socket(PAIR, pws, interval=INTERVAL) + bm.start() + load_historical() + +def process_ws(msg, db): + if msg[ws.EVENT_TYPE] == ws.ERROR_EVENT: + logger.error(msg) + exit(1) + + candle = WSCandle(msg) + + if candle.closed: + logger.debug(f'New candle: {candle}') + db.add(candle) + db.commit() + +def load_historical(): + klines = client.get_historical_klines('BTCUSDT', '1m', '10 minutes ago UTC') + + # last isn't closed and will be added by ws + klines = klines[:-1] + candles = [] + + for kline in klines: + candles.append(Candle(PAIR, kline)) + + db.add_all(candles) + db.commit() + +if __name__ == '__main__': + main() diff --git a/pgdata/.mkdir b/pgdata/.mkdir new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e2a21ec --- /dev/null +++ b/setup.py @@ -0,0 +1,29 @@ +from setuptools import setup +from os import path + +here = path.abspath(path.dirname(__file__)) +with open(path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + +setup( + name='binance-db', + version='0.0.1', + description='Binance data cache', + long_description=long_description, + long_description_content_type='text/markdown', + url='https://github.com/StoicPerlman/binance-db', + author='Sam Kleiner', + author_email='sam@skleiner.com', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + ], + keywords='binance data cache', + packages=['binance_db'], + install_requires=['sqlalchemy', 'python-binance'] +)