slow down binance client.get_symbol_info() calls

Hitting API limits quite frequently when querying for symbol info on a
token that no longer exists.

Added a few breakers and a lru cache for repeated calls to the same
symbol.

Author: Azulinho

lib/bot.py

@@ -9,6 +9,7 @@
 from os.path import basename, exists
 from time import sleep
 from typing import Any, Dict, List, Tuple
+from functools import lru_cache
 
 import requests
 import udatetime
@@ -162,7 +163,9 @@ def __init__(
         # price.log service
         self.price_log_service: str = config["PRICE_LOG_SERVICE_URL"]
 
-    def extract_order_data(self, order_details, coin) -> Dict[str, Any]:
+    def extract_order_data(
+        self, order_details, coin
+    ) -> Tuple[bool, Dict[str, Any]]:
         """calculate average price and volume for a buy order"""
 
         # Each order will be fullfilled by different traders, and made of
@@ -183,13 +186,20 @@ def extract_order_data(self, order_details, coin) -> Dict[str, Any]:
 
         avg: float = total / qty
 
-        volume: float = float(self.calculate_volume_size(coin))
-        logging.debug(f"{coin.symbol} -> volume:{volume} avgPrice:{avg}")
+        ok, _volume = self.calculate_volume_size(coin)
+        if ok:
+            volume: float = float(_volume)
 
-        return {
-            "avgPrice": float(avg),
-            "volume": float(volume),
-        }
+            logging.debug(f"{coin.symbol} -> volume:{volume} avgPrice:{avg}")
+
+            return (
+                True,
+                {
+                    "avgPrice": float(avg),
+                    "volume": float(volume),
+                },
+            )
+        return (False, {})
 
     def run_strategy(self, coin) -> None:
         """runs a specific strategy against a coin"""
@@ -329,13 +339,17 @@ def place_sell_order(self, coin):
             orders = self.client.get_all_orders(symbol=coin.symbol, limit=1)
             logging.debug(orders)
             # calculate how much we got based on the total lines in our order
-            coin.price = self.extract_order_data(order_details, coin)[
-                "avgPrice"
-            ]
+            ok, _value = self.extract_order_data(order_details, coin)
+            if not ok:
+                return False
+
+            coin.price = _value["avgPrice"]
             # retrieve the total number of units for this coin
-            coin.volume = self.extract_order_data(order_details, coin)[
-                "volume"
-            ]
+            ok, _value = self.extract_order_data(order_details, coin)
+            if not ok:
+                return False
+
+            coin.volume = _value["volume"]
 
         # and give this coin a new fresh date based on our recent actions
         coin.date = float(udatetime.now().timestamp())
@@ -448,13 +462,15 @@ def place_buy_order(self, coin, volume):
             logging.debug(orders)
             # our order will have been fullfilled by different traders,
             # find out the average price we paid accross all these sales.
-            coin.bought_at = self.extract_order_data(order_details, coin)[
-                "avgPrice"
-            ]
+            ok, _value = self.extract_order_data(order_details, coin)
+            if not ok:
+                return False
+            coin.bought_at = float(_value["avgPrice"])
             # retrieve the total number of units for this coin
-            coin.volume = self.extract_order_data(order_details, coin)[
-                "volume"
-            ]
+            ok, _volume = self.extract_order_data(order_details, coin)
+            if not ok:
+                return False
+            coin.volume = float(_volume["volume"])
         with open("log/binance.place_buy_order.log", "at") as f:
             f.write(f"{coin.symbol} {coin.date} {self.order_type} ")
             f.write(f"{bid} {coin.volume} {order_details}\n")
@@ -477,7 +493,10 @@ def buy_coin(self, coin) -> bool:
 
         # calculate how many units of this coin we can afford based on our
         # investment share.
-        volume = float(self.calculate_volume_size(coin))
+        ok, _volume = self.calculate_volume_size(coin)
+        if not ok:
+            return False
+        volume: float = float(_volume)
 
         # we never place binance orders in backtesting mode.
         if self.mode in ["testnet", "live"]:
@@ -612,7 +631,8 @@ def sell_coin(self, coin) -> bool:
         )
         return True
 
-    def get_step_size(self, symbol: str) -> str:
+    @lru_cache(1024)
+    def get_step_size(self, symbol: str) -> Tuple[bool, str]:
         """retrieves and caches the decimal step size for a coin in binance"""
 
         # each coin in binance uses a number of decimal points, these can vary
@@ -631,29 +651,42 @@ def get_step_size(self, symbol: str) -> str:
         else:
             try:
                 info = self.client.get_symbol_info(symbol)
+
+                if not info:
+                    return (False, "")
+
+                if "filters" not in info:
+                    return (False, "")
             except BinanceAPIException as error_msg:
                 logging.error(error_msg)
-                return str(-1)
+                if "Too much request weight used;" in str(error_msg):
+                    sleep(60)
+                return (False, "")
 
         for d in info["filters"]:
             if "filterType" in d.keys():
                 if d["filterType"] == "LOT_SIZE":
                     step_size = d["stepSize"]
 
-        if self.mode == "backtesting" and not exists(f_path):
-            with open(f_path, "w") as f:
-                f.write(json.dumps(info))
+                    if self.mode == "backtesting" and not exists(f_path):
+                        with open(f_path, "w") as f:
+                            f.write(json.dumps(info))
 
-        with open("log/binance.step_size.log", "at") as f:
-            f.write(f"{symbol} {step_size}\n")
-        return step_size
+                    with open("log/binance.step_size.log", "at") as f:
+                        f.write(f"{symbol} {step_size}\n")
+                    return (True, step_size)
+        return (False, "")
 
-    def calculate_volume_size(self, coin) -> float:
+    def calculate_volume_size(self, coin) -> Tuple[bool, float]:
         """calculates the amount of coin we are to buy"""
 
         # calculates the number of units we are about to buy based on the number
         # of decimal points used, the share of the investment and the price
-        step_size: str = self.get_step_size(coin.symbol)
+        ok, _step_size = self.get_step_size(coin.symbol)
+        if ok:
+            step_size: str = _step_size
+        else:
+            return (False, 0)
 
         investment: float = percent(self.investment, self.re_invest_percentage)
 
@@ -667,7 +700,7 @@ def calculate_volume_size(self, coin) -> float:
             )
         with open("log/binance.volume.log", "at") as f:
             f.write(f"{coin.symbol} {step_size} {investment} {volume}\n")
-        return volume
+        return (True, volume)
 
     @retry(wait=wait_exponential(multiplier=1, max=3))
     def get_binance_prices(self) -> List[Dict[str, str]]:

lib/helpers.py

@@ -2,13 +2,16 @@
 import logging
 import math
 import pickle  # nosec
+import re
 from datetime import datetime
 from functools import lru_cache
 from os.path import exists, getctime
+from time import time, sleep
 
 import udatetime
 from binance.client import Client
 from filelock import SoftFileLock
+from tenacity import retry, wait_exponential
 
 
 def mean(values: list) -> float:
@@ -41,6 +44,7 @@ def c_from_timestamp(date: float) -> datetime:
     return datetime.fromtimestamp(date)
 
 
+@retry(wait=wait_exponential(multiplier=1, max=3))
 def cached_binance_client(access_key: str, secret_key: str) -> Client:
     """retry wrapper for binance client first call"""
 
@@ -64,6 +68,16 @@ def cached_binance_client(access_key: str, secret_key: str) -> Client:
                 _client = Client(access_key, secret_key)
             except Exception as err:
                 logging.warning(f"API client exception: {err}")
+                if "much request weight used" in str(err):
+                    timestamp = (
+                        int(re.findall(r"IP banned until (\d+)", str(err))[0])
+                        / 1000
+                    )
+                    logging.info(
+                        f"Pausing until {datetime.fromtimestamp(timestamp)}"
+                    )
+                    while int(time()) < timestamp:
+                        sleep(1)
                 raise Exception from err
             with open(cachefile, "wb") as f:
                 pickle.dump(_client, f)

tests/test_bot.py

@@ -320,7 +320,7 @@ def test_sell_coin_using_market_order_in_testnet(self, bot, coin):
                     ],
                 ) as _:
                     with mock.patch.object(
-                        bot, "get_step_size", return_value="0.00001000"
+                        bot, "get_step_size", return_value=(True, "0.00001000")
                     ) as _:
                         assert bot.sell_coin(coin) is True
                         assert bot.wallet == []
@@ -357,7 +357,7 @@ def test_sell_coin_using_limit_order_in_testnet(self, bot, coin):
                 },
             ) as _:
                 with mock.patch.object(
-                    bot, "get_step_size", return_value="0.00001000"
+                    bot, "get_step_size", return_value=(True, "0.00001000")
                 ) as _:
                     with mock.patch.object(
                         bot.client,
@@ -456,16 +456,17 @@ def test_get_step_size(self, bot):
             },
         ) as _:
             result = bot.get_step_size("BTCUSDT")
-            assert result == "0.00001000"
+            assert result == (True, "0.00001000")
 
     def test_extract_order_data(self):
         pass
 
     def test_calculate_volume_size(self, bot, coin):
         with mock.patch.object(
-            bot, "get_step_size", return_value="0.00001000"
+            bot, "get_step_size", return_value=(True, "0.00001000")
         ) as _:
-            volume = bot.calculate_volume_size(coin)
+            ok, volume = bot.calculate_volume_size(coin)
+            assert ok == True
             assert volume == 0.5
 
     def test_get_binance_prices(self, bot):
@@ -710,7 +711,7 @@ def test_buy_coin_when_coin_is_naughty(self, bot, coin):
         bot.buy_coin(coin)
         assert bot.wallet == []
 
-    @mock.patch("lib.bot.Bot.get_step_size", return_value="0.00001000")
+    @mock.patch("lib.bot.Bot.get_step_size", return_value=(True, "0.00001000"))
     def test_buy_coin_in_backtesting(self, _, bot, coin):
         bot.mode = "backtesting"
         coin.price = 100
@@ -767,7 +768,7 @@ def test_buy_coin_using_market_order_in_testnet(self, bot, coin):
                     ],
                 ) as _:
                     with mock.patch.object(
-                        bot, "get_step_size", return_value="0.00001000"
+                        bot, "get_step_size", return_value=(True, "0.00001000")
                     ) as _:
 
                         assert bot.buy_coin(coin) is True
@@ -803,7 +804,7 @@ def test_buy_coin_using_limit_order_in_testnet(self, bot, coin):
                 },
             ) as _:
                 with mock.patch.object(
-                    bot, "get_step_size", return_value="0.00001000"
+                    bot, "get_step_size", return_value=(True, "0.00001000")
                 ) as _:
                     with mock.patch.object(
                         bot.client,