feat: log internal exceptions for notification purposes

dank_mids/controller.py

@@ -17,7 +17,8 @@
 from dank_mids._demo_mode import demo_logger
 from dank_mids.helpers import _session
 from dank_mids.loggers import main_logger, sort_lazy_logger
-from dank_mids.requests import JSONRPCBatch, Multicall, RPCRequest, eth_call
+from dank_mids.requests import (JSONRPCBatch, Multicall, RPCRequest,
+                                _log_exception, eth_call)
 from dank_mids.types import BlockId, ChainId
 from dank_mids.uid import UIDGenerator
 from dank_mids.worker import DankWorker
@@ -139,6 +140,7 @@ def done_callback(fut: asyncio.Future) -> None:
                     self._futs.remove(fut)
                 except ValueError as e:
                     if str(e) != "list.remove(x): x not in list":
+                        _log_exception(e)
                         raise
 
         fut = asyncio.ensure_future(self._exception_daemon())
@@ -170,6 +172,7 @@ async def _exception_daemon(self, _raise: bool = False) -> None:
                             self._futs.remove(fut)
                         except ValueError as e:
                             if str(e) != "list.remove(x): x not in list":
+                                _log_exception(e)
                                 raise
 
             await asyncio.sleep(5)

dank_mids/requests.py

@@ -35,6 +35,7 @@
     from dank_mids.worker import DankWorker
 
 RETRY_ERRS = ["connection reset by peer","request entity too large","server disconnected","execution aborted (timeout = 5s)"]
+
 
 class ResponseNotReady(Exception):
     pass
@@ -54,6 +55,7 @@ def _reattempt_call_and_return_exception(target: ChecksumAddress, calldata: byte
     try:
         return w3.eth.call({"to": target, "data": calldata}, block)
     except Exception as e:
+        _log_exception(e)
         return e
 
 def _err_response(e: Exception) -> RPCError:
@@ -78,6 +80,11 @@ def _err_response(e: Exception) -> RPCError:
         raise e
     return {'code': -32000, 'message': err_msg, 'data': ''}
 
+def _log_exception(e: Exception) -> None:
+    # TODO: Better filter what we choose to log here
+    if all(err not in str(e).lower() for err in RETRY_ERRS):
+        main_logger.exception(e)
+
 
 _Response = TypeVar("_Response", RPCResponse, List[RPCResponse])
 
@@ -375,6 +382,7 @@ async def get_response(self) -> None:
         try:
             await self.set_response(await self.worker(*self.params))
         except Exception as e:
+            _log_exception(e)
             await (self.bisect_and_retry() if self.should_retry(e) else self.set_response(e))  # type: ignore [misc]
         demo_logger.info(f'request {rid} for multicall {self.bid} complete')  # type: ignore
 
@@ -419,8 +427,10 @@ def post_sync(endpoint, data) -> Union[bytes, Tuple[str, Exception]]:
         response.raise_for_status()
         return response.json()
     except requests.HTTPError as e:
+        _log_exception(e)
         return e.args[0], e
     except Exception as e:
+        _log_exception(e)
         # This really shouldn't be running but just in case
         return response.reason, e
 
@@ -475,6 +485,7 @@ async def get_response(self) -> None:
             # NOTE: We do this inline so we don't have to allocate the response to memory
             await self.set_response(self.validate_responses(await self.post_sync()))
         except Exception as e:
+            _log_exception(e)
             await (self.bisect_and_retry() if self.should_retry(e) else self.set_response(e))
         demo_logger.info(f'request {rid} for jsonrpc batch {self.jid} complete')  # type: ignore
 
@@ -506,6 +517,7 @@ async def post(self) -> Union[Dict, List[bytes]]:
             try:
                 return await responses.json(content_type=responses.content_type)
             except Exception as e:
+                _log_exception(e)
                 counts = self.method_counts
                 decoded = responses._body.decode()
                 main_logger.info(f"json batch id: {self.jid} | len: {len(self)} | total calls: {self.total_calls}", )