Aave v2 event reader (#209)

Part of tradingstrategy-ai/trade-executor#888

.gitmodules

@@ -38,3 +38,6 @@
 [submodule "contracts/velvet"]
 	path = contracts/velvet
 	url = https://github.com/Velvet-Capital/protocol-v2-public.git
+[submodule "contracts/aave-v2"]
+	path = contracts/aave-v2
+	url = https://github.com/aave/protocol-v2.git

CHANGELOG.md

@@ -1,10 +1,11 @@
+# 0.25.6
+
+- Add Aave v2 event reader support
+
+
 # 0.25.5
 
 - Handle HTTP 410 retryable, as returned by dRPC
-- 
-# 0.25.4
-
-- TODO
 
 # 0.25.4
 

Makefile

@@ -73,6 +73,11 @@ aavev3:
 	@mkdir -p eth_defi/abi/aave_v3
 	@find contracts/aave-v3-deploy/artifacts/@aave -iname "*.json" -not -iname "*.dbg.json" -exec cp {} eth_defi/abi/aave_v3 \;
 
+aavev2:
+	@(cd contracts/aave-v2 && npm ci && npm run compile) > /dev/null
+	@mkdir -p eth_defi/abi/aave_v2
+	@find contracts/aave-v2/artifacts/contracts -iname "*.json" -not -iname "*.dbg.json" -exec cp {} eth_defi/abi/aave_v2 \;
+
 # Compile and copy Enzyme contract ABIs from their Github repository
 # Needs pnpm: curl -fsSL https://get.pnpm.io/install.sh | sh -
 #

docs/source/api/aave_v2/index.rst

@@ -0,0 +1,15 @@
+Aave v2 API
+-----------
+
+This is Python documentation for high-level `Aave lending protocol <https://tradingstrategy.ai/glossary/aave>`_ APIs.
+
+Functionality includes:
+
+- Reading current and historical Aave data and metrics
+
+.. autosummary::
+   :toctree: _autosummary_aave_v2
+   :recursive:
+
+   eth_defi.aave_v2.constants
+   eth_defi.aave_v2.events

docs/source/api/index.rst

@@ -20,6 +20,7 @@ API documentation
    usdc/index
    uniswap_v2/index
    uniswap_v3/index
+   aave_v2/index
    aave_v3/index
    one_delta/index
    enzyme/index

eth_defi/aave_v2/constants.py

@@ -0,0 +1,58 @@
+"""Aave v2 constants."""
+
+from typing import NamedTuple
+
+from eth_defi.aave_v3.constants import (  # noqa: passthrough imports, don't remove
+    MAX_AMOUNT,
+    AaveVersion,
+)
+
+
+class AaveV2Network(NamedTuple):
+    # Network name
+    name: str
+
+    # Aave v2 lending pool address
+    pool_address: str
+
+    # Block number when the pool was created
+    pool_created_at_block: int
+
+
+# https://docs.aave.com/developers/v/2.0/deployed-contracts/deployed-contracts
+AAVE_V2_NETWORK_CHAINS: dict[int, str] = {
+    1: "ethereum",
+    137: "polygon",
+    43114: "avalanche",
+}
+
+AAVE_V2_NETWORKS: dict[str, AaveV2Network] = {
+    # Ethereum Mainnet
+    "ethereum": AaveV2Network(
+        name="Ethereum",
+        pool_address="0x7d2768de32b0b80b7a3454c06bdac94a69ddc7a9",
+        # https://etherscan.io/tx/0x7d77cc7523a491fa670bfefa0a386ab036b6511d6d9fa6c2cf5c07b349dc9d3a
+        pool_created_at_block=11362579,
+    ),
+    # Polygon Mainnet
+    "polygon": AaveV2Network(
+        name="Polygon",
+        pool_address="0x8dFf5E27EA6b7AC08EbFdf9eB090F32ee9a30fcf",
+        # https://polygonscan.com/tx/0xb5a63fed49e97a58135b012fa14d83e680a0f3cd3aefeb551228d6e3640dbec9
+        pool_created_at_block=12687245,
+    ),
+    # Avalanche C-Chain
+    "avalanche": AaveV2Network(
+        name="Avalanche",
+        pool_address="0x4F01AeD16D97E3aB5ab2B501154DC9bb0F1A5A2C",
+        # https://snowtrace.io/tx/0x5db8b8c3026d4a433ca67cbc120540ab6f8897b3aff37e78ba014ac505d167bc?chainId=43114
+        pool_created_at_block=4607005,
+    ),
+}
+
+
+def get_aave_v2_network_by_chain_id(chain_id: int) -> AaveV2Network:
+    if chain_id not in AAVE_V2_NETWORK_CHAINS:
+        raise ValueError(f"Unsupported chain id: {chain_id}")
+    network_slug = AAVE_V2_NETWORK_CHAINS[chain_id]
+    return AAVE_V2_NETWORKS[network_slug]

eth_defi/aave_v2/events.py

@@ -0,0 +1,73 @@
+"""Aave v2 event reader.
+
+Efficiently read Aave v2 from a blockchain.
+
+Currently we are tracking these events:
+
+- ReserveDataUpdated
+"""
+
+import logging
+from typing import Callable
+
+from eth_defi.aave_v3.constants import AaveVersion
+from eth_defi.aave_v3.events import _fetch_aave_events_to_csv
+from eth_defi.event_reader.reorganisation_monitor import ReorganisationMonitor
+from eth_defi.event_reader.state import ScanState
+
+logger = logging.getLogger(__name__)
+
+
+def aave_v2_fetch_events_to_csv(
+    json_rpc_url: str,
+    state: ScanState,
+    aave_network_name: str,
+    start_block: int,
+    end_block: int,
+    output_folder: str = "/tmp",
+    max_workers: int = 16,
+    log_info: Callable = print,
+    reorg_monitor: ReorganisationMonitor | None = None,
+):
+    """Fetch all tracked Aave v2 events to CSV files for notebook analysis.
+
+    Creates a CSV file with the event data:
+
+    - `/tmp/aave-v2-{aave_network_name.lower()}-reservedataupdated.csv`
+
+    A progress bar and estimation on the completion is rendered for console / Jupyter notebook using `tqdm`.
+
+    The scan be resumed using `state` storage to retrieve the last scanned block number from the previous round.
+    However, the mechanism here is no perfect and only good for notebook use - for advanced
+    persistent usage like database backed scans, please write your own scan loop using proper transaction management.
+
+    .. note ::
+
+        Any Ethereum address is lowercased in the resulting dataset and is not checksummed.
+
+    :param json_rpc_url: JSON-RPC URL
+    :param start_block: First block to process (inclusive), default is block xxx (when Aave v2 xxx was created on mainnet)
+    :param end_block: Last block to process (inclusive), default is block xxx (1000 block after default start block)
+    :param aave_network_name: Network name, e.g. 'Polygon'
+    :param state: Store the current scan state, so we can resume
+    :param output_folder: Folder to contain output CSV files, default is /tmp folder
+    :param max_workers:
+        How many threads to allocate for JSON-RPC IO.
+        You can increase your EVM node output a bit by making a lot of parallel requests,
+        until you exhaust your nodes IO capacity. Experiement with different values
+        and see how your node performs.
+    :param log_info: Which function to use to output info messages about the progress
+    """
+
+    return _fetch_aave_events_to_csv(
+        json_rpc_url=json_rpc_url,
+        state=state,
+        aave_network_name=aave_network_name,
+        start_block=start_block,
+        end_block=end_block,
+        output_folder=output_folder,
+        max_workers=max_workers,
+        log_info=log_info,
+        reorg_monitor=reorg_monitor,
+        aave_version=AaveVersion.V2,
+    )

eth_defi/aave_v3/constants.py

@@ -1,4 +1,5 @@
 """Aave v3 constants."""
+
 import enum
 import json
 import os
@@ -218,6 +219,11 @@ class AaveV3InterestRateMode(enum.IntEnum):
     VARIABLE = 2
 
 
+class AaveVersion(enum.Enum):
+    V2 = "v2"
+    V3 = "v3"
+
+
 # Max amount user can withdraw or repay
 # This is type(uint256).max in solidity
 # https://github.com/aave/aave-v3-core/blob/e0bfed13240adeb7f05cb6cbe5e7ce78657f0621/contracts/protocol/libraries/logic/SupplyLogic.sol#L123

eth_defi/aave_v3/events.py

@@ -6,35 +6,38 @@
 
 - ReserveDataUpdated
 """
+
 import csv
 import datetime
 import logging
 from pathlib import Path
+from typing import Callable
 
 from requests.adapters import HTTPAdapter
 from tqdm.auto import tqdm
 from web3 import Web3
 
+from eth_defi.aave_v2.constants import AAVE_V2_NETWORKS
 from eth_defi.aave_v3.constants import (
     AAVE_V3_NETWORKS,
+    AaveVersion,
     aave_v3_get_token_name_by_deposit_address,
 )
 from eth_defi.abi import get_contract
 from eth_defi.event_reader.conversion import (
     convert_int256_bytes_to_int,
+    convert_jsonrpc_value_to_int,
     convert_uint256_string_to_address,
     decode_data,
-    convert_jsonrpc_value_to_int,
 )
 from eth_defi.event_reader.logresult import LogContext
 from eth_defi.event_reader.reader import LogResult, read_events_concurrent
+from eth_defi.event_reader.reorganisation_monitor import ReorganisationMonitor
 from eth_defi.event_reader.state import ScanState
 from eth_defi.event_reader.web3factory import TunedWeb3Factory
 from eth_defi.event_reader.web3worker import create_thread_pool_executor
 from eth_defi.token import TokenDetails, fetch_erc20_details
 
-# from eth_defi.token import TokenDetails, fetch_erc20_details
-
 logger = logging.getLogger(__name__)
 
 
@@ -96,8 +99,19 @@ def _decode_base(log: LogResult) -> dict:
     }
 
 
-def decode_reserve_data_updated(aave_network_name: str, log: LogResult) -> dict:
-    """Process a ReserveDataUpdated event. The event signature is:
+def decode_reserve_data_updated(
+    aave_network_name: str,
+    log: LogResult,
+    aave_version: AaveVersion,
+) -> dict:
+    """Process a ReserveDataUpdated event.
+
+    .. note ::
+
+        Both Aave v2 and v3 have this same event, so we use the lending pool smart
+        contract to filter out the correct events.
+
+    The event signature is:
 
     .. code-block::
 
@@ -112,7 +126,11 @@ def decode_reserve_data_updated(aave_network_name: str, log: LogResult) -> dict:
         );
     """
     # Ensure the event comes from the correct smart contract
-    if log["address"].lower() != AAVE_V3_NETWORKS[aave_network_name.lower()].pool_address.lower():
+    if aave_version == AaveVersion.V2:
+        pool_address = AAVE_V2_NETWORKS[aave_network_name.lower()].pool_address.lower()
+    else:
+        pool_address = AAVE_V3_NETWORKS[aave_network_name.lower()].pool_address.lower()
+    if log["address"].lower() != pool_address:
         return None
 
     # Do additional lookup for the token data
@@ -151,7 +169,10 @@ def decode_reserve_data_updated(aave_network_name: str, log: LogResult) -> dict:
     )
 
     # Detect token name from reserve address (None if not found)
-    result["token"] = aave_v3_get_token_name_by_deposit_address(deposit_address)
+    if aave_version == AaveVersion.V3:
+        result["token"] = aave_v3_get_token_name_by_deposit_address(deposit_address)
+    else:
+        result["token"] = None
 
     logger.debug(f'EVENT: block={log["blockNumber"]} tx={log["transactionHash"]} token={result["token"]} reserve={deposit_address} liquidity_rate={liquidity_rate} stable_borrow_rate={stable_borrow_rate} variable_borrow_rate={variable_borrow_rate} liquidity_index={liquidity_index} variable_borrow_index={variable_borrow_rate}')
 
@@ -166,7 +187,8 @@ def aave_v3_fetch_events_to_csv(
     end_block: int,
     output_folder: str = "/tmp",
     max_workers: int = 16,
-    log_info=print,
+    log_info: Callable = print,
+    reorg_monitor: ReorganisationMonitor | None = None,
 ):
     """Fetch all tracked Aave v3 events to CSV files for notebook analysis.
 
@@ -197,6 +219,34 @@ def aave_v3_fetch_events_to_csv(
         and see how your node performs.
     :param log_info: Which function to use to output info messages about the progress
     """
+
+    return _fetch_aave_events_to_csv(
+        json_rpc_url=json_rpc_url,
+        state=state,
+        aave_network_name=aave_network_name,
+        start_block=start_block,
+        end_block=end_block,
+        output_folder=output_folder,
+        max_workers=max_workers,
+        log_info=log_info,
+        reorg_monitor=reorg_monitor,
+        aave_version=AaveVersion.V3,
+    )
+
+
+def _fetch_aave_events_to_csv(
+    json_rpc_url: str,
+    state: ScanState,
+    aave_network_name: str,
+    start_block: int,
+    end_block: int,
+    output_folder: str = "/tmp",
+    max_workers: int = 16,
+    log_info=print,
+    reorg_monitor: ReorganisationMonitor | None = None,
+    aave_version: AaveVersion = AaveVersion.V3,
+):
+    """Fetch all tracked Aave (v2 or v3) events to CSV file."""
     token_cache = TokenCache()
     http_adapter = HTTPAdapter(pool_connections=max_workers, pool_maxsize=max_workers)
     web3_factory = TunedWeb3Factory(json_rpc_url, http_adapter)
@@ -222,7 +272,7 @@ def aave_v3_fetch_events_to_csv(
     buffers = {}
 
     for event_name, mapping in event_mapping.items():
-        file_path = f"{output_folder}/aave-v3-{aave_network_name.lower()}-{event_name.lower()}.csv"
+        file_path = f"{output_folder}/aave-{aave_version.value}-{aave_network_name.lower()}-{event_name.lower()}.csv"
         exists_already = Path(file_path).exists()
         file_handler = open(file_path, "a")
         csv_writer = csv.DictWriter(file_handler, fieldnames=mapping["field_names"])
@@ -288,13 +338,15 @@ def update_progress(
             notify=update_progress,
             chunk_size=100,
             context=token_cache,
+            reorg_mon=reorg_monitor,
+            extract_timestamps=None,
         ):
             try:
                 # write to correct buffer
                 event_name = log_result["event"].event_name
                 buffer = buffers[event_name]["buffer"]
                 decode_function = event_mapping[event_name]["decode_function"]
-                decoded_result = decode_function(aave_network_name, log_result)
+                decoded_result = decode_function(aave_network_name, log_result, aave_version)
                 # Note: decoded_result is None if the event is e.g. from Aave v2 contract
                 if decoded_result:
                     logger.debug(f"Adding event to buffer: {event_name}")