From 72e8f97e17579471a1d8b6556d0a79738da21df5 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 4 Nov 2024 16:43:34 +0000 Subject: [PATCH 1/6] Update metrics collector timeframe --- metrics/src/collector.py | 18 +++++++++--------- metrics/src/metrics_types.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/metrics/src/collector.py b/metrics/src/collector.py index fb5208e..0fd4bf0 100644 --- a/metrics/src/collector.py +++ b/metrics/src/collector.py @@ -21,7 +21,7 @@ import logging import asyncio import aiohttp -from datetime import datetime, timedelta +from datetime import datetime, date, timedelta from typing import Tuple, Optional, Dict, List from aiohttp import ClientError, ClientSession @@ -62,7 +62,7 @@ def get_empty_address_counter() -> AddressCounter: 'token_transfers_count': '0', 'transactions_count': '0', 'validations_count': '0', - 'transactions_last_day': 0, + 'transactions_today': 0, 'transactions_last_7_days': 0, 'transactions_last_30_days': 0, } @@ -81,22 +81,22 @@ async def fetch_address_data( response.raise_for_status() current_data: Dict = await response.json() + logger.debug(f'Explorer response for {address}: {json.dumps(current_data, indent=2)}') await update_transaction_counts(chain_name, app_name, address, current_data) - today = datetime.now().date() - yesterday = today - timedelta(days=1) + today = date.today() week_ago = today - timedelta(days=7) month_ago = today - timedelta(days=30) - transactions_last_day = await get_address_transaction_counts( - chain_name, app_name, address, yesterday, yesterday + transactions_today = await get_address_transaction_counts( + chain_name, app_name, address, today, today ) transactions_last_7_days = await get_address_transaction_counts( - chain_name, app_name, address, week_ago, yesterday + chain_name, app_name, address, week_ago, today ) transactions_last_30_days = await get_address_transaction_counts( - chain_name, app_name, address, month_ago, yesterday + chain_name, app_name, address, month_ago, today ) result: AddressCounter = { @@ -104,7 +104,7 @@ async def fetch_address_data( 'token_transfers_count': str(current_data.get('token_transfers_count', '0')), 'transactions_count': str(current_data.get('transactions_count', '0')), 'validations_count': str(current_data.get('validations_count', '0')), - 'transactions_last_day': transactions_last_day, + 'transactions_today': transactions_today, 'transactions_last_7_days': transactions_last_7_days, 'transactions_last_30_days': transactions_last_30_days, } diff --git a/metrics/src/metrics_types.py b/metrics/src/metrics_types.py index 8ee6c5e..949d441 100644 --- a/metrics/src/metrics_types.py +++ b/metrics/src/metrics_types.py @@ -27,7 +27,7 @@ class AddressCounter(TypedDict): token_transfers_count: str transactions_count: str validations_count: str - transactions_last_day: int + transactions_today: int transactions_last_7_days: int transactions_last_30_days: int From 50423fce732bdeba32a85ff5a097164b58bf74f6 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 4 Nov 2024 16:47:08 +0000 Subject: [PATCH 2/6] Update metrics collector tests --- metrics/tests/conftest.py | 2 +- metrics/tests/counters.json | 84 ++++++++++++------------- metrics/tests/test_metrics_collector.py | 4 +- 3 files changed, 45 insertions(+), 45 deletions(-) diff --git a/metrics/tests/conftest.py b/metrics/tests/conftest.py index 29d2501..54baed9 100644 --- a/metrics/tests/conftest.py +++ b/metrics/tests/conftest.py @@ -86,7 +86,7 @@ def latest_day_counters(): @pytest.fixture def mock_db_data(): return { - 'transactions_last_day': 50, + 'transactions_today': 50, 'transactions_last_7_days': 300, 'transactions_last_30_days': 1000, } diff --git a/metrics/tests/counters.json b/metrics/tests/counters.json index 59b3cef..c1fa3c7 100644 --- a/metrics/tests/counters.json +++ b/metrics/tests/counters.json @@ -7,7 +7,7 @@ "token_transfers_count": "123", "transactions_count": "1234", "validations_count": "12", - "transactions_last_day": 50, + "transactions_today": 50, "transactions_last_7_days": 250, "transactions_last_30_days": 1000 }, @@ -16,7 +16,7 @@ "token_transfers_count": "234", "transactions_count": "2345", "validations_count": "23", - "transactions_last_day": 60, + "transactions_today": 60, "transactions_last_7_days": 300, "transactions_last_30_days": 1200 }, @@ -25,7 +25,7 @@ "token_transfers_count": "345", "transactions_count": "3456", "validations_count": "34", - "transactions_last_day": 70, + "transactions_today": 70, "transactions_last_7_days": 350, "transactions_last_30_days": 1400 } @@ -36,7 +36,7 @@ "token_transfers_count": "456", "transactions_count": "4567", "validations_count": "45", - "transactions_last_day": 80, + "transactions_today": 80, "transactions_last_7_days": 400, "transactions_last_30_days": 1600 }, @@ -45,7 +45,7 @@ "token_transfers_count": "567", "transactions_count": "5678", "validations_count": "56", - "transactions_last_day": 90, + "transactions_today": 90, "transactions_last_7_days": 450, "transactions_last_30_days": 1800 } @@ -58,7 +58,7 @@ "token_transfers_count": "168", "transactions_count": "1689", "validations_count": "16", - "transactions_last_day": 5, + "transactions_today": 5, "transactions_last_7_days": 25, "transactions_last_30_days": 100 } @@ -73,7 +73,7 @@ "token_transfers_count": "125", "transactions_count": "1284", "validations_count": "13", - "transactions_last_day": 52, + "transactions_today": 52, "transactions_last_7_days": 252, "transactions_last_30_days": 1052 }, @@ -82,7 +82,7 @@ "token_transfers_count": "236", "transactions_count": "2405", "validations_count": "24", - "transactions_last_day": 62, + "transactions_today": 62, "transactions_last_7_days": 302, "transactions_last_30_days": 1262 }, @@ -91,7 +91,7 @@ "token_transfers_count": "347", "transactions_count": "3526", "validations_count": "35", - "transactions_last_day": 72, + "transactions_today": 72, "transactions_last_7_days": 352, "transactions_last_30_days": 1472 } @@ -102,7 +102,7 @@ "token_transfers_count": "458", "transactions_count": "4647", "validations_count": "46", - "transactions_last_day": 82, + "transactions_today": 82, "transactions_last_7_days": 402, "transactions_last_30_days": 1682 }, @@ -111,7 +111,7 @@ "token_transfers_count": "569", "transactions_count": "5768", "validations_count": "57", - "transactions_last_day": 92, + "transactions_today": 92, "transactions_last_7_days": 452, "transactions_last_30_days": 1892 } @@ -124,7 +124,7 @@ "token_transfers_count": "169", "transactions_count": "1694", "validations_count": "17", - "transactions_last_day": 6, + "transactions_today": 6, "transactions_last_7_days": 26, "transactions_last_30_days": 106 } @@ -139,7 +139,7 @@ "token_transfers_count": "127", "transactions_count": "1336", "validations_count": "14", - "transactions_last_day": 54, + "transactions_today": 54, "transactions_last_7_days": 254, "transactions_last_30_days": 1106 }, @@ -148,7 +148,7 @@ "token_transfers_count": "238", "transactions_count": "2467", "validations_count": "25", - "transactions_last_day": 64, + "transactions_today": 64, "transactions_last_7_days": 304, "transactions_last_30_days": 1326 }, @@ -157,7 +157,7 @@ "token_transfers_count": "349", "transactions_count": "3598", "validations_count": "36", - "transactions_last_day": 74, + "transactions_today": 74, "transactions_last_7_days": 354, "transactions_last_30_days": 1546 } @@ -168,7 +168,7 @@ "token_transfers_count": "460", "transactions_count": "4729", "validations_count": "47", - "transactions_last_day": 84, + "transactions_today": 84, "transactions_last_7_days": 404, "transactions_last_30_days": 1766 }, @@ -177,7 +177,7 @@ "token_transfers_count": "571", "transactions_count": "5860", "validations_count": "58", - "transactions_last_day": 94, + "transactions_today": 94, "transactions_last_7_days": 454, "transactions_last_30_days": 1986 } @@ -190,7 +190,7 @@ "token_transfers_count": "170", "transactions_count": "1700", "validations_count": "18", - "transactions_last_day": 7, + "transactions_today": 7, "transactions_last_7_days": 27, "transactions_last_30_days": 113 } @@ -205,7 +205,7 @@ "token_transfers_count": "129", "transactions_count": "1390", "validations_count": "15", - "transactions_last_day": 56, + "transactions_today": 56, "transactions_last_7_days": 256, "transactions_last_30_days": 1162 }, @@ -214,7 +214,7 @@ "token_transfers_count": "240", "transactions_count": "2531", "validations_count": "26", - "transactions_last_day": 66, + "transactions_today": 66, "transactions_last_7_days": 306, "transactions_last_30_days": 1392 }, @@ -223,7 +223,7 @@ "token_transfers_count": "351", "transactions_count": "3672", "validations_count": "37", - "transactions_last_day": 76, + "transactions_today": 76, "transactions_last_7_days": 356, "transactions_last_30_days": 1622 } @@ -234,7 +234,7 @@ "token_transfers_count": "462", "transactions_count": "4813", "validations_count": "48", - "transactions_last_day": 86, + "transactions_today": 86, "transactions_last_7_days": 406, "transactions_last_30_days": 1852 }, @@ -243,7 +243,7 @@ "token_transfers_count": "573", "transactions_count": "5954", "validations_count": "59", - "transactions_last_day": 96, + "transactions_today": 96, "transactions_last_7_days": 456, "transactions_last_30_days": 2082 } @@ -256,7 +256,7 @@ "token_transfers_count": "171", "transactions_count": "1707", "validations_count": "19", - "transactions_last_day": 8, + "transactions_today": 8, "transactions_last_7_days": 28, "transactions_last_30_days": 121 } @@ -271,7 +271,7 @@ "token_transfers_count": "131", "transactions_count": "1446", "validations_count": "16", - "transactions_last_day": 58, + "transactions_today": 58, "transactions_last_7_days": 258, "transactions_last_30_days": 1220 }, @@ -280,7 +280,7 @@ "token_transfers_count": "242", "transactions_count": "2597", "validations_count": "27", - "transactions_last_day": 68, + "transactions_today": 68, "transactions_last_7_days": 308, "transactions_last_30_days": 1460 }, @@ -289,7 +289,7 @@ "token_transfers_count": "353", "transactions_count": "3748", "validations_count": "38", - "transactions_last_day": 78, + "transactions_today": 78, "transactions_last_7_days": 358, "transactions_last_30_days": 1700 } @@ -300,7 +300,7 @@ "token_transfers_count": "464", "transactions_count": "4899", "validations_count": "49", - "transactions_last_day": 88, + "transactions_today": 88, "transactions_last_7_days": 408, "transactions_last_30_days": 1940 }, @@ -309,7 +309,7 @@ "token_transfers_count": "575", "transactions_count": "6050", "validations_count": "60", - "transactions_last_day": 98, + "transactions_today": 98, "transactions_last_7_days": 458, "transactions_last_30_days": 2180 } @@ -322,7 +322,7 @@ "token_transfers_count": "172", "transactions_count": "1715", "validations_count": "20", - "transactions_last_day": 9, + "transactions_today": 9, "transactions_last_7_days": 29, "transactions_last_30_days": 130 } @@ -337,7 +337,7 @@ "token_transfers_count": "133", "transactions_count": "1504", "validations_count": "17", - "transactions_last_day": 60, + "transactions_today": 60, "transactions_last_7_days": 260, "transactions_last_30_days": 1280 }, @@ -346,7 +346,7 @@ "token_transfers_count": "244", "transactions_count": "2665", "validations_count": "28", - "transactions_last_day": 70, + "transactions_today": 70, "transactions_last_7_days": 310, "transactions_last_30_days": 1530 }, @@ -355,7 +355,7 @@ "token_transfers_count": "355", "transactions_count": "3826", "validations_count": "39", - "transactions_last_day": 80, + "transactions_today": 80, "transactions_last_7_days": 360, "transactions_last_30_days": 1780 } @@ -366,7 +366,7 @@ "token_transfers_count": "466", "transactions_count": "4987", "validations_count": "50", - "transactions_last_day": 90, + "transactions_today": 90, "transactions_last_7_days": 410, "transactions_last_30_days": 2030 }, @@ -375,7 +375,7 @@ "token_transfers_count": "577", "transactions_count": "6148", "validations_count": "61", - "transactions_last_day": 100, + "transactions_today": 100, "transactions_last_7_days": 460, "transactions_last_30_days": 2280 } @@ -388,7 +388,7 @@ "token_transfers_count": "173", "transactions_count": "1724", "validations_count": "21", - "transactions_last_day": 10, + "transactions_today": 10, "transactions_last_7_days": 30, "transactions_last_30_days": 140 } @@ -403,7 +403,7 @@ "token_transfers_count": "135", "transactions_count": "1564", "validations_count": "18", - "transactions_last_day": 62, + "transactions_today": 62, "transactions_last_7_days": 262, "transactions_last_30_days": 1342 }, @@ -412,7 +412,7 @@ "token_transfers_count": "246", "transactions_count": "2735", "validations_count": "29", - "transactions_last_day": 72, + "transactions_today": 72, "transactions_last_7_days": 312, "transactions_last_30_days": 1602 }, @@ -421,7 +421,7 @@ "token_transfers_count": "357", "transactions_count": "3906", "validations_count": "40", - "transactions_last_day": 82, + "transactions_today": 82, "transactions_last_7_days": 362, "transactions_last_30_days": 1862 } @@ -432,7 +432,7 @@ "token_transfers_count": "468", "transactions_count": "5077", "validations_count": "51", - "transactions_last_day": 92, + "transactions_today": 92, "transactions_last_7_days": 412, "transactions_last_30_days": 2122 }, @@ -441,7 +441,7 @@ "token_transfers_count": "579", "transactions_count": "6248", "validations_count": "62", - "transactions_last_day": 102, + "transactions_today": 102, "transactions_last_7_days": 462, "transactions_last_30_days": 2382 } @@ -454,7 +454,7 @@ "token_transfers_count": "174", "transactions_count": "1734", "validations_count": "22", - "transactions_last_day": 11, + "transactions_today": 11, "transactions_last_7_days": 31, "transactions_last_30_days": 151 } diff --git a/metrics/tests/test_metrics_collector.py b/metrics/tests/test_metrics_collector.py index 1fa579f..77bd3ac 100644 --- a/metrics/tests/test_metrics_collector.py +++ b/metrics/tests/test_metrics_collector.py @@ -22,7 +22,7 @@ async def test_fetch_address_data_success( patch('src.collector.get_address_transaction_counts') as mock_get_counts, ): mock_get_counts.side_effect = [ - mock_db_data['transactions_last_day'], + mock_db_data['transactions_today'], mock_db_data['transactions_last_7_days'], mock_db_data['transactions_last_30_days'], ] @@ -39,7 +39,7 @@ async def test_fetch_address_data_success( assert result['validations_count'] == mock_address_data['validations_count'] # Verify historical data - assert result['transactions_last_day'] == mock_db_data['transactions_last_day'] + assert result['transactions_today'] == mock_db_data['transactions_today'] assert result['transactions_last_7_days'] == mock_db_data['transactions_last_7_days'] assert result['transactions_last_30_days'] == mock_db_data['transactions_last_30_days'] From adaceb5bea472bee859f3563823415929fcb031a Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 4 Nov 2024 19:09:20 +0000 Subject: [PATCH 3/6] Add DB to test pipeline, add test for get_db_counts --- .github/workflows/test_metrics.yml | 9 ++ metrics/requirements-dev.txt | 3 +- metrics/src/collector.py | 53 ++++---- metrics/tests/conftest.py | 24 +++- metrics/tests/prepare_db.py | 82 +++++++++++++ metrics/tests/test_data.json | 156 ++++++++++++++++++++++++ metrics/tests/test_metrics_collector.py | 35 ++++-- 7 files changed, 322 insertions(+), 40 deletions(-) create mode 100644 metrics/tests/prepare_db.py create mode 100644 metrics/tests/test_data.json diff --git a/.github/workflows/test_metrics.yml b/.github/workflows/test_metrics.yml index d005486..d806e16 100644 --- a/.github/workflows/test_metrics.yml +++ b/.github/workflows/test_metrics.yml @@ -13,6 +13,11 @@ jobs: env: ETH_ENDPOINT: ${{ secrets.ETH_ENDPOINT }} PYTHONPATH: ${{ github.workspace }}/metrics + MYSQL_USER: test + MYSQL_PASSWORD: test + MYSQL_ROOT_PASSWORD: test + MYSQL_HOST: 127.0.0.1 + MYSQL_DATABASE: metrics steps: - uses: actions/checkout@v4 with: @@ -26,5 +31,9 @@ jobs: run: pip install -r requirements.txt && pip install -r requirements-dev.txt - name: Lint with ruff run: ruff check src/ + - name: Run MySQL container + run: docker compose up -d mysql + - name: Prepare database + run: python tests/prepare_db.py - name: Run metrics tests run: pytest tests/ diff --git a/metrics/requirements-dev.txt b/metrics/requirements-dev.txt index f5896f0..21b4e2b 100644 --- a/metrics/requirements-dev.txt +++ b/metrics/requirements-dev.txt @@ -3,4 +3,5 @@ pytest==8.3.3 Faker==28.4.1 pytest-aiohttp==1.0.5 eth-typing==4.0.0 -eth-utils==4.0.0 \ No newline at end of file +eth-utils==4.0.0 +types-peewee==3.17.7.20241017 \ No newline at end of file diff --git a/metrics/src/collector.py b/metrics/src/collector.py index 0fd4bf0..7663ddf 100644 --- a/metrics/src/collector.py +++ b/metrics/src/collector.py @@ -84,34 +84,39 @@ async def fetch_address_data( logger.debug(f'Explorer response for {address}: {json.dumps(current_data, indent=2)}') await update_transaction_counts(chain_name, app_name, address, current_data) - - today = date.today() - week_ago = today - timedelta(days=7) - month_ago = today - timedelta(days=30) - - transactions_today = await get_address_transaction_counts( - chain_name, app_name, address, today, today - ) - transactions_last_7_days = await get_address_transaction_counts( - chain_name, app_name, address, week_ago, today - ) - transactions_last_30_days = await get_address_transaction_counts( - chain_name, app_name, address, month_ago, today - ) - - result: AddressCounter = { - 'gas_usage_count': str(current_data.get('gas_usage_count', '0')), - 'token_transfers_count': str(current_data.get('token_transfers_count', '0')), - 'transactions_count': str(current_data.get('transactions_count', '0')), - 'validations_count': str(current_data.get('validations_count', '0')), - 'transactions_today': transactions_today, - 'transactions_last_7_days': transactions_last_7_days, - 'transactions_last_30_days': transactions_last_30_days, - } + result = await get_db_counts(current_data, chain_name, app_name, address) logger.info(f'Fetched data for {address} at {url}: {result}') return result +async def get_db_counts( + current_data: Dict, chain_name: str, app_name: str, address: str +) -> AddressCounter: + today = date.today() + tomorrow = today + timedelta(days=1) + week_ago = today - timedelta(days=7) + month_ago = today - timedelta(days=30) + + transactions_today = await get_address_transaction_counts( + chain_name, app_name, address, today, tomorrow + ) + transactions_last_7_days = await get_address_transaction_counts( + chain_name, app_name, address, week_ago, today + ) + transactions_last_30_days = await get_address_transaction_counts( + chain_name, app_name, address, month_ago, today + ) + return { + 'gas_usage_count': str(current_data.get('gas_usage_count', '0')), + 'token_transfers_count': str(current_data.get('token_transfers_count', '0')), + 'transactions_count': str(current_data.get('transactions_count', '0')), + 'validations_count': str(current_data.get('validations_count', '0')), + 'transactions_today': transactions_today, + 'transactions_last_7_days': transactions_last_7_days, + 'transactions_last_30_days': transactions_last_30_days, + } + + async def get_address_counters( session: ClientSession, network: str, chain_name: str, app_name: str, address: str ) -> AddressCounter: diff --git a/metrics/tests/conftest.py b/metrics/tests/conftest.py index 54baed9..17a406a 100644 --- a/metrics/tests/conftest.py +++ b/metrics/tests/conftest.py @@ -1,9 +1,11 @@ import os import json import pytest +from datetime import date from faker import Faker from aiohttp import web - +from unittest.mock import patch +from metrics.tests.prepare_db import load_test_data fake = Faker() @@ -41,9 +43,11 @@ } TEST_NETWORK = 'testnet' -TEST_CHAIN = 'chain2' -TEST_ADDRESS = '0x1234' +TEST_CHAIN = 'test-chain' +TEST_ADDRESS = '0x1234567890123456789012345678901234567890' TEST_APP = 'test-app' +MOCK_DATE = date(2024, 5, 4) +TEST_DATA = load_test_data() @pytest.fixture @@ -61,6 +65,11 @@ def sample_metadata(): return SAMPLE_METADATA +@pytest.fixture +def sample_counters(): + return TEST_DATA + + def load_counters(): current_dir = os.path.dirname(os.path.abspath(__file__)) json_file_path = os.path.join(current_dir, 'counters.json') @@ -93,7 +102,7 @@ def mock_db_data(): @pytest.fixture -def mock_address_data(): +def address_data(): return { 'gas_usage_count': '16935', 'token_transfers_count': '174', @@ -102,6 +111,13 @@ def mock_address_data(): } +@pytest.fixture +def mock_today(): + with patch('src.collector.date') as mock_date: + mock_date.today.return_value = MOCK_DATE + yield mock_date + + @pytest.fixture def mock_chain_stats_data(): return CHAIN_STATS diff --git a/metrics/tests/prepare_db.py b/metrics/tests/prepare_db.py new file mode 100644 index 0000000..1b98f37 --- /dev/null +++ b/metrics/tests/prepare_db.py @@ -0,0 +1,82 @@ +import os +import json +import sys +import logging +from datetime import datetime +from peewee import IntegrityError + +from src.models import db, Address, TransactionCount + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def init_database(): + logger.info('Initializing test database...') + + try: + db.connect() + db.create_tables([Address, TransactionCount]) + logger.info('Database tables created successfully') + except Exception as e: + logger.error(f'Failed to create tables: {e}') + sys.exit(1) + + +def load_test_data(): + data_file = os.path.join(os.path.dirname(__file__), 'test_data.json') + try: + with open(data_file) as f: + return json.load(f) + except Exception as e: + logger.error(f'Failed to load test data: {e}') + sys.exit(1) + + +def populate_database(test_data): + chain_name = test_data['chain_name'] + app_name = test_data['app_name'] + + with db.atomic(): + for address_data in test_data['addresses']: + address = address_data['address'] + + try: + addr_record = Address.create( + chain_name=chain_name, address=address, app_name=app_name + ) + logger.info(f'Created address record for {address}') + except IntegrityError: + logger.info(f'Address {address} already exists, skipping creation') + addr_record = Address.get(Address.address == address) + + for day_data in address_data['daily_data']: + try: + TransactionCount.create( + address=addr_record, + date=datetime.strptime(day_data['date'], '%Y-%m-%d').date(), + total_transactions=day_data['total_transactions'], + daily_transactions=day_data['daily_transactions'], + ) + logger.info(f"Created transaction record for {address} on {day_data['date']}") + except IntegrityError: + logger.info( + f"Transaction record for {address} on {day_data['date']} already exists" + ) + + +def main(): + try: + init_database() + test_data = load_test_data() + populate_database(test_data) + logger.info('Database initialization completed successfully') + except Exception as e: + logger.error(f'Failed to initialize database: {e}') + sys.exit(1) + finally: + db.close() + + +if __name__ == '__main__': + main() diff --git a/metrics/tests/test_data.json b/metrics/tests/test_data.json new file mode 100644 index 0000000..7939c84 --- /dev/null +++ b/metrics/tests/test_data.json @@ -0,0 +1,156 @@ +{ + "chain_name": "test-chain", + "app_name": "test-app", + "addresses": [ + { + "address": "0x1234567890123456789012345678901234567890", + "daily_data": [ + { + "date": "2024-04-21", + "total_transactions": 1000, + "daily_transactions": 50 + }, + { + "date": "2024-04-22", + "total_transactions": 1050, + "daily_transactions": 60 + }, + { + "date": "2024-04-23", + "total_transactions": 1110, + "daily_transactions": 45 + }, + { + "date": "2024-04-24", + "total_transactions": 1155, + "daily_transactions": 70 + }, + { + "date": "2024-04-25", + "total_transactions": 1225, + "daily_transactions": 55 + }, + { + "date": "2024-04-26", + "total_transactions": 1280, + "daily_transactions": 65 + }, + { + "date": "2024-04-27", + "total_transactions": 1345, + "daily_transactions": 40 + }, + { + "date": "2024-04-28", + "total_transactions": 1385, + "daily_transactions": 75 + }, + { + "date": "2024-04-29", + "total_transactions": 1460, + "daily_transactions": 80 + }, + { + "date": "2024-04-30", + "total_transactions": 1540, + "daily_transactions": 45 + }, + { + "date": "2024-05-01", + "total_transactions": 1585, + "daily_transactions": 70 + }, + { + "date": "2024-05-02", + "total_transactions": 1655, + "daily_transactions": 55 + }, + { + "date": "2024-05-03", + "total_transactions": 1710, + "daily_transactions": 65 + }, + { + "date": "2024-05-04", + "total_transactions": 1775, + "daily_transactions": 40 + } + ] + }, + { + "address": "0x9876543210987654321098765432109876543210", + "daily_data": [ + { + "date": "2024-04-21", + "total_transactions": 2000, + "daily_transactions": 100 + }, + { + "date": "2024-04-22", + "total_transactions": 2100, + "daily_transactions": 90 + }, + { + "date": "2024-04-23", + "total_transactions": 2190, + "daily_transactions": 85 + }, + { + "date": "2024-04-24", + "total_transactions": 2275, + "daily_transactions": 95 + }, + { + "date": "2024-04-25", + "total_transactions": 2370, + "daily_transactions": 110 + }, + { + "date": "2024-04-26", + "total_transactions": 2480, + "daily_transactions": 80 + }, + { + "date": "2024-04-27", + "total_transactions": 2560, + "daily_transactions": 105 + }, + { + "date": "2024-04-28", + "total_transactions": 2665, + "daily_transactions": 115 + }, + { + "date": "2024-04-29", + "total_transactions": 2780, + "daily_transactions": 95 + }, + { + "date": "2024-04-30", + "total_transactions": 2875, + "daily_transactions": 100 + }, + { + "date": "2024-05-01", + "total_transactions": 2975, + "daily_transactions": 90 + }, + { + "date": "2024-05-02", + "total_transactions": 3065, + "daily_transactions": 105 + }, + { + "date": "2024-05-03", + "total_transactions": 3170, + "daily_transactions": 85 + }, + { + "date": "2024-05-04", + "total_transactions": 3255, + "daily_transactions": 95 + } + ] + } + ] +} \ No newline at end of file diff --git a/metrics/tests/test_metrics_collector.py b/metrics/tests/test_metrics_collector.py index 77bd3ac..314853f 100644 --- a/metrics/tests/test_metrics_collector.py +++ b/metrics/tests/test_metrics_collector.py @@ -1,7 +1,7 @@ import pytest from unittest.mock import patch from typing import Dict -from src.collector import get_chain_stats, fetch_address_data +from src.collector import get_chain_stats, fetch_address_data, get_db_counts from conftest import TEST_NETWORK, TEST_CHAIN, TEST_ADDRESS, TEST_APP pytestmark = pytest.mark.asyncio @@ -15,7 +15,7 @@ async def test_get_chain_stats(mock_chain_stats_data: Dict, client, mock_explore async def test_fetch_address_data_success( - client, mock_explorer_url, mock_address_data: Dict[str, str], mock_db_data: Dict[str, int] + client, mock_explorer_url, address_data: Dict[str, str], mock_db_data: Dict[str, int] ) -> None: with ( patch('src.collector.update_transaction_counts') as mock_update, @@ -31,17 +31,30 @@ async def test_fetch_address_data_success( client, '/api/v2/addresses/0x1234/counters', TEST_CHAIN, TEST_APP, TEST_ADDRESS ) - # Verify the result type and content assert isinstance(result, dict) - assert result['gas_usage_count'] == mock_address_data['gas_usage_count'] - assert result['token_transfers_count'] == mock_address_data['token_transfers_count'] - assert result['transactions_count'] == mock_address_data['transactions_count'] - assert result['validations_count'] == mock_address_data['validations_count'] - - # Verify historical data + assert result['gas_usage_count'] == address_data['gas_usage_count'] + assert result['token_transfers_count'] == address_data['token_transfers_count'] + assert result['transactions_count'] == address_data['transactions_count'] + assert result['validations_count'] == address_data['validations_count'] assert result['transactions_today'] == mock_db_data['transactions_today'] assert result['transactions_last_7_days'] == mock_db_data['transactions_last_7_days'] assert result['transactions_last_30_days'] == mock_db_data['transactions_last_30_days'] - - # Verify database was updated mock_update.assert_called_once() + + +async def test_get_address_counters(mock_today, sample_counters, address_data) -> None: + result = await get_db_counts(address_data, TEST_CHAIN, TEST_APP, TEST_ADDRESS) + assert isinstance(result, dict) + assert result['gas_usage_count'] == address_data['gas_usage_count'] + assert result['token_transfers_count'] == address_data['token_transfers_count'] + assert result['transactions_count'] == address_data['transactions_count'] + assert result['validations_count'] == address_data['validations_count'] + assert 'transactions_today' in result + assert 'transactions_last_7_days' in result + assert 'transactions_last_30_days' in result + + current_day_counters = sample_counters['addresses'][0]['daily_data'][-1] + + current_day_counters['transactions_today'] = result['transactions_today'] + current_day_counters['transactions_last_7_days'] = result['transactions_last_7_days'] + current_day_counters['transactions_last_30_days'] = result['transactions_last_30_days'] From ddf5166f5af4f78894742786eda10ced08eaff2b Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 4 Nov 2024 19:11:37 +0000 Subject: [PATCH 4/6] Fix db service name in test pipeline --- .github/workflows/test_metrics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_metrics.yml b/.github/workflows/test_metrics.yml index d806e16..6f65919 100644 --- a/.github/workflows/test_metrics.yml +++ b/.github/workflows/test_metrics.yml @@ -32,7 +32,7 @@ jobs: - name: Lint with ruff run: ruff check src/ - name: Run MySQL container - run: docker compose up -d mysql + run: docker compose up -d db - name: Prepare database run: python tests/prepare_db.py - name: Run metrics tests From 793b1b3a379cd6e234100ad4be55ef6e1c10b7be Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 4 Nov 2024 19:19:52 +0000 Subject: [PATCH 5/6] Add retries for db connecton for prepare db --- metrics/tests/prepare_db.py | 109 ++++++++++++++++++++++++++---------- 1 file changed, 78 insertions(+), 31 deletions(-) diff --git a/metrics/tests/prepare_db.py b/metrics/tests/prepare_db.py index 1b98f37..8a9c9fe 100644 --- a/metrics/tests/prepare_db.py +++ b/metrics/tests/prepare_db.py @@ -1,23 +1,51 @@ import os import json import sys +import time import logging from datetime import datetime -from peewee import IntegrityError +from peewee import IntegrityError, OperationalError from src.models import db, Address, TransactionCount logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +MAX_RETRIES = 10 +RETRY_DELAY = 3 +CONNECTION_TIMEOUT = 5 + + +def wait_for_database(): + for attempt in range(MAX_RETRIES): + try: + db.connect(reuse_if_open=True) + db.close() + logger.info('Database connection successful') + return True + except OperationalError as e: + if attempt < MAX_RETRIES - 1: + logger.warning( + f'Database connection attempt {attempt + 1} failed, ' + f'retrying in {RETRY_DELAY} seconds... Error: {e}' + ) + time.sleep(RETRY_DELAY) + else: + logger.error(f'Failed to connect to database after {MAX_RETRIES} attempts') + return False + return False + def init_database(): logger.info('Initializing test database...') + if not wait_for_database(): + sys.exit(1) + try: - db.connect() - db.create_tables([Address, TransactionCount]) - logger.info('Database tables created successfully') + with db.connection_context(): + db.create_tables([Address, TransactionCount]) + logger.info('Database tables created successfully') except Exception as e: logger.error(f'Failed to create tables: {e}') sys.exit(1) @@ -37,45 +65,64 @@ def populate_database(test_data): chain_name = test_data['chain_name'] app_name = test_data['app_name'] - with db.atomic(): - for address_data in test_data['addresses']: - address = address_data['address'] - - try: - addr_record = Address.create( - chain_name=chain_name, address=address, app_name=app_name + for attempt in range(MAX_RETRIES): + try: + with db.atomic(): + for address_data in test_data['addresses']: + address = address_data['address'] + + try: + addr_record = Address.create( + chain_name=chain_name, address=address, app_name=app_name + ) + logger.info(f'Created address record for {address}') + except IntegrityError: + logger.info(f'Address {address} already exists, skipping creation') + addr_record = Address.get(Address.address == address) + + for day_data in address_data['daily_data']: + try: + TransactionCount.create( + address=addr_record, + date=datetime.strptime(day_data['date'], '%Y-%m-%d').date(), + total_transactions=day_data['total_transactions'], + daily_transactions=day_data['daily_transactions'], + ) + logger.info( + f"Created transaction record for {address} on {day_data['date']}" + ) + except IntegrityError: + logger.info( + f"Transaction record for {address} on {day_data['date']} " + "already exists, skipping" + ) + return True + except OperationalError as e: + if attempt < MAX_RETRIES - 1: + logger.warning( + f'Database population attempt {attempt + 1} failed, ' + f'retrying in {RETRY_DELAY} seconds... Error: {e}' ) - logger.info(f'Created address record for {address}') - except IntegrityError: - logger.info(f'Address {address} already exists, skipping creation') - addr_record = Address.get(Address.address == address) - - for day_data in address_data['daily_data']: - try: - TransactionCount.create( - address=addr_record, - date=datetime.strptime(day_data['date'], '%Y-%m-%d').date(), - total_transactions=day_data['total_transactions'], - daily_transactions=day_data['daily_transactions'], - ) - logger.info(f"Created transaction record for {address} on {day_data['date']}") - except IntegrityError: - logger.info( - f"Transaction record for {address} on {day_data['date']} already exists" - ) + time.sleep(RETRY_DELAY) + else: + logger.error(f'Failed to populate database after {MAX_RETRIES} attempts') + return False + return False def main(): try: init_database() test_data = load_test_data() - populate_database(test_data) + if not populate_database(test_data): + sys.exit(1) logger.info('Database initialization completed successfully') except Exception as e: logger.error(f'Failed to initialize database: {e}') sys.exit(1) finally: - db.close() + if not db.is_closed(): + db.close() if __name__ == '__main__': From e3b49f2eaf5fc405b05adc19cabd3e7a879e0795 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 4 Nov 2024 19:22:08 +0000 Subject: [PATCH 6/6] Add retries for db connecton for prepare db --- metrics/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/tests/conftest.py b/metrics/tests/conftest.py index 17a406a..ba3df13 100644 --- a/metrics/tests/conftest.py +++ b/metrics/tests/conftest.py @@ -5,7 +5,7 @@ from faker import Faker from aiohttp import web from unittest.mock import patch -from metrics.tests.prepare_db import load_test_data +from tests.prepare_db import load_test_data fake = Faker()