From 62ee39d34976c038755083c540e1d4bc0b165bd8 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 5 Jun 2024 12:47:30 +0100 Subject: [PATCH 1/3] Add metrics collector to proxy --- .gitignore | 1 + docker-compose.yml | 17 ++++++ metrics/Dockerfile | 16 ++++++ metrics/requirements.txt | 2 + metrics/src/__init__.py | 0 metrics/src/collector.py | 114 +++++++++++++++++++++++++++++++++++++++ metrics/src/config.py | 50 +++++++++++++++++ metrics/src/explorer.py | 46 ++++++++++++++++ metrics/src/gas.py | 48 +++++++++++++++++ metrics/src/logs.py | 53 ++++++++++++++++++ metrics/src/main.py | 48 +++++++++++++++++ 11 files changed, 395 insertions(+) create mode 100644 metrics/Dockerfile create mode 100644 metrics/requirements.txt create mode 100644 metrics/src/__init__.py create mode 100644 metrics/src/collector.py create mode 100644 metrics/src/config.py create mode 100644 metrics/src/explorer.py create mode 100644 metrics/src/gas.py create mode 100644 metrics/src/logs.py create mode 100644 metrics/src/main.py diff --git a/.gitignore b/.gitignore index c706308..0cb48fb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__ abi.json chains.json +metrics.json conf/upstreams/*.conf conf/chains/*.conf \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 717ff7c..2ef6c94 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,3 +33,20 @@ services: options: max-file: "200" max-size: "500m" + metrics: + environment: + ETH_ENDPOINT: ${ETH_ENDPOINT} + NETWORK_NAME: ${NETWORK_NAME} + image: metrics:latest + container_name: metrics + build: + context: ./metrics + dockerfile: Dockerfile + volumes: + - ./data:/data + logging: + driver: "json-file" + options: + max-file: "5" + max-size: "50m" + restart: unless-stopped \ No newline at end of file diff --git a/metrics/Dockerfile b/metrics/Dockerfile new file mode 100644 index 0000000..f71693b --- /dev/null +++ b/metrics/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12.3-bookworm + +RUN apt-get update + +RUN mkdir /usr/src/metrics /data +WORKDIR /usr/src/metrics + +COPY requirements.txt ./ +RUN pip3 install --no-cache-dir -r requirements.txt + +COPY . . + +ENV PYTHONPATH="/usr/src/metrics" +ENV COLUMNS=80 + +CMD python /usr/src/metrics/src/main.py diff --git a/metrics/requirements.txt b/metrics/requirements.txt new file mode 100644 index 0000000..1174916 --- /dev/null +++ b/metrics/requirements.txt @@ -0,0 +1,2 @@ +web3==6.19.0 +requests==2.32.3 \ No newline at end of file diff --git a/metrics/src/__init__.py b/metrics/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metrics/src/collector.py b/metrics/src/collector.py new file mode 100644 index 0000000..0ebdd92 --- /dev/null +++ b/metrics/src/collector.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +# +# This file is part of portal-metrics +# +# Copyright (C) 2024 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import json +import logging +import asyncio +import aiohttp +from datetime import datetime +from typing import Any, Dict, List, Tuple + +import requests + +from explorer import get_address_counters_url, get_chain_stats +from gas import calc_avg_gas_price +from config import METRICS_FILEPATH + +logger = logging.getLogger(__name__) + + +def get_metadata_url(network_name: str): + # return f'https://raw.githubusercontent.com/skalenetwork/skale-network/master/metadata/{network_name}/chains.json' # noqa + return f'https://raw.githubusercontent.com/skalenetwork/skale-network/update-mainnet-chains-metadata/metadata/{network_name}/chains.json' # noqa + + +def download_metadata(network_name: str): + url = get_metadata_url(network_name) + response = requests.get(url) + response.raise_for_status() + return response.json() + + +async def get_address_counters(session, network, chain_name, address): + url = get_address_counters_url(network, chain_name, address) + async with session.get(url) as response: + return await response.json() + + +async def get_all_address_counters(network, chain_name, addresses): + results = {} + async with aiohttp.ClientSession() as session: + tasks = [] + for address in addresses: + tasks.append(get_address_counters(session, network, chain_name, address)) + + responses = await asyncio.gather(*tasks) + + for address, response in zip(addresses, responses): + results[address] = response + + return results + + +async def _fetch_counters_for_app(network_name, chain_name, app_name, app_info): + logger.info(f'fetching counters for app {app_name}') + if 'contracts' in app_info: + counters = await get_all_address_counters(network_name, chain_name, app_info['contracts']) + return app_name, counters + return app_name, None + + +async def fetch_counters_for_apps(chain_info, network_name, chain_name): + tasks = [] + for app_name, app_info in chain_info['apps'].items(): + task = _fetch_counters_for_app(network_name, chain_name, app_name, app_info) + tasks.append(task) + return await asyncio.gather(*tasks) + + +def transform_to_dict(apps_counters: List[Tuple[str, Any]] | None) -> Dict[str, Any]: + if not apps_counters: + return {} + results = {} + for app_name, counters in apps_counters: + results[app_name] = counters + return results + + +def collect_metrics(network_name: str): + metadata = download_metadata(network_name) + metrics = {} + for chain_name, chain_info in metadata.items(): + apps_counters = None + chain_stats = get_chain_stats(network_name, chain_name) + if 'apps' in chain_info: + apps_counters = asyncio.run( + fetch_counters_for_apps(chain_info, network_name, chain_name)) + metrics[chain_name] = { + 'chain_stats': chain_stats, + 'apps_counters': transform_to_dict(apps_counters), + } + data = { + 'metrics': metrics, + 'gas': int(calc_avg_gas_price()), + 'last_updated': int(datetime.now().timestamp()) + } + logger.info(f'Saving metrics to {METRICS_FILEPATH}') + with open(METRICS_FILEPATH, 'w') as f: + json.dump(data, f, indent=4, sort_keys=True) diff --git a/metrics/src/config.py b/metrics/src/config.py new file mode 100644 index 0000000..0a31b43 --- /dev/null +++ b/metrics/src/config.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# +# This file is part of portal-metrics +# +# Copyright (C) 2024 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os + +ERROR_TIMEOUT = os.getenv('MONITOR_INTERVAL', 60) +MONITOR_INTERVAL = os.getenv('MONITOR_INTERVAL', 10800) +NETWORK_NAME = os.getenv('NETWORK_NAME', 'mainnet') +ENDPOINT = os.environ['ETH_ENDPOINT'] + +PROXY_ENDPOINTS = { + 'mainnet': 'mainnet.skalenodes.com', + 'legacy': 'legacy-proxy.skaleserver.com', + 'regression': 'regression-proxy.skalenodes.com', + 'testnet': 'testnet.skalenodes.com' +} + +BASE_EXPLORER_URLS = { + 'mainnet': 'explorer.mainnet.skalenodes.com', + 'legacy': 'legacy-explorer.skaleserver.com', + 'regression': 'regression-explorer.skalenodes.com', + 'testnet': 'explorer.testnet.skalenodes.com' +} + +STATS_API = { + 'mainnet': 'https://stats.explorer.mainnet.skalenodes.com/v2/stats', +} + +HTTPS_PREFIX = "https://" + +BLOCK_SAMPLING = 100 +GAS_ESTIMATION_ITERATIONS = 300 + +METRICS_FILEPATH = os.path.join('/', 'data', 'metrics.json') diff --git a/metrics/src/explorer.py b/metrics/src/explorer.py new file mode 100644 index 0000000..0fdd3a2 --- /dev/null +++ b/metrics/src/explorer.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# +# This file is part of portal-metrics +# +# Copyright (C) 2024 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import requests +from typing import Any + +from config import BASE_EXPLORER_URLS, HTTPS_PREFIX + +logger = logging.getLogger(__name__) + + +def _get_explorer_url(network, chain_name): + explorer_base_url = BASE_EXPLORER_URLS[network] + return HTTPS_PREFIX + chain_name + '.' + explorer_base_url + + +def get_chain_stats(network: str, chain_name: str) -> Any: + try: + explorer_url = _get_explorer_url(network, chain_name) + response = requests.get(f'{explorer_url}/api/v2/stats') + return response.json() + except Exception as e: + logger.error(f'Failed to get chain stats: {e}') + return None + + +def get_address_counters_url(network: str, chain_name: str, address: str) -> str: + explorer_url = _get_explorer_url(network, chain_name) + return f'{explorer_url}/api/v2/addresses/{address}/counters' diff --git a/metrics/src/gas.py b/metrics/src/gas.py new file mode 100644 index 0000000..72e918c --- /dev/null +++ b/metrics/src/gas.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# +# This file is part of portal-metrics +# +# Copyright (C) 2024 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +from web3 import Web3 +from config import ENDPOINT, GAS_ESTIMATION_ITERATIONS, BLOCK_SAMPLING + +logger = logging.getLogger(__name__) + + +def init_w3(): + logger.info(f'Connecting to {ENDPOINT}...') + return Web3(Web3.HTTPProvider(ENDPOINT)) + + +def calc_avg_gas_price(): + block_num = GAS_ESTIMATION_ITERATIONS * BLOCK_SAMPLING + logger.info(f'Calculating average gas price for the last {block_num} blocks') + w3 = init_w3() + block_number = w3.eth.block_number + total_gas_used = 0 + + logger.info(f'Getting historic block gas prices...') + for index in range(GAS_ESTIMATION_ITERATIONS): + block_number = block_number - BLOCK_SAMPLING * index + block = w3.eth.get_block(block_number) + total_gas_used += block['baseFeePerGas'] + + avg_gas_price = total_gas_used / GAS_ESTIMATION_ITERATIONS + avg_gas_price_gwei = Web3.from_wei(avg_gas_price, 'gwei') + logger.info(f'avg_gas_price_gwei: {avg_gas_price_gwei}') + return avg_gas_price_gwei diff --git a/metrics/src/logs.py b/metrics/src/logs.py new file mode 100644 index 0000000..d4c430a --- /dev/null +++ b/metrics/src/logs.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# +# This file is part of portal-metrics +# +# Copyright (C) 2024 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +import sys +import logging +import logging.handlers as py_handlers +from logging import Formatter, StreamHandler + + +LOG_FORMAT = '[%(asctime)s %(levelname)s] %(name)s:%(lineno)d - %(threadName)s - %(message)s' +LOG_FILEPATH = os.path.join(os.getcwd(), 'portal-metrics.log') + +LOG_FILE_SIZE_MB = 300 +LOG_FILE_SIZE_BYTES = LOG_FILE_SIZE_MB * 1000000 +LOG_BACKUP_COUNT = 3 + +MONITOR_INTERVAL = 600 + + +def get_file_handler(log_filepath, log_level): + formatter = Formatter(LOG_FORMAT) + f_handler = py_handlers.RotatingFileHandler( + log_filepath, maxBytes=LOG_FILE_SIZE_BYTES, + backupCount=LOG_BACKUP_COUNT) + f_handler.setFormatter(formatter) + f_handler.setLevel(log_level) + return f_handler + + +def init_default_logger(): + formatter = Formatter(LOG_FORMAT) + f_handler = get_file_handler(LOG_FILEPATH, logging.INFO) + stream_handler = StreamHandler(sys.stderr) + stream_handler.setFormatter(formatter) + stream_handler.setLevel(logging.INFO) + logging.basicConfig(level=logging.DEBUG, handlers=[f_handler, stream_handler]) diff --git a/metrics/src/main.py b/metrics/src/main.py new file mode 100644 index 0000000..b6b5326 --- /dev/null +++ b/metrics/src/main.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# +# This file is part of portal-metrics +# +# Copyright (C) 2024 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import sys +import logging +from time import sleep + +from logs import init_default_logger +from collector import collect_metrics +from config import MONITOR_INTERVAL, ERROR_TIMEOUT, NETWORK_NAME, PROXY_ENDPOINTS + +logger = logging.getLogger(__name__) + + +def run_metrics_loop(): + if NETWORK_NAME not in PROXY_ENDPOINTS: + logger.error(f'Unsupported network: {NETWORK_NAME}') + sys.exit(1) + while True: + logger.info('Metrics collector iteration started...') + try: + collect_metrics(NETWORK_NAME) + logger.info(f'Metrics collector iteration done, sleeping for {MONITOR_INTERVAL}s...') + sleep(MONITOR_INTERVAL) + except Exception as e: + logger.error(f'Something went wrong: {e}') + sleep(ERROR_TIMEOUT) + + +if __name__ == '__main__': + init_default_logger() + run_metrics_loop() From c899e43e18a2d9b4e0d3000f2bbccf454598d52e Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 5 Jun 2024 13:10:29 +0100 Subject: [PATCH 2/3] Remove reduntant variable --- metrics/src/logs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/metrics/src/logs.py b/metrics/src/logs.py index d4c430a..5d023ee 100644 --- a/metrics/src/logs.py +++ b/metrics/src/logs.py @@ -31,8 +31,6 @@ LOG_FILE_SIZE_BYTES = LOG_FILE_SIZE_MB * 1000000 LOG_BACKUP_COUNT = 3 -MONITOR_INTERVAL = 600 - def get_file_handler(log_filepath, log_level): formatter = Formatter(LOG_FORMAT) From e5e58d5eb0b406a2881039a8f0d39efa247e0e43 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 5 Jun 2024 19:25:56 +0100 Subject: [PATCH 3/3] Handle empty string NETWORK_NAME --- metrics/src/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/src/config.py b/metrics/src/config.py index 0a31b43..69cf76b 100644 --- a/metrics/src/config.py +++ b/metrics/src/config.py @@ -21,7 +21,7 @@ ERROR_TIMEOUT = os.getenv('MONITOR_INTERVAL', 60) MONITOR_INTERVAL = os.getenv('MONITOR_INTERVAL', 10800) -NETWORK_NAME = os.getenv('NETWORK_NAME', 'mainnet') +NETWORK_NAME = os.getenv('NETWORK_NAME', 'mainnet') or 'mainnet' ENDPOINT = os.environ['ETH_ENDPOINT'] PROXY_ENDPOINTS = {