diff --git a/build-and-run-docker-image.sh b/build-and-run-docker-image.sh index ae061727..b7ba40e6 100644 --- a/build-and-run-docker-image.sh +++ b/build-and-run-docker-image.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -docker build -t build/pump-eth -f ./pumps/ethereum/Dockerfile ./ -docker run -e CHAIN=ETHEREUM -e KAFKA_TRANSACTION_BATCH=1000 --net=host build/pump-eth \ No newline at end of file +docker build -t build/dump-eth -f ./dumps/ethereum/Dockerfile ./ +docker run -e CHAIN=ETHEREUM --net=host build/dump-eth \ No newline at end of file diff --git a/devops/dev/docker-compose.yml b/devops/dev/docker-compose.yml index c2337460..937e0238 100644 --- a/devops/dev/docker-compose.yml +++ b/devops/dev/docker-compose.yml @@ -104,8 +104,8 @@ services: grafana: network_mode: host - container_name: grafana3 + container_name: grafana restart: on-failure - image: grafana/grafana:5.0.0 + image: grafana/grafana:5.0.4 # ports: # - "3000:3000" \ No newline at end of file diff --git a/devops/metrics/Ethereum Address summary.json b/devops/metrics/Ethereum Address summary.json deleted file mode 100644 index fee7fa07..00000000 --- a/devops/metrics/Ethereum Address summary.json +++ /dev/null @@ -1,1250 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_LOCAL_PROMETEUS", - "label": "Local Prometeus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.0.0-beta4" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_PROMETEUS}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum_over_time(address_summary_delta_store_seconds_sum{chain=\"ETHEREUM\"}[1m])/sum_over_time(address_summary_delta_store_seconds_count{chain=\"ETHEREUM\"}[1m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Address Delta Store TIme {{topic}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Address Delta Store Speed", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_PROMETEUS}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 27, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum_over_time(address_summary_commit_kafka_seconds_sum{chain=\"ETHEREUM\"}[1m])/sum_over_time(address_summary_commit_kafka_seconds_count{chain=\"ETHEREUM\"}[1m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Address Summary Commit Kafka Time {{topic}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Address Summary Commit Kafka Speed", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_PROMETEUS}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 29, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "1000 * sum_over_time(address_summary_download_cassandra_seconds_sum{chain=\"ETHEREUM\"}[1m])/sum_over_time(address_summary_download_cassandra_seconds_count{chain=\"ETHEREUM\"}[1m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Address Summary Download Cassandra Time {{topic}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Address Summary Download Cassandra Speed", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_PROMETEUS}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 28, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum_over_time(address_summary_commit_cassandra_seconds_sum{chain=\"ETHEREUM\"}[1m])/sum_over_time(address_summary_commit_cassandra_seconds_count{chain=\"ETHEREUM\"}[1m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Address Summary Commit Cassandra Time {{topic}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Address Summary Commit Cassandra Speed", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_LOCAL_PROMETEUS}", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 18 - }, - "id": 14, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "Locks", - "prefixFontSize": "70%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "address_summary_apply_lock_counter_total{chain=\"ETHEREUM\", topic=\"ETHEREUM_TX_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Apply Tx Locks", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_LOCAL_PROMETEUS}", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 18 - }, - "id": 16, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "Locks", - "prefixFontSize": "70%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "address_summary_apply_lock_counter_total{chain=\"ETHEREUM\", topic=\"ETHEREUM_BLOCK_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Apply Block Locks", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_LOCAL_PROMETEUS}", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 18 - }, - "id": 18, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "Locks", - "prefixFontSize": "70%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "address_summary_apply_lock_counter_total{chain=\"ETHEREUM\", topic=\"ETHEREUM_UNCLE_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Apply Uncle Locks", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_LOCAL_PROMETEUS}", - "decimals": null, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 18 - }, - "id": 22, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "Remaining", - "prefixFontSize": "70%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pump_topic_last_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_TX_PUMP\"} - ignoring(instance)address_summary_topic_current_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_TX_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Tx Offsets To Apply Remaining", - "refId": "A" - } - ], - "thresholds": "", - "title": "Tx Offsets To Apply Remaining", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_LOCAL_PROMETEUS}", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 18 - }, - "id": 20, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "Remaining", - "prefixFontSize": "70%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pump_topic_last_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_BLOCK_PUMP\"} - ignoring(instance)address_summary_topic_current_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_BLOCK_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Block Offsets To Apply Remaining", - "refId": "A" - } - ], - "thresholds": "", - "title": "Block Offsets To Apply Remaining", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_LOCAL_PROMETEUS}", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 18 - }, - "id": 24, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "Remaining", - "prefixFontSize": "70%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pump_topic_last_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_UNCLE_PUMP\"} - ignoring(instance)address_summary_topic_current_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_UNCLE_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Uncle Offsets To Apply Remaining", - "refId": "A" - } - ], - "thresholds": "", - "title": "Uncle Offsets To Apply Remaining", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_PROMETEUS}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 21 - }, - "id": 31, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "500 * rate(address_summary_commit_kafka_seconds_count[10s])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{topic}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Updates per seconds", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "wps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_PROMETEUS}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 21 - }, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "address_summary_topic_current_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_UNCLE_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Current Address Summary Uncle Offset", - "refId": "A" - }, - { - "expr": "pump_topic_last_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_UNCLE_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Last Uncle Offset", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Last Applied Uncle Offset", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_PROMETEUS}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 30 - }, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "address_summary_topic_current_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_BLOCK_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Current Address Summary Block Offset", - "refId": "A" - }, - { - "expr": "pump_topic_last_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_BLOCK_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Last Ethereum Block Offset", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Last Applied Block Offset", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_PROMETEUS}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 30 - }, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "address_summary_topic_current_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_TX_PUMP\"}", - "format": "time_series", - "interval": "20s", - "intervalFactor": 1, - "legendFormat": "Ethereum Txs Address Summary Processed Offsets", - "refId": "A" - }, - { - "expr": "pump_topic_last_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_TX_PUMP\"}", - "format": "time_series", - "interval": "20s", - "intervalFactor": 1, - "legendFormat": "Ethereum Txs Last Offset", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Last Applied Tx Offset", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "refresh": "5s", - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Ethereum address summary", - "uid": "b7-IQzekk", - "version": 5 -} \ No newline at end of file diff --git a/devops/metrics/Ethereum Api Statistics.json b/devops/metrics/Ethereum Api Statistics.json deleted file mode 100644 index 657d783e..00000000 --- a/devops/metrics/Ethereum Api Statistics.json +++ /dev/null @@ -1,430 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_LOCAL_PROMETHEUS", - "label": "Local Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.0.0-beta4" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 6, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 1, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(http_requests_processing_seconds{quantile=\"0.995\"}) by (uri)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{uri}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "99.5% Percentile", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "transparent": true, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 10, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 1, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(http_requests_processing_seconds{quantile=\"0.95\"}) by (uri)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{uri}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "95% Percentile", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "transparent": true, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 12 - }, - "id": 12, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 1, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(http_requests_processing_seconds_count[10s])) by (uri)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{uri}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Request per seconds", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transparent": true, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_LOCAL_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 12 - }, - "id": 8, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 1, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(http_requests_processing_seconds_count[10s])) by (instance)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Request per seconds", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "cumulative" - }, - "transparent": true, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "refresh": "5s", - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Ethereum Api Statistics", - "uid": "0pWvBkekz", - "version": 45 -} \ No newline at end of file diff --git a/devops/metrics/Ethereum Dump.json b/devops/metrics/Ethereum Dump.json deleted file mode 100644 index 4b022d79..00000000 --- a/devops/metrics/Ethereum Dump.json +++ /dev/null @@ -1,601 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_BLOCKCHAIN", - "label": "Blockchain", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.0.0-beta4" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_BLOCKCHAIN}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "dump_topic_current_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_TX_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Current Dump Tx Offset", - "refId": "A" - }, - { - "expr": "pump_topic_last_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_TX_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Last Tx Offset", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Last Dumped Tx Offset", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_BLOCKCHAIN}", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 0 - }, - "id": 8, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "Remaining", - "prefixFontSize": "70%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pump_topic_last_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_TX_PUMP\"} - ignoring(instance)dump_topic_current_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_TX_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Tx Offsets To Dump Remaining", - "refId": "A" - } - ], - "thresholds": "", - "title": "Tx Offsets To Dump Remaining", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_BLOCKCHAIN}", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 0 - }, - "id": 10, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "Remaining", - "prefixFontSize": "70%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pump_topic_last_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_BLOCK_PUMP\"} - ignoring(instance)dump_topic_current_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_BLOCK_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Block Offsets To Dump Remaining", - "refId": "A" - } - ], - "thresholds": "", - "title": "Block Offsets To Dump Remaining", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "total" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_BLOCKCHAIN}", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 0 - }, - "id": 12, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "Remaining", - "prefixFontSize": "70%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pump_topic_last_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_UNCLE_PUMP\"} - ignoring(instance)dump_topic_current_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_UNCLE_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Uncle Offsets To Dump Remaining", - "refId": "A" - } - ], - "thresholds": "", - "title": "Uncle Offsets To Dump Remaining", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_BLOCKCHAIN}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 3 - }, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "dump_topic_current_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_UNCLE_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Current Dump Uncle Offset", - "refId": "A" - }, - { - "expr": "pump_topic_last_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_UNCLE_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Last Uncle Offset", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Last Dumped Uncle Offset", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_BLOCKCHAIN}", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "dump_topic_current_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_BLOCK_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Current Dump Block Offset", - "refId": "A" - }, - { - "expr": "pump_topic_last_offset{chain=\"ETHEREUM\",topic=\"ETHEREUM_BLOCK_PUMP\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Last Block Offset", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Last Dumped Block Offset", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Ethereum dump", - "uid": "VnPYhm6kz", - "version": 15 -} \ No newline at end of file diff --git a/devops/metrics/Ethereum Indexation Dashboard.json b/devops/metrics/Ethereum Indexation Dashboard.json deleted file mode 100644 index 65738ced..00000000 --- a/devops/metrics/Ethereum Indexation Dashboard.json +++ /dev/null @@ -1,686 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_STAGING_PROMETHEUS", - "label": "Staging Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.0.0-beta4" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "cacheTimeout": null, - "colorBackground": true, - "colorValue": false, - "colors": [ - "#629e51", - "#7eb26d", - "#d44a3a" - ], - "datasource": "${DS_STAGING_PROMETHEUS}", - "decimals": null, - "format": "locale", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 7, - "y": 0 - }, - "id": 6, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "pump_last_processed_block{chain=\"ETHEREUM\"}", - "format": "time_series", - "hide": false, - "instant": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Last Indexed Block", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": true, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "${DS_STAGING_PROMETHEUS}", - "format": "Bps", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 10, - "y": 0 - }, - "id": 12, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "rate(pump_block_size_bytes_sum{chain=\"ETHEREUM\"}[10s])", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Dowload Speed", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "cacheTimeout": null, - "colorBackground": true, - "colorValue": false, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": "${DS_STAGING_PROMETHEUS}", - "format": "s", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 13, - "y": 0 - }, - "id": 14, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "(pump_last_network_block{chain=\"ETHEREUM\"} - pump_last_processed_block{chain=\"ETHEREUM\"}) / rate(pump_last_processed_block{chain=\"ETHEREUM\"}[10m])", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Time To Full Index", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_STAGING_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 4 - }, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum_over_time(pump_block_size_bytes_sum{chain=\"ETHEREUM\"}[1m])/sum_over_time(pump_block_size_bytes_count{chain=\"ETHEREUM\"}[1m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Block Size", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Block Size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_STAGING_PROMETHEUS}", - "decimals": null, - "fill": 1, - "gridPos": { - "h": 7, - "w": 7, - "x": 8, - "y": 4 - }, - "hideTimeOverride": false, - "id": 2, - "legend": { - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pump_last_processed_block{chain=\"ETHEREUM\"}", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Indexed Block", - "refId": "A" - }, - { - "expr": "pump_last_network_block{chain=\"ETHEREUM\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Network Block", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Last Pumped Block", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transparent": true, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "locale", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_STAGING_PROMETHEUS}", - "fill": 1, - "gridPos": { - "h": 7, - "w": 8, - "x": 15, - "y": 4 - }, - "id": 4, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(pump_last_processed_block{chain=\"ETHEREUM\"}[10m])", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Blocks Per Second", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "locale", - "label": "block/sec", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_STAGING_PROMETHEUS}", - "decimals": null, - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 11 - }, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "1000 * sum_over_time(pump_bundle_download_seconds_sum{chain=\"ETHEREUM\"}[1m])/sum_over_time(pump_bundle_download_seconds_count{chain=\"ETHEREUM\"}[1m])", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Single Block Download Time", - "refId": "A" - }, - { - "expr": "1000 * sum_over_time(pump_bundle_kafka_store_seconds_sum{chain=\"ETHEREUM\"}[1m])/sum_over_time(pump_bundle_kafka_store_seconds_count{chain=\"ETHEREUM\"}[1m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Kafka Transaction Batch Store Time", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Block download speed", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "refresh": "5s", - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Ethereum Indexation Dashboard", - "uid": "J0p4xxqzz", - "version": 18 -} \ No newline at end of file diff --git a/devops/prod/cassandra-service.yaml b/devops/prod/cassandra-service.yaml deleted file mode 100644 index f2a83dce..00000000 --- a/devops/prod/cassandra-service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - app: elassandra - name: elassandra -spec: - clusterIP: None - ports: - - port: 9042 - selector: - app: elassandra \ No newline at end of file diff --git a/devops/prod/cassandra-statefulset.yaml b/devops/prod/cassandra-statefulset.yaml deleted file mode 100644 index c8ebe8ea..00000000 --- a/devops/prod/cassandra-statefulset.yaml +++ /dev/null @@ -1,85 +0,0 @@ -apiVersion: apps/v1beta2 -kind: StatefulSet -metadata: - name: elassandra - labels: - app: elassandra -spec: - serviceName: elassandra - replicas: 3 - selector: - matchLabels: - app: elassandra - template: - metadata: - labels: - app: elassandra - spec: - containers: - - name: elassandra - image: strapdata/elassandra:5.5.0.8 - ports: - - containerPort: 7000 - name: intra-node - - containerPort: 7001 - name: tls-intra-node - - containerPort: 7199 - name: jmx - - containerPort: 9042 - name: cql - - containerPort: 9200 - name: elastic-http - - containerPort: 9300 - name: elastic-intra - resources: - requests: - memory: 30Gi - securityContext: - capabilities: - add: - - IPC_LOCK - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "PID=$(pidof java) && kill $PID && while ps -p $PID > /dev/null; do sleep 1; done"] - env: - - name: CASSANDRA_NUM_TOKENS - value: "16" - - name: CASSANDRA_SEEDS - value: "elassandra-0.elassandra.default.svc.cluster.local" - - name: CASSANDRA_CLUSTER_NAME - value: "CYBER_SEARCH" - - name: CASSANDRA_DC - value: "PRODUCTION" - - name: CASSANDRA_AUTO_BOOTSTRAP - value: "true" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - # These volume mounts are persistent. They are like inline claims, - # but not exactly because the names need to match exactly one of - # the stateful pod volumes. - volumeMounts: - - name: elassandra-data - mountPath: /var/lib/cassandra - # These are converted to volume claims by the controller - # and mounted at the paths mentioned above. - # do not use these in production until ssd GCEPersistentDisk or other ssd pd - volumeClaimTemplates: - - metadata: - name: elassandra-data - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 300Gi - storageClassName: fast ---- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: fast -provisioner: kubernetes.io/gce-pd -parameters: - type: pd-ssd \ No newline at end of file diff --git a/devops/prod/chains-compose.yml b/devops/prod/chains-compose.yml deleted file mode 100644 index 1bb46de0..00000000 --- a/devops/prod/chains-compose.yml +++ /dev/null @@ -1,42 +0,0 @@ -version: '3.0' -services: - - parity-eth: - container_name: parity-eth - restart: always - image: parity/parity:stable - ports: - - "8545:8545" - command: "--db-path /cyberdata --jsonrpc-hosts all --jsonrpc-interface all --jsonrpc-threads 8" - volumes: - - /cyberdata/eth:/cyberdata - - parity-etc: - container_name: parity-etc - restart: always - image: parity/parity:stable - ports: - - "18545:8545" - command: "--chain classic --db-path /cyberdata --jsonrpc-hosts all --jsonrpc-interface all --jsonrpc-threads 8" - volumes: - - /cyberdata/eth_c:/cyberdata - - bitcoind: - container_name: bitcoind - restart: always - image: ruimarinho/bitcoin-core:0.15.1 - ports: - - "8332:8332" - command: "-server -rest -txindex -rpcpassword=fund.cyber -rpcuser=fund.cyber -rpcallowip=0.0.0.0/0" - volumes: - - /cyberdata/bitcoind:/home/bitcoin/.bitcoin - - abc: - container_name: abc - restart: always - image: hlebalbau/bitcoin-abc-docker:0.16.1 - ports: - - "7332:7332" - command: "bitcoind -server -rest -txindex -rpcpassword=fund.cyber -rpcuser=fund.cyber -rpcallowip=::/0 -rpcport=7332" - volumes: - - /cyberdata/abc:/home/bitcoin/.bitcoin \ No newline at end of file diff --git a/devops/prod/chains-pumps-compose.yaml b/devops/prod/chains-pumps-compose.yaml deleted file mode 100644 index 9367c100..00000000 --- a/devops/prod/chains-pumps-compose.yaml +++ /dev/null @@ -1,337 +0,0 @@ -apiVersion: v1 -items: -- apiVersion: v1 - kind: Service - metadata: - annotations: - kompose.cmd: kompose convert -f chains-pumps.yml -o chains-pumps.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: cs-address-service-btc - name: cs-address-service-btc - spec: - clusterIP: None - ports: - - name: headless - port: 55555 - targetPort: 0 - selector: - io.kompose.service: cs-address-service-btc - status: - loadBalancer: {} -- apiVersion: v1 - kind: Service - metadata: - annotations: - kompose.cmd: kompose convert -f chains-pumps.yml -o chains-pumps.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: cs-address-service-eth - name: cs-address-service-eth - spec: - clusterIP: None - ports: - - name: headless - port: 55555 - targetPort: 0 - selector: - io.kompose.service: cs-address-service-eth - status: - loadBalancer: {} -- apiVersion: v1 - kind: Service - metadata: - annotations: - kompose.cmd: kompose convert -f chains-pumps.yml -o chains-pumps.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: cs-chain-pumps-btc - name: cs-chain-pumps-btc - spec: - clusterIP: None - ports: - - name: headless - port: 55555 - targetPort: 0 - selector: - io.kompose.service: cs-chain-pumps-btc - status: - loadBalancer: {} -- apiVersion: v1 - kind: Service - metadata: - annotations: - kompose.cmd: kompose convert -f chains-pumps.yml -o chains-pumps.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: cs-chain-pumps-bth - name: cs-chain-pumps-bth - spec: - clusterIP: None - ports: - - name: headless - port: 55555 - targetPort: 0 - selector: - io.kompose.service: cs-chain-pumps-bth - status: - loadBalancer: {} -- apiVersion: v1 - kind: Service - metadata: - annotations: - kompose.cmd: kompose convert -f chains-pumps.yml -o chains-pumps.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: cs-chain-pumps-etc - name: cs-chain-pumps-etc - spec: - clusterIP: None - ports: - - name: headless - port: 55555 - targetPort: 0 - selector: - io.kompose.service: cs-chain-pumps-etc - status: - loadBalancer: {} -- apiVersion: v1 - kind: Service - metadata: - annotations: - kompose.cmd: kompose convert -f chains-pumps.yml -o chains-pumps.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: cs-chain-pumps-eth - name: cs-chain-pumps-eth - spec: - clusterIP: None - ports: - - name: headless - port: 55555 - targetPort: 0 - selector: - io.kompose.service: cs-chain-pumps-eth - status: - loadBalancer: {} -- apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - annotations: - kompose.cmd: kompose convert -f chains-pumps.yml -o chains-pumps.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: cs-address-service-btc - name: cs-address-service-btc - spec: - replicas: 1 - strategy: {} - template: - metadata: - creationTimestamp: null - labels: - io.kompose.service: cs-address-service-btc - spec: - containers: - - env: - - name: CASSANDRA_HOSTS - value: elassandra - - name: CS_CHAINS_TO_PUMP - value: BITCOIN,BITCOIN_CASH - - name: CS_LOG_LEVEL - value: INFO - - name: KAFKA_BROKERS - value: kafka-broker:9092 - image: cybernode/address-service:release-v1.0.0.RC5 - name: address-service - resources: {} - restartPolicy: Always - status: {} -- apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - annotations: - kompose.cmd: kompose convert -f chains-pumps.yml -o chains-pumps.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: cs-address-service-eth - name: cs-address-service-eth - spec: - replicas: 1 - strategy: {} - template: - metadata: - creationTimestamp: null - labels: - io.kompose.service: cs-address-service-eth - spec: - containers: - - env: - - name: CASSANDRA_HOSTS - value: elassandra - - name: CS_CHAINS_TO_PUMP - value: ETHEREUM,ETHEREUM_CLASSIC - - name: CS_LOG_LEVEL - value: INFO - - name: KAFKA_BROKERS - value: kafka-broker:9092 - image: cybernode/address-service:release-v1.0.0.RC5 - name: address-service - resources: {} - restartPolicy: Always - status: {} -- apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - annotations: - kompose.cmd: kompose convert -f chains-pumps.yml -o chains-pumps.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: cs-chain-pumps-btc - name: cs-chain-pumps-btc - spec: - replicas: 1 - strategy: {} - template: - metadata: - creationTimestamp: null - labels: - io.kompose.service: cs-chain-pumps-btc - spec: - containers: - - env: - - name: BITCOIND_URL - value: http://fund.cyber:fund.cyber@bitcoind:8332 - - name: CASSANDRA_HOSTS - value: elassandra - - name: CS_CHAINS_TO_PUMP - value: BITCOIN - - name: CS_LOG_LEVEL - value: INFO - - name: KAFKA_BROKERS - value: kafka-broker:9092 - image: cybernode/cs-chain-pumps:release-v1.0.0.RC5 - name: pump-btc - resources: {} - restartPolicy: Always - status: {} -- apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - annotations: - kompose.cmd: kompose convert -f chains-pumps.yml -o chains-pumps.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: cs-chain-pumps-bth - name: cs-chain-pumps-bth - spec: - replicas: 1 - strategy: {} - template: - metadata: - creationTimestamp: null - labels: - io.kompose.service: cs-chain-pumps-bth - spec: - containers: - - env: - - name: ABC_URL - value: http://fund.cyber:fund.cyber@abc:7332 - - name: CASSANDRA_HOSTS - value: elassandra - - name: CS_CHAINS_TO_PUMP - value: BITCOIN_CASH - - name: CS_LOG_LEVEL - value: INFO - - name: KAFKA_BROKERS - value: kafka-broker:9092 - image: cybernode/cs-chain-pumps:release-v1.0.0.RC5 - name: pump-bth - resources: {} - restartPolicy: Always - status: {} -- apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - annotations: - kompose.cmd: kompose convert -f chains-pumps.yml -o chains-pumps.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: cs-chain-pumps-etc - name: cs-chain-pumps-etc - spec: - replicas: 1 - strategy: {} - template: - metadata: - creationTimestamp: null - labels: - io.kompose.service: cs-chain-pumps-etc - spec: - containers: - - env: - - name: CASSANDRA_HOSTS - value: elassandra - - name: CS_CHAINS_TO_PUMP - value: ETHEREUM_CLASSIC - - name: CS_LOG_LEVEL - value: INFO - - name: KAFKA_BROKERS - value: kafka-broker:9092 - - name: PARITY_ETC_URL - value: http://parity-etc:18545 - image: cybernode/cs-chain-pumps:release-v1.0.0.RC5 - name: pump-etc - resources: {} - restartPolicy: Always - status: {} -- apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - annotations: - kompose.cmd: kompose convert -f chains-pumps.yml -o chains-pumps.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: cs-chain-pumps-eth - name: cs-chain-pumps-eth - spec: - replicas: 1 - strategy: {} - template: - metadata: - creationTimestamp: null - labels: - io.kompose.service: cs-chain-pumps-eth - spec: - containers: - - env: - - name: CASSANDRA_HOSTS - value: elassandra - - name: CS_CHAINS_TO_PUMP - value: ETHEREUM - - name: CS_LOG_LEVEL - value: INFO - - name: KAFKA_BROKERS - value: kafka-broker:9092 - - name: PARITY_ETH_URL - value: http://parity-eth:8545 - image: cybernode/cs-chain-pumps:release-v1.0.0.RC5 - name: pump-eth - resources: {} - restartPolicy: Always - status: {} -kind: List -metadata: {} - diff --git a/devops/prod/chains-pumps-services.yaml b/devops/prod/chains-pumps-services.yaml deleted file mode 100644 index 4550cbc7..00000000 --- a/devops/prod/chains-pumps-services.yaml +++ /dev/null @@ -1,313 +0,0 @@ -apiVersion: v1 -items: -- apiVersion: v1 - kind: Service - metadata: - annotations: - kompose.cmd: kompose convert -f chains-compose.yml -o chains-services.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: abc - name: abc - spec: - ports: - - name: "7332" - port: 7332 - targetPort: 7332 - selector: - io.kompose.service: abc - status: - loadBalancer: {} -- apiVersion: v1 - kind: Service - metadata: - annotations: - kompose.cmd: kompose convert -f chains-compose.yml -o chains-services.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: bitcoind - name: bitcoind - spec: - ports: - - name: "8332" - port: 8332 - targetPort: 8332 - selector: - io.kompose.service: bitcoind - status: - loadBalancer: {} -- apiVersion: v1 - kind: Service - metadata: - annotations: - kompose.cmd: kompose convert -f chains-compose.yml -o chains-services.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: parity-etc - name: parity-etc - spec: - ports: - - name: "18545" - port: 18545 - targetPort: 8545 - selector: - io.kompose.service: parity-etc - status: - loadBalancer: {} -- apiVersion: v1 - kind: Service - metadata: - annotations: - kompose.cmd: kompose convert -f chains-compose.yml -o chains-services.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: parity-eth - name: parity-eth - spec: - ports: - - name: "8545" - port: 8545 - targetPort: 8545 - selector: - io.kompose.service: parity-eth - status: - loadBalancer: {} -- apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - annotations: - kompose.cmd: kompose convert -f chains-compose.yml -o chains-services.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: abc - name: abc - spec: - replicas: 1 - strategy: - type: Recreate - template: - metadata: - creationTimestamp: null - labels: - io.kompose.service: abc - spec: - containers: - - args: - - bitcoind - - -server - - -rest - - -txindex - - -rpcpassword=cyber - - -rpcuser=cyber - - -rpcallowip=::/0 - - -rpcport=7332 - image: hlebalbau/bitcoin-abc-docker:0.16.1 - name: abc - ports: - - containerPort: 7332 - resources: {} - volumeMounts: - - mountPath: /home/bitcoin/.bitcoin - name: abc-claim0 - restartPolicy: Always - volumes: - - name: abc-claim0 - persistentVolumeClaim: - claimName: abc-claim0 - status: {} -- apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - creationTimestamp: null - labels: - io.kompose.service: abc-claim0 - name: abc-claim0 - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 500Gi - status: {} -- apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - annotations: - kompose.cmd: kompose convert -f chains-compose.yml -o chains-services.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: bitcoind - name: bitcoind - spec: - replicas: 1 - strategy: - type: Recreate - template: - metadata: - creationTimestamp: null - labels: - io.kompose.service: bitcoind - spec: - containers: - - args: - - -server - - -rest - - -txindex - - -rpcpassword=cyber - - -rpcuser=cyber - - -rpcallowip=0.0.0.0/0 - image: ruimarinho/bitcoin-core:0.15.1 - name: bitcoind - ports: - - containerPort: 8332 - resources: {} - volumeMounts: - - mountPath: /home/bitcoin/.bitcoin - name: bitcoind-claim0 - restartPolicy: Always - volumes: - - name: bitcoind-claim0 - persistentVolumeClaim: - claimName: bitcoind-claim0 - status: {} -- apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - creationTimestamp: null - labels: - io.kompose.service: bitcoind-claim0 - name: bitcoind-claim0 - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 500Gi - status: {} -- apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - annotations: - kompose.cmd: kompose convert -f chains-compose.yml -o chains-services.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: parity-etc - name: parity-etc - spec: - replicas: 1 - strategy: - type: Recreate - template: - metadata: - creationTimestamp: null - labels: - io.kompose.service: parity-etc - spec: - containers: - - args: - - --chain - - classic - - --db-path - - /cyberdata - - --jsonrpc-hosts - - all - - --jsonrpc-interface - - all - - --jsonrpc-threads - - "8" - image: parity/parity:stable - name: parity-etc - ports: - - containerPort: 8545 - resources: {} - volumeMounts: - - mountPath: /cyberdata - name: parity-etc-claim0 - restartPolicy: Always - volumes: - - name: parity-etc-claim0 - persistentVolumeClaim: - claimName: parity-etc-claim0 - status: {} -- apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - creationTimestamp: null - labels: - io.kompose.service: parity-etc-claim0 - name: parity-etc-claim0 - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 500Gi - status: {} -- apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - annotations: - kompose.cmd: kompose convert -f chains-compose.yml -o chains-services.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: parity-eth - name: parity-eth - spec: - replicas: 1 - strategy: - type: Recreate - template: - metadata: - creationTimestamp: null - labels: - io.kompose.service: parity-eth - spec: - containers: - - args: - - --db-path - - /cyberdata - - --jsonrpc-hosts - - all - - --jsonrpc-interface - - all - - --jsonrpc-threads - - "8" - image: parity/parity:stable - name: parity-eth - ports: - - containerPort: 8545 - resources: {} - volumeMounts: - - mountPath: /cyberdata - name: parity-eth-claim0 - restartPolicy: Always - volumes: - - name: parity-eth-claim0 - persistentVolumeClaim: - claimName: parity-eth-claim0 - status: {} -- apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - creationTimestamp: null - labels: - io.kompose.service: parity-eth-claim0 - name: parity-eth-claim0 - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 500Gi - status: {} -kind: List -metadata: {} - diff --git a/devops/prod/chains-pumps.yml b/devops/prod/chains-pumps.yml deleted file mode 100644 index bdc984db..00000000 --- a/devops/prod/chains-pumps.yml +++ /dev/null @@ -1,66 +0,0 @@ -version: '3.0' -services: - - cs-chain-pumps-btc: - container_name: pump-btc - image: cybernode/cs-chain-pumps:release-v1.0.0.RC5 - restart: unless-stopped - environment: - CASSANDRA_HOSTS: elassandra - KAFKA_BROKERS: kafka-broker:9092 - CS_CHAINS_TO_PUMP: BITCOIN - BITCOIND_URL: http://fund.cyber:fund.cyber@bitcoind:8332 - CS_LOG_LEVEL: INFO - - cs-chain-pumps-bth: - container_name: pump-bth - image: cybernode/cs-chain-pumps:release-v1.0.0.RC5 - restart: unless-stopped - environment: - CASSANDRA_HOSTS: elassandra - KAFKA_BROKERS: kafka-broker:9092 - CS_CHAINS_TO_PUMP: BITCOIN_CASH - ABC_URL: http://fund.cyber:fund.cyber@abc:7332 - CS_LOG_LEVEL: INFO - - cs-chain-pumps-eth: - container_name: pump-eth - image: cybernode/cs-chain-pumps:release-v1.0.0.RC5 - restart: unless-stopped - environment: - CASSANDRA_HOSTS: elassandra - KAFKA_BROKERS: kafka-broker:9092 - CS_CHAINS_TO_PUMP: ETHEREUM - PARITY_ETH_URL: http://parity-eth:8545 - CS_LOG_LEVEL: INFO - - cs-chain-pumps-etc: - container_name: pump-etc - image: cybernode/cs-chain-pumps:release-v1.0.0.RC5 - restart: on-failure - environment: - CASSANDRA_HOSTS: elassandra - KAFKA_BROKERS: kafka-broker:9092 - CS_CHAINS_TO_PUMP: ETHEREUM_CLASSIC - PARITY_ETC_URL: http://parity-etc:18545 - CS_LOG_LEVEL: INFO - - cs-address-service-btc: - container_name: address-service - image: cybernode/address-service:release-v1.0.0.RC5 - restart: unless-stopped - environment: - CS_CHAINS_TO_PUMP: BITCOIN,BITCOIN_CASH - CASSANDRA_HOSTS: elassandra - KAFKA_BROKERS: kafka-broker:9092 - CS_LOG_LEVEL: INFO - - cs-address-service-eth: - container_name: address-service - image: cybernode/address-service:release-v1.0.0.RC5 - restart: unless-stopped - environment: - CS_CHAINS_TO_PUMP: ETHEREUM,ETHEREUM_CLASSIC - CASSANDRA_HOSTS: elassandra - KAFKA_BROKERS: kafka-broker:9092 - CS_LOG_LEVEL: INFO \ No newline at end of file diff --git a/devops/prod/elassandra-bootstrap.cql b/devops/prod/elassandra-bootstrap.cql deleted file mode 100644 index d79ff350..00000000 --- a/devops/prod/elassandra-bootstrap.cql +++ /dev/null @@ -1,34 +0,0 @@ -CREATE KEYSPACE IF NOT EXISTS search - WITH REPLICATION = {'class' : 'NetworkTopologyStrategy','PRODUCTION' : 3}; - - - -CREATE KEYSPACE IF NOT EXISTS bitcoin - WITH REPLICATION = {'class' : 'NetworkTopologyStrategy','PRODUCTION' : 3}; - -CREATE KEYSPACE IF NOT EXISTS bitcoin_cash - WITH REPLICATION = {'class' : 'NetworkTopologyStrategy','PRODUCTION' : 3}; - -CREATE KEYSPACE IF NOT EXISTS ethereum - WITH REPLICATION = {'class' : 'NetworkTopologyStrategy','PRODUCTION' : 3}; - -CREATE KEYSPACE IF NOT EXISTS ethereum_classic - WITH REPLICATION = {'class' : 'NetworkTopologyStrategy','PRODUCTION' : 3}; - - - -CREATE KEYSPACE IF NOT EXISTS cyber_system - WITH REPLICATION = {'class' : 'NetworkTopologyStrategy','PRODUCTION' : 3}; - -CREATE KEYSPACE IF NOT EXISTS pump - WITH REPLICATION = {'class' : 'NetworkTopologyStrategy','PRODUCTION' : 3}; - - - -CREATE TABLE IF NOT EXISTS cyber_system.schema_version ( - application_id text, - version int, - apply_time timestamp, - migration_hash int, - PRIMARY KEY (application_id, version) -) WITH CLUSTERING ORDER BY (version DESC); \ No newline at end of file diff --git a/devops/prod/kafka-compose.yml b/devops/prod/kafka-compose.yml deleted file mode 100644 index 94c5f192..00000000 --- a/devops/prod/kafka-compose.yml +++ /dev/null @@ -1,40 +0,0 @@ -version: '3.0' -services: - - zoo: - container_name: zoo - restart: always - image: confluentinc/cp-zookeeper:4.0.0 - environment: - ZOOKEEPER_TICK_TIME: 2000 - ZOOKEEPER_CLIENT_PORT: 2181 - KAFKA_LOG4J_ROOT_LOGLEVEL: ERROR - volumes: - - /cyberdata/staging/zoo/data:/var/lib/zookeeper/data - - /cyberdata/staging/zoo/log:/var/lib/zookeeper/log - ports: - - "2181:2181" - - kafka-broker: - container_name: kafka-broker - restart: always - image: confluentinc/cp-kafka:4.0.0 - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zoo:2181 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker:9092 - KAFKA_JMX_PORT: 1099 - KAFKA_LOG4J_ROOT_LOGLEVEL: ERROR - KAFKA_LOG4J_ROOT_LOGLEVEL: WARN - KAFKA_DELETE_TOPIC_ENABLE: "true" - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_DEFAULT_REPLICATION_FACTOR: 1 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - volumes: - - /cyberdata/staging/kafka:/var/lib/kafka - ports: - - "9092:9092" - - "1099:1099" \ No newline at end of file diff --git a/devops/prod/kafka-services.yaml b/devops/prod/kafka-services.yaml deleted file mode 100644 index 7a3ace00..00000000 --- a/devops/prod/kafka-services.yaml +++ /dev/null @@ -1,194 +0,0 @@ -apiVersion: v1 -items: -- apiVersion: v1 - kind: Service - metadata: - annotations: - kompose.cmd: kompose convert -f kafka-compose.yml -o kafka-services.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: kafka-broker - name: kafka-broker - spec: - ports: - - name: "9092" - port: 9092 - targetPort: 9092 - - name: "1099" - port: 1099 - targetPort: 1099 - selector: - io.kompose.service: kafka-broker - status: - loadBalancer: {} -- apiVersion: v1 - kind: Service - metadata: - annotations: - kompose.cmd: kompose convert -f kafka-compose.yml -o kafka-services.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: zoo - name: zoo - spec: - ports: - - name: "2181" - port: 2181 - targetPort: 2181 - selector: - io.kompose.service: zoo - status: - loadBalancer: {} -- apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - annotations: - kompose.cmd: kompose convert -f kafka-compose.yml -o kafka-services.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: kafka-broker - name: kafka-broker - spec: - replicas: 1 - strategy: - type: Recreate - template: - metadata: - creationTimestamp: null - labels: - io.kompose.service: kafka-broker - spec: - containers: - - env: - - name: KAFKA_ADVERTISED_LISTENERS - value: PLAINTEXT://kafka-broker:9092 - - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE - value: "true" - - name: KAFKA_BROKER_ID - value: "1" - - name: KAFKA_DEFAULT_REPLICATION_FACTOR - value: "1" - - name: KAFKA_DELETE_TOPIC_ENABLE - value: "true" - - name: KAFKA_JMX_PORT - value: "1099" - - name: KAFKA_LOG4J_ROOT_LOGLEVEL - value: WARN - - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR - value: "1" - - name: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR - value: "1" - - name: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR - value: "1" - - name: KAFKA_ZOOKEEPER_CONNECT - value: zoo:2181 - image: confluentinc/cp-kafka:4.0.0 - name: kafka-broker - ports: - - containerPort: 9092 - - containerPort: 1099 - resources: {} - volumeMounts: - - mountPath: /var/lib/kafka - name: kafka-broker-claim0 - restartPolicy: Always - volumes: - - name: kafka-broker-claim0 - persistentVolumeClaim: - claimName: kafka-broker-claim0 - status: {} -- apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - creationTimestamp: null - labels: - io.kompose.service: kafka-broker-claim0 - name: kafka-broker-claim0 - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 500Gi - status: {} -- apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - annotations: - kompose.cmd: kompose convert -f kafka-compose.yml -o kafka-services.yaml - kompose.version: 1.5.0 (999278f) - creationTimestamp: null - labels: - io.kompose.service: zoo - name: zoo - spec: - replicas: 1 - strategy: - type: Recreate - template: - metadata: - creationTimestamp: null - labels: - io.kompose.service: zoo - spec: - containers: - - env: - - name: KAFKA_LOG4J_ROOT_LOGLEVEL - value: ERROR - - name: ZOOKEEPER_CLIENT_PORT - value: "2181" - - name: ZOOKEEPER_TICK_TIME - value: "2000" - image: confluentinc/cp-zookeeper:4.0.0 - name: zoo - ports: - - containerPort: 2181 - resources: {} - volumeMounts: - - mountPath: /var/lib/zookeeper/data - name: zoo-claim0 - - mountPath: /var/lib/zookeeper/log - name: zoo-claim1 - restartPolicy: Always - volumes: - - name: zoo-claim0 - persistentVolumeClaim: - claimName: zoo-claim0 - - name: zoo-claim1 - persistentVolumeClaim: - claimName: zoo-claim1 - status: {} -- apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - creationTimestamp: null - labels: - io.kompose.service: zoo-claim0 - name: zoo-claim0 - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - status: {} -- apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - creationTimestamp: null - labels: - io.kompose.service: zoo-claim1 - name: zoo-claim1 - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - status: {} -kind: List -metadata: {} - diff --git a/devops/sink-cassandra-search-api b/devops/sink-cassandra-search-api deleted file mode 100644 index eeddde83..00000000 --- a/devops/sink-cassandra-search-api +++ /dev/null @@ -1,17 +0,0 @@ -connector.class=com.datamountaineer.streamreactor.connect.cassandra.sink.CassandraSinkConnector -connect.cassandra.key.space=search -topics=search_request_processing_stats -tasks.max=1 -connect.cassandra.kcql=INSERT INTO request_processing_stats SELECT * FROM search_request_processing_stats -connect.cassandra.fetch.size=5000 -connect.cassandra.max.retries=20 -connect.progress.enabled=false -connect.cassandra.error.policy=retry -connect.cassandra.contact.points=cs-elassandra -connect.cassandra.port=9042 -connect.cassandra.threadpool.size=2 -name=cassandra-sink-search-api -connect.cassandra.retry.interval=30000 -connect.progress.enabled=false -connect.cassandra.consistency.level=LOCAL_ONE -connect.progress.enabled=true \ No newline at end of file diff --git a/dumps/ethereum/Dockerfile b/dumps/ethereum/Dockerfile index c748f7fe..6f17887e 100644 --- a/dumps/ethereum/Dockerfile +++ b/dumps/ethereum/Dockerfile @@ -1,10 +1,10 @@ # Build Stage -FROM openjdk:9-jdk-slim AS build +FROM openjdk:10-jdk-slim AS build COPY ./ /build WORKDIR /build RUN ./gradlew clean :dumps:ethereum:assemble # Container with application -FROM openjdk:9-jre-slim +FROM openjdk:10-jre-slim COPY --from=build /build/dumps/ethereum/build/libs /cyberapp/bin ENTRYPOINT ["java", "-jar", "/cyberapp/bin/ethereum.jar"] \ No newline at end of file