diff --git a/README.md b/README.md index e13f678..8496242 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![CircleCI](https://circleci.com/gh/w3f/polkadot-watcher-validator.svg?style=svg)](https://circleci.com/gh/w3f/polkadot-watcher-validator) -# polkadot-watcher-validator +# polkadot-watcher-validator ## How to Run @@ -25,8 +25,7 @@ It can then monitor the status of the node, leveraging on mechanisms such as the ### Monitoring Features -- An Active validator has been detected offline -- An Active validator is risking to be caught offline, please act ASAP +- A validator has been reported for Slash - A validator is not seleceted by Phragmen alorithm to be part of the active set - A validator has changed his payout address destination - A validator has an unexpected payout address destination @@ -40,11 +39,10 @@ If an expected destination address is specified in the config file, it is implic ### Resources -- validators staking and heartbeats: https://wiki.polkadot.network/docs/en/learn-staking#unresponsiveness - session: https://wiki.polkadot.network/docs/en/glossary#session - era: https://wiki.polkadot.network/docs/en/glossary#era - polkadotJs library (raccomended, Nodejs + typescript): https://polkadot.js.org/docs/ -- event, validators offline: https://polkadot.js.org/docs/substrate/events#someofflinevecidentificationtuple +- event, validator SlashReported: https://polkadot.js.org/docs/substrate/events/#slashreportedaccountid32-perbill-u32 ## Configuration diff --git a/charts/polkadot-watcher/templates/alertrules.yaml b/charts/polkadot-watcher/templates/alertrules.yaml index f7203e7..3cee696 100644 --- a/charts/polkadot-watcher/templates/alertrules.yaml +++ b/charts/polkadot-watcher/templates/alertrules.yaml @@ -44,33 +44,15 @@ spec: severity: critical origin: {{ .Values.prometheusRules.origin }} {{ end }} - - alert: ValidatorOffline + - alert: ValidatorSlashed annotations: - message: 'Target {{`{{ $labels.name }}`}} was reported offline, an advanced double check can be carried here. Check the account for eventual slashes. This message is going to RESOLVE by itself soon.' - runbook_url: "https://github.com/w3f/infrastructure/wiki/Validator-Offline" - expr: max without(instance,pod) (increase(polkadot_validator_offline_reports{environment="{{ .Values.config.environment }}"}[5m])) > 0 + message: 'Target {{`{{ $labels.name }}`}} was reported for Slash, an advanced double check can be carried here. This message is going to RESOLVE by itself soon.' + runbook_url: "https://github.com/w3f/infrastructure/wiki/Validator-Slashed" + expr: max without(instance,pod) (increase(polkadot_validator_slashed_reports{environment="{{ .Values.config.environment }}"}[5m])) > 0 for: 30s labels: severity: critical - origin: {{ .Values.prometheusRules.origin }} - {{ if ne .Values.prometheusRules.offlineRisk false }} - - alert: ValidatorOfflineRiskLong - annotations: - message: 'Target {{`{{ $labels.name }}`}} has either not authored any block or sent any heartbeat yet in this session. It is risking to be caught offline' - expr: max without(instance,pod) (last_over_time(polkadot_validator_offline_risk_state{environment="{{ .Values.config.environment }}"}[10m])) > 0 - for: 10m - labels: - severity: critical - origin: {{ .Values.prometheusRules.origin }} - - alert: ValidatorOfflineRiskShort - annotations: - message: 'Target {{`{{ $labels.name }}`}} has either not authored any block or sent any heartbeat yet in this session. It is risking to be caught offline' - expr: max without(instance,pod) (last_over_time(polkadot_validator_offline_risk_state{environment="{{ .Values.config.environment }}"}[10m])) > 0 - for: 8m - labels: - severity: warning origin: {{ .Values.prometheusRules.origin }} - {{ end }} - alert: ValidatorRewardDestinationChanged annotations: message: 'Target {{`{{ $labels.name }}`}} may have changed his reward destination recently, please double check. This message is going to RESOLVE by itself soon.' diff --git a/charts/polkadot-watcher/values.yaml b/charts/polkadot-watcher/values.yaml index 35aace9..f91adfa 100644 --- a/charts/polkadot-watcher/values.yaml +++ b/charts/polkadot-watcher/values.yaml @@ -28,7 +28,6 @@ prometheusRules: labels: app: w3f origin: cluster - offlineRisk: true producerStall: true resources: diff --git a/src/prometheus.ts b/src/prometheus.ts index e65cfea..4f393c1 100644 --- a/src/prometheus.ts +++ b/src/prometheus.ts @@ -4,11 +4,8 @@ import { PromClient } from './types'; export class Prometheus implements PromClient { - static readonly nameOfflineRiskMetric = 'polkadot_validator_offline_risk_state'; - private blocksProducedReports: promClient.Counter<"network" | "name" | "address" | "environment">; - private offlineReports: promClient.Counter<"network" | "name" | "address" | "environment">; - private stateOfflineRisk: promClient.Gauge<"network" | "name" | "address" | "environment">; + private slashedReports: promClient.Counter<"network" | "name" | "address" | "environment">; private stateOutOfActiveSet: promClient.Gauge<"network" | "name" | "address" | "environment">; private payeeChangedReports: promClient.Counter<"network" | "name" | "address" | "environment">; @@ -32,33 +29,14 @@ export class Prometheus implements PromClient { increaseBlocksProducedReports(name: string, address: string): void { this.blocksProducedReports.inc({network:this.network, name, address, environment: this.environment }) - this.resetStatusOfflineRisk(name, address) //solve potential risk status - } - - increaseOfflineReports(name: string, address: string): void { - this.offlineReports.inc({network:this.network, name, address, environment: this.environment }); - } - - setStatusOfflineRisk(name: string, address: string): void { - this.stateOfflineRisk.set({network:this.network, name, address, environment: this.environment }, 1); } - resetStatusOfflineRisk(name: string, address: string): void { - this.stateOfflineRisk.set({network:this.network, name, address, environment: this.environment }, 0); - } - - isStatusOfflineRiskFiring(name: string, address: string): boolean { - try { - return promClient.register.getSingleMetric(Prometheus.nameOfflineRiskMetric)['hashMap'][`name:${name},network:${this.network},address:${address},environment:${this.environment}`]['value'] == 1 - } catch (error) { - this.resetStatusOfflineRisk(name, address) - return promClient.register.getSingleMetric(Prometheus.nameOfflineRiskMetric)['hashMap'][`name:${name},network:${this.network},address:${address},environment:${this.environment}`]['value'] == 1 - } + increaseSlashedReports(name: string, address: string): void { + this.slashedReports.inc({network:this.network, name, address, environment: this.environment }); } setStatusOutOfActiveSet(name: string, address: string): void{ this.stateOutOfActiveSet.set({network:this.network, name, address, environment: this.environment }, 1); - this.resetStatusOfflineRisk(name,address) //solve potential risk status } resetStatusOutOfActiveSet(name: string, address: string): void{ @@ -95,14 +73,9 @@ export class Prometheus implements PromClient { help: 'Number of blocks produced by a validator', labelNames: ['network', 'name', 'address', 'environment'] }); - this.offlineReports = new promClient.Gauge({ - name: 'polkadot_validator_offline_reports', - help: 'Times a validator has been reported offline', - labelNames: ['network', 'name', 'address', 'environment'] - }); - this.stateOfflineRisk = new promClient.Gauge({ - name: Prometheus.nameOfflineRiskMetric, - help: 'Whether a validator has not produced a block and neither has sent an expected heartbeat yet. It is risking to be caught offline', + this.slashedReports = new promClient.Gauge({ + name: 'polkadot_validator_slashed_reports', + help: 'Times a validator has been reported for slashing', labelNames: ['network', 'name', 'address', 'environment'] }); this.stateOutOfActiveSet = new promClient.Gauge({ diff --git a/src/subscriber.ts b/src/subscriber.ts index b01ab69..194a722 100644 --- a/src/subscriber.ts +++ b/src/subscriber.ts @@ -9,9 +9,8 @@ import { InputConfig, Subscribable, PromClient, - ValidatorImOnlineParameters } from './types'; -import { getActiveEraIndex, isHeadAfterHeartbeatBlockThreshold, hasValidatorProvedOnline, isNewSessionEvent, isOfflineEvent } from './utils'; +import { getActiveEraIndex } from './utils'; export class Subscriber { private validators: Array; @@ -40,7 +39,7 @@ export class Subscriber { public triggerConnectivityTest(): void { const testAccountName = "CONNECTIVITY_TEST_NO_ACTION_REQUIRED" - this.promClient.increaseOfflineReports(testAccountName,testAccountName); + this.promClient.increaseSlashedReports(testAccountName,testAccountName); } private async _initInstanceVariables(): Promise{ @@ -60,7 +59,7 @@ export class Subscriber { private async _handleNewHeadSubscriptions(): Promise { this.api.rpc.chain.subscribeNewHeads(async (header) => { this._producerHandler(header); - this._validatorStatusHandler(header); + this._validatorStatusHandler(); this._payeeChangeHandler(header); this._commissionChangeHandler(header); this._checkUnexpected(); @@ -98,11 +97,11 @@ export class Subscriber { events.forEach(async (record) => { const { event } = record; - if(isOfflineEvent(event)){ - this._offlineEventHandler(event) + if(this.api.events.staking.SlashReported.is(event)){ + this._slashedEventHandler(event) } - if(isNewSessionEvent(event)){ + if(this.api.events.session.NewSession.is(event)){ await this._newSessionEventHandler() } }); @@ -123,18 +122,16 @@ export class Subscriber { } } - private async _validatorStatusHandler(header: Header): Promise { - const parameters = await this._getImOnlineParametersAtomic(header) + private async _validatorStatusHandler(): Promise { this.validators.forEach(async account => { - const validatorActiveSetIndex = parameters.validatorActiveSet.indexOf(account.address) + const validatorActiveSetIndex = this.validatorActiveSet.indexOf(account.address) if ( validatorActiveSetIndex < 0 ) { - this.logger.debug(`Target ${account.name} is not present in the validation active set of era ${parameters.eraIndex}`); + this.logger.debug(`Target ${account.name} is not present in the validation active set of era ${this.currentEraIndex}`); this.promClient.setStatusOutOfActiveSet(account.name,account.address); } else { this.promClient.resetStatusOutOfActiveSet(account.name,account.address); - await this._checkOfflineRiskStatus(parameters,account,validatorActiveSetIndex) } }) @@ -208,28 +205,18 @@ export class Subscriber { } } - private async _checkOfflineRiskStatus(parameters: ValidatorImOnlineParameters,validator: Subscribable,validatorActiveSetIndex: number): Promise{ - if ( await hasValidatorProvedOnline(validator,validatorActiveSetIndex,parameters.sessionIndex,this.api) ) { - this.promClient.resetStatusOfflineRisk(validator.name,validator.address); - } else if(parameters.isHeartbeatExpected) { - this.logger.info(`Target ${validator.name} has either not authored any block or sent any heartbeat yet in session:${parameters.sessionIndex}/era:${parameters.eraIndex}`); - this.promClient.setStatusOfflineRisk(validator.name,validator.address); - } - // else let it be as it is. - // with this solution, if a validator has been caught offline, it will eventually remain in a risk status also for the first half of the subsequent session. - } - - private _offlineEventHandler(event: Event): void { + private _slashedEventHandler(event: Event): void { + const items = event.data[0]; (items as Tuple).forEach((item) => { - const offlineValidator = item[0]; - this.logger.debug(`${offlineValidator} found offline`); - const account = this.validators.find((subject) => subject.address == offlineValidator); + const validator = item[0]; + this.logger.debug(`${validator} has been reported for Slash`); + const account = this.validators.find((subject) => subject.address == validator); if (account) { - this.logger.info(`Really bad... Target ${account.name} found offline`); - this.promClient.increaseOfflineReports(account.name, account.address); + this.logger.info(`Really bad... Target ${account.name} has been reported for Slash`); + this.promClient.increaseSlashedReports(account.name, account.address); } }); } @@ -249,25 +236,9 @@ export class Subscriber { await this._initValidatorsControllers(); } - private async _getImOnlineParametersAtomic(header: Header): Promise { - - const sessionIndex = this.sessionIndex - const eraIndex = this.currentEraIndex - const validatorActiveSet = this.validatorActiveSet - this.logger.debug(`Current EraIndex: ${eraIndex}\tCurrent SessionIndex: ${sessionIndex}`); - const isHeartbeatExpected = await isHeadAfterHeartbeatBlockThreshold(this.api,header) - - return { - isHeartbeatExpected, - sessionIndex, - eraIndex, - validatorActiveSet - } - } - private _initCounterMetrics(): void { this._initBlocksProducedMetrics(); - this._initOfflineReportsMetrics() + this._initSlashedReportsMetrics() this._initPayeeChangedMetrics(); this._initCommissionChangedMetrics(); } @@ -280,11 +251,11 @@ export class Subscriber { }); } - private _initOfflineReportsMetrics(): void { + private _initSlashedReportsMetrics(): void { this.validators.forEach((account) => { // always increase counters even the first time, so that we initialize the time series // https://github.com/prometheus/prometheus/issues/1673 - this.promClient.increaseOfflineReports(account.name, account.address); + this.promClient.increaseSlashedReports(account.name, account.address); }); } diff --git a/src/types.ts b/src/types.ts index cff21a5..5af71dc 100644 --- a/src/types.ts +++ b/src/types.ts @@ -36,10 +36,7 @@ export interface InputConfigFromGit { export interface PromClient { increaseBlocksProducedReports(name: string, address: string): void; - increaseOfflineReports(name: string, address: string): void; - setStatusOfflineRisk(name: string, address: string): void; - resetStatusOfflineRisk(name: string, address: string): void; - isStatusOfflineRiskFiring(name: string, address: string): boolean; + increaseSlashedReports(name: string, address: string): void; setStatusOutOfActiveSet(name: string, address: string): void; resetStatusOutOfActiveSet(name: string, address: string): void; increasePayeeChangedReports(name: string, address: string): void; @@ -73,7 +70,6 @@ export interface MatrixbotMsg { } export interface ValidatorImOnlineParameters { - isHeartbeatExpected: boolean; sessionIndex: SessionIndex; eraIndex: number; validatorActiveSet: Vec; diff --git a/src/utils.ts b/src/utils.ts index 4ce90e8..c7b7518 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,56 +1,16 @@ /*eslint @typescript-eslint/no-use-before-define: ["error", { "variables": false }]*/ import { ApiPromise } from '@polkadot/api'; -import { Event } from '@polkadot/types/interfaces/system'; -import { SessionIndex, Header } from '@polkadot/types/interfaces'; -import { Subscribable } from './types'; -import { ZeroBN } from './constants'; import { LoggerSingleton } from './logger'; const logger = LoggerSingleton.getInstance() -export const isNewSessionEvent = (event: Event): boolean => { - return event.section == 'session' && event.method == 'NewSession'; -} - -export const isOfflineEvent = (event: Event): boolean => { - return event.section == 'imOnline' && event.method == 'SomeOffline'; -} - -export const hasValidatorProvedOnline = async (account: Subscribable, validatorIndex: number, sessionIndex: SessionIndex, api: ApiPromise): Promise => { - return await _hasValidatorAuthoredBlocks(account,sessionIndex,api) || await _hasValidatorSentHeartbeats(validatorIndex,sessionIndex,api) -} - export const getActiveEraIndex = async (api: ApiPromise): Promise => { return (await api.query.staking.activeEra()).toJSON()['index']; } -export const isHeadAfterHeartbeatBlockThreshold = async (api: ApiPromise, header: Header): Promise => { - return false - //I'm online pallet got removed: https://github.com/paritytech/polkadot-sdk/issues/4359 - const currentBlock = header.number.toBn() - const blockThreshold = await api.query.imOnline.heartbeatAfter() //threshold after which an heartbeat is expected - logger.debug(`Current Block: ${currentBlock}\tHeartbeatBlock Threshold: ${blockThreshold}`); - return currentBlock.cmp(blockThreshold) > 0 -} - export async function asyncForEach(array: Array, callback: (arg0: T, arg1: number, arg2: Array) => void): Promise { for (let index = 0; index < array.length; index++) { await callback(array[index], index, array); } -} - -const _hasValidatorAuthoredBlocks = async (validator: Subscribable, sessionIndex: SessionIndex, api: ApiPromise): Promise => { - return true - //I'm online pallet got removed: https://github.com/paritytech/polkadot-sdk/issues/4359 - const numBlocksAuthored = await api.query.imOnline.authoredBlocks(sessionIndex,validator.address) - return numBlocksAuthored.cmp(ZeroBN) > 0 -} - -const _hasValidatorSentHeartbeats = async (validatorIndex: number, sessionIndex: SessionIndex, api: ApiPromise): Promise => { - return true - //I'm online pallet got removed: https://github.com/paritytech/polkadot-sdk/issues/4359 - if (validatorIndex < 0) return false; - const hb = await api.query.imOnline.receivedHeartbeats(sessionIndex,validatorIndex) - return hb.toHuman() ? true : false } \ No newline at end of file diff --git a/test/mocks.ts b/test/mocks.ts index a560c86..bac3719 100644 --- a/test/mocks.ts +++ b/test/mocks.ts @@ -6,8 +6,7 @@ import { PromClient } from "../src/types"; export class PrometheusMock implements PromClient { private _blocksProducedReports = 0; - private _offlineReports = 0; - private _stateOfflineRisk = 0; + private _slashedReports = 0; private _stateOutOfActiveSet = 0; private _payeeChangedReports = 0; private _commissionChangedReports = 0; @@ -16,24 +15,14 @@ export class PrometheusMock implements PromClient { increaseBlocksProducedReports(name: string, address: string): void { this._blocksProducedReports++; - this.resetStatusOfflineRisk(name) //solve potential risk status } - increaseOfflineReports(name: string, address: string): void { - this._offlineReports++; + increaseSlashedReports(name: string, address: string): void { + this._slashedReports++; } - setStatusOfflineRisk(name: string): void { - this._stateOfflineRisk = 1 - } - resetStatusOfflineRisk(name: string): void { - this._stateOfflineRisk = 0 - } - isStatusOfflineRiskFiring(name: string): boolean {return false} - setStatusOutOfActiveSet(name: string): void { this._stateOutOfActiveSet = 1 - this.resetStatusOfflineRisk(name) //solve potential risk status } resetStatusOutOfActiveSet(name: string): void { this._stateOutOfActiveSet = 0 @@ -62,11 +51,8 @@ export class PrometheusMock implements PromClient { get blocksProducedReports(): number { return this._blocksProducedReports; } - get offlineReports(): number { - return this._offlineReports; - } - get statusOfflineRisk(): number { - return this._stateOfflineRisk; + get slashedReports(): number { + return this._slashedReports; } get statusOutOfActiveSet(): number { return this._stateOutOfActiveSet; diff --git a/test/prometheus/alertrules.yaml b/test/prometheus/alertrules.yaml index 7cefc72..cd3148b 100644 --- a/test/prometheus/alertrules.yaml +++ b/test/prometheus/alertrules.yaml @@ -6,52 +6,18 @@ evaluation_interval: 1m tests: - interval: 1m input_series: - - series: 'polkadot_validator_offline_risk_state{network="kusama",name="node0",address="Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5",environment="production"}' - values: '0 0 1+0x10 0 0' # 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 - series: 'polkadot_validator_out_of_active_set_state{network="kusama",name="node0",address="Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5",environment="production"}' values: '0 0 1 1 1 0 0+0x190 1+0x20' - series: 'polkadot_validator_payee_changed_reports{network="kusama",name="node0",address="Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5",environment="production"}' values: '0 0 1 1 1 1 1' - series: 'polkadot_validator_commission_changed_reports{network="kusama",name="node0",address="Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5",environment="production"}' values: '0 0 1 1 1 1 1' - - series: 'polkadot_validator_offline_reports{network="kusama",name="node0",address="Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5",environment="production"}' + - series: 'polkadot_validator_slashed_reports{network="kusama",name="node0",address="Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5",environment="production"}' values: '0 1 0 0 0 0 0' - series: 'polkadot_validator_blocks_produced{network="kusama",name="node0",address="Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5",environment="production"}' values: '1 1+0x190' # more than 3 hour - alert_rule_test: - # Test ValidatorOfflineSession alert - - eval_time: 9m # Values: 0 0 1 1 - alertname: ValidatorOfflineRiskShort - exp_alerts: - - eval_time: 10m # Values: 0 0 1 1 1 1 1 1 1 1 - alertname: ValidatorOfflineRiskShort - exp_alerts: - - exp_labels: - severity: warning - origin: cluster - name: node0 - address: Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5 - network: kusama - environment: production - exp_annotations: - message: 'Target node0 has either not authored any block or sent any heartbeat yet in this session. It is risking to be caught offline' - - eval_time: 12m # Values: 0 0 1 1 1 1 1 1 1 1 1 1 - alertname: ValidatorOfflineRiskLong - exp_alerts: - - exp_labels: - severity: critical - origin: cluster - name: node0 - network: kusama - address: Gt6HqWBhdu4Sy1u8ASTbS1qf2Ac5gwdegwr8tWN8saMxPt5 - environment: production - exp_annotations: - message: 'Target node0 has either not authored any block or sent any heartbeat yet in this session. It is risking to be caught offline' - - eval_time: 13m # Values: 0 0 1 1 1 1 1 1 1 1 1 1 0 - alertname: ValidatorOfflineRiskLong - exp_alerts: - + alert_rule_test: # Test ValidatorOutOfActiveSet alert - eval_time: 3m # Values: 0 0 1 1 alertname: ValidatorOutOfActiveSet @@ -109,12 +75,12 @@ tests: alertname: ValidatorCommissionRateChanged exp_alerts: - # Test ValidatorOffline alert + # Test ValidatorSlashed alert - eval_time: 1m # Values: 0 0 - alertname: ValidatorOffline + alertname: ValidatorSlashed exp_alerts: - eval_time: 2m # Values: 0 1 - alertname: ValidatorOffline + alertname: ValidatorSlashed exp_alerts: - exp_labels: severity: critical @@ -124,10 +90,10 @@ tests: network: kusama environment: production exp_annotations: - message: 'Target node0 was reported offline, an advanced double check can be carried here. Check the account for eventual slashes. This message is going to RESOLVE by itself soon.' - runbook_url: "https://github.com/w3f/infrastructure/wiki/Validator-Offline" + message: 'Target node0 was reported for Slash, an advanced double check can be carried here. This message is going to RESOLVE by itself soon.' + runbook_url: "https://github.com/w3f/infrastructure/wiki/Validator-Slashed" - eval_time: 6m # Values: 0 1 0 0 0 0 - alertname: ValidatorOffline + alertname: ValidatorSlashed exp_alerts: # Test ProducerStallX alert diff --git a/test/subscriber.ts b/test/subscriber.ts index 4b56e42..63e763b 100644 --- a/test/subscriber.ts +++ b/test/subscriber.ts @@ -92,8 +92,7 @@ describe('Subscriber cfg1, with a started network', async () => { await delay(6000); prometheus.blocksProducedReports.should.be.gt(1); //counters are init to 1 - prometheus.offlineReports.should.be.eq(1) //counters are init to 1 - prometheus.statusOfflineRisk.should.be.eq(0) + prometheus.slashedReports.should.be.eq(1) //counters are init to 1 prometheus.statusOutOfActiveSet.should.be.eq(0) prometheus.payeeChangedReports.should.be.eq(1) //counters are init to 1 prometheus.commissionChangedReports.should.be.eq(1) //counters are init to 1 @@ -168,8 +167,7 @@ describe('Subscriber cfg2, with a started network', () => { await delay(6000); prometheus.blocksProducedReports.should.be.eq(1); //counters are init to 1 - prometheus.offlineReports.should.be.eq(1) //counters are init to 1 - prometheus.statusOfflineRisk.should.be.eq(0) + prometheus.slashedReports.should.be.eq(1) //counters are init to 1 prometheus.statusOutOfActiveSet.should.be.eq(1) prometheus.payeeChangedReports.should.be.eq(1) //counters are init to 1 prometheus.commissionChangedReports.should.be.eq(1) //counters are init to 1