diff --git a/apps/api/src/billing/services/wallet-balance-reload-check/wallet-balance-reload-check-instrumentation.service.ts b/apps/api/src/billing/services/wallet-balance-reload-check/wallet-balance-reload-check-instrumentation.service.ts new file mode 100644 index 0000000000..874d2476e3 --- /dev/null +++ b/apps/api/src/billing/services/wallet-balance-reload-check/wallet-balance-reload-check-instrumentation.service.ts @@ -0,0 +1,154 @@ +import { LoggerService } from "@akashnetwork/logging"; +import type { Counter, Histogram, Meter } from "@opentelemetry/api"; +import { singleton } from "tsyringe"; + +import { MetricsService } from "@src/core"; + +@singleton() +export class WalletBalanceReloadCheckInstrumentationService { + private readonly meter: Meter; + private readonly jobExecutions: Counter; + private readonly jobDuration: Histogram; + private readonly reloadsTriggered: Counter; + private readonly reloadsSkipped: Counter; + private readonly reloadFailures: Counter; + private readonly validationErrors: Counter; + private readonly schedulingErrors: Counter; + private readonly reloadAmounts: Histogram; + private readonly balanceCoverageRatio: Histogram; + private readonly projectedCost: Histogram; + + private readonly logger = LoggerService.forContext("WalletBalanceReloadCheckHandler"); + + constructor(private readonly metricsService: MetricsService) { + this.meter = this.metricsService.getMeter("wallet-balance-reload-check", "1.0.0"); + + this.jobExecutions = this.metricsService.createCounter(this.meter, "wallet_balance_reload_check_job_executions_total", { + description: "Total number of wallet balance reload check job executions" + }); + + this.jobDuration = this.metricsService.createHistogram(this.meter, "wallet_balance_reload_check_job_duration_ms", { + description: "Duration of wallet balance reload check job execution in milliseconds", + unit: "ms" + }); + + this.reloadsTriggered = this.metricsService.createCounter(this.meter, "wallet_balance_reload_check_reloads_triggered_total", { + description: "Total number of wallet balance reloads triggered" + }); + + this.reloadsSkipped = this.metricsService.createCounter(this.meter, "wallet_balance_reload_check_reloads_skipped_total", { + description: "Total number of wallet balance reloads skipped" + }); + + this.reloadFailures = this.metricsService.createCounter(this.meter, "wallet_balance_reload_check_reload_failures_total", { + description: "Total number of wallet balance reload failures" + }); + + this.validationErrors = this.metricsService.createCounter(this.meter, "wallet_balance_reload_check_validation_errors_total", { + description: "Total number of validation errors by error type" + }); + + this.schedulingErrors = this.metricsService.createCounter(this.meter, "wallet_balance_reload_check_scheduling_errors_total", { + description: "Total number of errors when scheduling next check" + }); + + this.reloadAmounts = this.metricsService.createHistogram(this.meter, "wallet_balance_reload_check_reload_amount_usd", { + description: "Amount of wallet balance reloads in USD", + unit: "USD" + }); + + this.balanceCoverageRatio = this.metricsService.createHistogram(this.meter, "wallet_balance_reload_check_balance_coverage_ratio", { + description: "Ratio of current balance to projected cost (balance / costUntilTargetDate)" + }); + + this.projectedCost = this.metricsService.createHistogram(this.meter, "wallet_balance_reload_check_projected_cost_usd", { + description: "Projected deployment cost until target date in USD", + unit: "USD" + }); + } + + recordJobExecution(durationMs: number, success: boolean, userId: string): void { + this.jobExecutions.add(1, { + status: success ? "success" : "failure" + }); + this.jobDuration.record(durationMs, { + status: success ? "success" : "failure" + }); + this.logger.info({ + event: "WALLET_BALANCE_RELOAD_CHECK_JOB_COMPLETED", + durationMs, + status: success ? "success" : "failure", + userId + }); + } + + recordReloadTriggered(amount: number, balance: number, threshold: number, costUntilTargetDate: number, logContext: Record): void { + this.reloadsTriggered.add(1); + this.reloadAmounts.record(amount); + // Only record balance coverage ratio if cost is greater than 0 to avoid division by zero + if (costUntilTargetDate > 0) { + this.balanceCoverageRatio.record(balance / costUntilTargetDate); + } + this.projectedCost.record(costUntilTargetDate); + this.logger.info({ + ...logContext, + amount, + event: "WALLET_BALANCE_RELOADED" + }); + } + + recordReloadSkipped( + balance: number, + threshold: number, + costUntilTargetDate: number, + reason: "zero_cost" | "sufficient_balance", + logContext: Record + ): void { + this.reloadsSkipped.add(1, { + reason + }); + // Only record balance coverage ratio if cost is greater than 0 to avoid division by zero + // For zero_cost reason, costUntilTargetDate will be 0, so we skip recording the ratio + if (costUntilTargetDate > 0) { + this.balanceCoverageRatio.record(balance / costUntilTargetDate); + } + this.projectedCost.record(costUntilTargetDate); + this.logger.info({ + ...logContext, + event: "WALLET_BALANCE_RELOAD_SKIPPED" + }); + } + + recordReloadFailed(amount: number, error: unknown, logContext: Record): void { + this.reloadFailures.add(1, { + error_type: error instanceof Error ? error.constructor.name : "Unknown" + }); + this.reloadAmounts.record(amount); + this.logger.error({ + ...logContext, + event: "WALLET_BALANCE_RELOAD_FAILED", + error: error + }); + } + + recordValidationError(errorType: string, error: { event: string; message: string }, userId: string): void { + this.validationErrors.add(1, { + error_type: errorType + }); + this.logger.error({ + ...error, + userId: userId + }); + } + + recordSchedulingError(walletAddress: string, error: unknown): void { + this.schedulingErrors.add(1, { + error_type: error instanceof Error ? error.constructor.name : "Unknown" + }); + this.logger.error({ + event: "ERROR_SCHEDULING_NEXT_CHECK", + walletAddress, + error: error + }); + } +} diff --git a/apps/api/src/billing/services/wallet-balance-reload-check/wallet-balance-reload-check.handler.spec.ts b/apps/api/src/billing/services/wallet-balance-reload-check/wallet-balance-reload-check.handler.spec.ts index b25548b62d..61d29f6f87 100644 --- a/apps/api/src/billing/services/wallet-balance-reload-check/wallet-balance-reload-check.handler.spec.ts +++ b/apps/api/src/billing/services/wallet-balance-reload-check/wallet-balance-reload-check.handler.spec.ts @@ -7,10 +7,11 @@ import type { WalletSettingRepository } from "@src/billing/repositories"; import type { BalancesService } from "@src/billing/services/balances/balances.service"; import type { StripeService } from "@src/billing/services/stripe/stripe.service"; import type { WalletReloadJobService } from "@src/billing/services/wallet-reload-job/wallet-reload-job.service"; -import type { JobMeta, LoggerService } from "@src/core"; +import type { JobMeta } from "@src/core"; import type { DrainingDeploymentService } from "@src/deployment/services/draining-deployment/draining-deployment.service"; import type { JobPayload } from "../../../core"; import { WalletBalanceReloadCheckHandler } from "./wallet-balance-reload-check.handler"; +import type { WalletBalanceReloadCheckInstrumentationService } from "./wallet-balance-reload-check-instrumentation.service"; import { generateBalance } from "@test/seeders/balance.seeder"; import { generateMergedPaymentMethod as generatePaymentMethod } from "@test/seeders/payment-method.seeder"; @@ -30,7 +31,7 @@ describe(WalletBalanceReloadCheckHandler.name, () => { const costUntilTargetDateInFiat = 50.0; const expectedReloadAmount = 40.0; // max(50 - 10, 20) = 40 - const { handler, drainingDeploymentService, stripeService, loggerService, walletReloadJobService, job, jobMeta } = setup({ + const { handler, drainingDeploymentService, stripeService, instrumentationService, walletReloadJobService, job, jobMeta } = setup({ balance: { total: balance }, weeklyCostInDenom: costUntilTargetDateInDenom, weeklyCostInFiat: costUntilTargetDateInFiat @@ -70,9 +71,13 @@ describe(WalletBalanceReloadCheckHandler.name, () => { confirm: true, idempotencyKey: `${WalletBalanceReloadCheck.name}.${jobMeta.id}` }); - expect(loggerService.info).toHaveBeenCalledWith( + expect(instrumentationService.recordReloadTriggered).toHaveBeenCalledWith( + expectedReloadAmount, + balance, + expect.any(Number), + costUntilTargetDateInFiat, expect.objectContaining({ - event: "WALLET_BALANCE_RELOADED", + walletAddress: expect.any(String), balance, costUntilTargetDateInFiat }) @@ -113,7 +118,7 @@ describe(WalletBalanceReloadCheckHandler.name, () => { const costUntilTargetDateInDenom = 50_000_000; const costUntilTargetDateInFiat = 50.0; - const { handler, stripeService, loggerService, job, jobMeta } = setup({ + const { handler, stripeService, instrumentationService, job, jobMeta } = setup({ balance: { total: balance }, weeklyCostInDenom: costUntilTargetDateInDenom, weeklyCostInFiat: costUntilTargetDateInFiat @@ -122,9 +127,13 @@ describe(WalletBalanceReloadCheckHandler.name, () => { await handler.handle(job, jobMeta); expect(stripeService.createPaymentIntent).not.toHaveBeenCalled(); - expect(loggerService.info).toHaveBeenCalledWith( + expect(instrumentationService.recordReloadSkipped).toHaveBeenCalledWith( + balance, + expect.any(Number), + costUntilTargetDateInFiat, + "sufficient_balance", expect.objectContaining({ - event: "WALLET_BALANCE_RELOAD_SKIPPED", + walletAddress: expect.any(String), balance, costUntilTargetDateInFiat }) @@ -138,7 +147,7 @@ describe(WalletBalanceReloadCheckHandler.name, () => { const costUntilTargetDateInDenom = 50_000_000; const costUntilTargetDateInFiat = 50.0; - const { handler, stripeService, loggerService, job, jobMeta } = setup({ + const { handler, stripeService, instrumentationService, job, jobMeta } = setup({ balance: { total: balance }, weeklyCostInDenom: costUntilTargetDateInDenom, weeklyCostInFiat: costUntilTargetDateInFiat @@ -147,9 +156,13 @@ describe(WalletBalanceReloadCheckHandler.name, () => { await handler.handle(job, jobMeta); expect(stripeService.createPaymentIntent).not.toHaveBeenCalled(); - expect(loggerService.info).toHaveBeenCalledWith( + expect(instrumentationService.recordReloadSkipped).toHaveBeenCalledWith( + balance, + expect.any(Number), + costUntilTargetDateInFiat, + "sufficient_balance", expect.objectContaining({ - event: "WALLET_BALANCE_RELOAD_SKIPPED", + walletAddress: expect.any(String), balance, costUntilTargetDateInFiat }) @@ -187,7 +200,7 @@ describe(WalletBalanceReloadCheckHandler.name, () => { const weeklyCostInFiat = 50.0; const error = new Error("Failed to schedule"); - const { handler, walletReloadJobService, loggerService, job, jobMeta } = setup({ + const { handler, walletReloadJobService, instrumentationService, job, jobMeta } = setup({ balance: { total: balance }, weeklyCostInDenom, weeklyCostInFiat @@ -196,92 +209,101 @@ describe(WalletBalanceReloadCheckHandler.name, () => { await expect(handler.handle(job, jobMeta)).rejects.toThrow(error); - expect(loggerService.error).toHaveBeenCalledWith( - expect.objectContaining({ - event: "ERROR_SCHEDULING_NEXT_CHECK", - walletAddress: expect.any(String), - error - }) - ); + expect(instrumentationService.recordSchedulingError).toHaveBeenCalledWith(expect.any(String), error); }); it("logs validation error when wallet setting not found", async () => { - const { handler, walletSettingRepository, loggerService, job, jobMeta } = setup({ + const { handler, walletSettingRepository, instrumentationService, job, jobMeta } = setup({ walletSettingNotFound: true }); await handler.handle(job, jobMeta); - expect(loggerService.error).toHaveBeenCalledWith({ - event: "WALLET_SETTING_NOT_FOUND", - message: "Wallet setting not found. Skipping wallet balance reload check.", - userId: job.userId - }); + expect(instrumentationService.recordValidationError).toHaveBeenCalledWith( + "WALLET_SETTING_NOT_FOUND", + { + event: "WALLET_SETTING_NOT_FOUND", + message: "Wallet setting not found. Skipping wallet balance reload check." + }, + job.userId + ); expect(walletSettingRepository.findInternalByUserIdWithRelations).toHaveBeenCalledWith(job.userId); }); it("logs validation error when auto reload is disabled", async () => { - const { handler, walletSettingRepository, loggerService, job, jobMeta } = setup({ + const { handler, walletSettingRepository, instrumentationService, job, jobMeta } = setup({ autoReloadEnabled: false }); await handler.handle(job, jobMeta); expect(walletSettingRepository.findInternalByUserIdWithRelations).toHaveBeenCalledWith(job.userId); - expect(loggerService.error).toHaveBeenCalledWith({ - event: "AUTO_RELOAD_DISABLED", - message: "Auto reload disabled. Skipping wallet balance reload check.", - userId: job.userId - }); + expect(instrumentationService.recordValidationError).toHaveBeenCalledWith( + "AUTO_RELOAD_DISABLED", + { + event: "AUTO_RELOAD_DISABLED", + message: "Auto reload disabled. Skipping wallet balance reload check." + }, + job.userId + ); }); it("logs validation error when wallet is not initialized", async () => { - const { handler, walletSettingRepository, loggerService, job, jobMeta } = setup({ + const { handler, walletSettingRepository, instrumentationService, job, jobMeta } = setup({ wallet: UserWalletSeeder.create({ address: null }) }); await handler.handle(job, jobMeta); expect(walletSettingRepository.findInternalByUserIdWithRelations).toHaveBeenCalledWith(job.userId); - expect(loggerService.error).toHaveBeenCalledWith({ - event: "WALLET_NOT_INITIALIZED", - message: "Wallet not initialized. Skipping wallet balance reload check.", - userId: job.userId - }); + expect(instrumentationService.recordValidationError).toHaveBeenCalledWith( + "WALLET_NOT_INITIALIZED", + { + event: "WALLET_NOT_INITIALIZED", + message: "Wallet not initialized. Skipping wallet balance reload check." + }, + job.userId + ); }); it("logs validation error when user stripe customer ID is not set", async () => { const userWithoutStripe = UserSeeder.create(); const userWithNullStripe = { ...userWithoutStripe, stripeCustomerId: null }; - const { handler, walletSettingRepository, loggerService, job, jobMeta } = setup({ + const { handler, walletSettingRepository, instrumentationService, job, jobMeta } = setup({ user: userWithNullStripe }); await handler.handle(job, jobMeta); expect(walletSettingRepository.findInternalByUserIdWithRelations).toHaveBeenCalledWith(job.userId); - expect(loggerService.error).toHaveBeenCalledWith({ - event: "USER_STRIPE_CUSTOMER_ID_NOT_SET", - message: "User stripe customer ID not set. Skipping wallet balance reload check.", - userId: job.userId - }); + expect(instrumentationService.recordValidationError).toHaveBeenCalledWith( + "USER_STRIPE_CUSTOMER_ID_NOT_SET", + { + event: "USER_STRIPE_CUSTOMER_ID_NOT_SET", + message: "User stripe customer ID not set. Skipping wallet balance reload check." + }, + job.userId + ); }); it("logs validation error when default payment method cannot be retrieved", async () => { const balance = 15.0; - const { handler, loggerService, stripeService, job, jobMeta } = setup({ + const { handler, instrumentationService, stripeService, job, jobMeta } = setup({ balance: { total: balance } }); stripeService.getDefaultPaymentMethod.mockResolvedValue(undefined); await handler.handle(job, jobMeta); - expect(loggerService.error).toHaveBeenCalledWith({ - event: "DEFAULT_PAYMENT_METHOD_NOT_FOUND", - message: "Default payment method not found", - userId: job.userId - }); + expect(instrumentationService.recordValidationError).toHaveBeenCalledWith( + "DEFAULT_PAYMENT_METHOD_NOT_FOUND", + { + event: "DEFAULT_PAYMENT_METHOD_NOT_FOUND", + message: "Default payment method not found" + }, + job.userId + ); }); }); @@ -333,7 +355,14 @@ describe(WalletBalanceReloadCheckHandler.name, () => { const walletReloadJobService = mock(); const drainingDeploymentService = mock(); const stripeService = mock(); - const loggerService = mock(); + const instrumentationService = mock({ + recordJobExecution: jest.fn(), + recordReloadTriggered: jest.fn(), + recordReloadSkipped: jest.fn(), + recordReloadFailed: jest.fn(), + recordValidationError: jest.fn(), + recordSchedulingError: jest.fn() + }); const balance = input?.balance ?? { total: 50.0 }; const weeklyCostInDenom = input?.weeklyCostInDenom ?? 50_000_000; @@ -361,7 +390,7 @@ describe(WalletBalanceReloadCheckHandler.name, () => { walletReloadJobService, stripeService, drainingDeploymentService, - loggerService + instrumentationService ); return { @@ -371,7 +400,7 @@ describe(WalletBalanceReloadCheckHandler.name, () => { walletReloadJobService, drainingDeploymentService, stripeService, - loggerService, + instrumentationService, walletSetting, walletSettingWithWallet, wallet, diff --git a/apps/api/src/billing/services/wallet-balance-reload-check/wallet-balance-reload-check.handler.ts b/apps/api/src/billing/services/wallet-balance-reload-check/wallet-balance-reload-check.handler.ts index c664b232f3..8a87e69944 100644 --- a/apps/api/src/billing/services/wallet-balance-reload-check/wallet-balance-reload-check.handler.ts +++ b/apps/api/src/billing/services/wallet-balance-reload-check/wallet-balance-reload-check.handler.ts @@ -9,10 +9,11 @@ import { UserWalletOutput, WalletSettingOutput, WalletSettingRepository } from " import { BalancesService } from "@src/billing/services/balances/balances.service"; import { PaymentMethod, StripeService } from "@src/billing/services/stripe/stripe.service"; import { WalletReloadJobService } from "@src/billing/services/wallet-reload-job/wallet-reload-job.service"; -import { JobHandler, JobMeta, JobPayload, LoggerService } from "@src/core"; +import { JobHandler, JobMeta, JobPayload } from "@src/core"; import type { Require } from "@src/core/types/require.type"; import { DrainingDeploymentService } from "@src/deployment/services/draining-deployment/draining-deployment.service"; import { isPayingUser, PayingUser } from "../paying-user/paying-user"; +import { WalletBalanceReloadCheckInstrumentationService } from "./wallet-balance-reload-check-instrumentation.service"; type ValidationError = { event: string; @@ -53,19 +54,27 @@ export class WalletBalanceReloadCheckHandler implements JobHandler, job: JobMeta): Promise { - const resourcesResult = await this.#collectResources(payload); + const startTime = Date.now(); + let success = false; - if (resourcesResult.ok) { - await this.#tryToReload({ ...resourcesResult.val, job }); - await this.#scheduleNextCheck(resourcesResult.val); - } else { - return this.#finishWithValidationError(resourcesResult.val, payload.userId); + try { + const resourcesResult = await this.#collectResources(payload); + + if (resourcesResult.ok) { + await this.#tryToReload({ ...resourcesResult.val, job }); + await this.#scheduleNextCheck(resourcesResult.val); + success = true; + } else { + this.instrumentationService.recordValidationError(resourcesResult.val.event, resourcesResult.val, payload.userId); + return; + } + } finally { + const durationMs = Date.now() - startTime; + this.instrumentationService.recordJobExecution(durationMs, success, payload.userId); } } @@ -155,13 +164,6 @@ export class WalletBalanceReloadCheckHandler implements JobHandler["userId"]): void { - this.loggerService.error({ - ...error, - userId: userId - }); - } - async #tryToReload(resources: AllResources & { job: JobMeta }): Promise { const reloadTargetDate = addMilliseconds(new Date(), this.#RELOAD_COVERAGE_PERIOD_IN_MS); const costUntilTargetDateInDenom = await this.drainingDeploymentService.calculateAllDeploymentCostUntilDate(resources.wallet.address, reloadTargetDate); @@ -174,17 +176,19 @@ export class WalletBalanceReloadCheckHandler implements JobHandler= threshold) { - this.loggerService.info({ - ...log, - event: "WALLET_BALANCE_RELOAD_SKIPPED" - }); + if (costUntilTargetDateInFiat === 0) { + this.instrumentationService.recordReloadSkipped(resources.balance, threshold, costUntilTargetDateInFiat, "zero_cost", log); return; } - try { - const reloadAmountInFiat = Math.max(costUntilTargetDateInFiat - resources.balance, this.#MIN_RELOAD_AMOUNT_IN_USD); + if (resources.balance >= threshold) { + this.instrumentationService.recordReloadSkipped(resources.balance, threshold, costUntilTargetDateInFiat, "sufficient_balance", log); + return; + } + + const reloadAmountInFiat = Math.max(costUntilTargetDateInFiat - resources.balance, this.#MIN_RELOAD_AMOUNT_IN_USD); + try { await this.stripeService.createPaymentIntent({ customer: resources.user.stripeCustomerId, payment_method: resources.paymentMethod.id, @@ -193,17 +197,9 @@ export class WalletBalanceReloadCheckHandler implements JobHandler