From 20aa0c067439fd5fd6980cd9b7f203d965f15bf8 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Wed, 26 Feb 2025 00:44:01 -0500 Subject: [PATCH 01/17] Add rate limiting and error handling utilities for LLM and transaction processing --- src/actual-ai.ts | 47 +++++- src/llm-service.ts | 67 ++++++--- src/transaction-service.ts | 90 +++++++---- src/utils/error-utils.ts | 88 +++++++++++ src/utils/provider-limits.ts | 30 ++++ src/utils/rate-limiter.ts | 282 +++++++++++++++++++++++++++++++++++ 6 files changed, 550 insertions(+), 54 deletions(-) create mode 100644 src/utils/error-utils.ts create mode 100644 src/utils/provider-limits.ts create mode 100644 src/utils/rate-limiter.ts diff --git a/src/actual-ai.ts b/src/actual-ai.ts index f6d508b..b41d524 100644 --- a/src/actual-ai.ts +++ b/src/actual-ai.ts @@ -1,5 +1,6 @@ import { ActualAiServiceI, ActualApiServiceI, TransactionServiceI } from './types'; import suppressConsoleLogsAsync from './utils'; +import { formatError } from './utils/error-utils'; class ActualAiService implements ActualAiServiceI { private readonly transactionService: TransactionServiceI; @@ -28,19 +29,40 @@ class ActualAiService implements ActualAiServiceI { await this.syncAccounts(); } } catch (error) { - console.error('Bank sync failed, continuing with existing transactions:', error); + console.error( + 'Bank sync failed, continuing with existing transactions:', + formatError(error), + ); } // These should run even if sync failed await this.transactionService.migrateToTags(); - await this.transactionService.processTransactions(); + + try { + await this.transactionService.processTransactions(); + } catch (error) { + if (this.isRateLimitError(error)) { + console.error('Rate limit reached during transaction processing. Consider:'); + console.error('1. Adjusting rate limits in provider-limits.ts'); + console.error('2. Switching to a provider with higher limits'); + console.error('3. Breaking your processing into smaller batches'); + } else { + console.error( + 'An error occurred during transaction processing:', + formatError(error), + ); + } + } } catch (error) { - console.error('An error occurred:', error); + console.error( + 'An error occurred:', + formatError(error), + ); } finally { try { await this.actualApiService.shutdownApi(); } catch (shutdownError) { - console.error('Error during API shutdown:', shutdownError); + console.error('Error during API shutdown:', formatError(shutdownError)); } } } @@ -51,9 +73,24 @@ class ActualAiService implements ActualAiServiceI { await suppressConsoleLogsAsync(async () => this.actualApiService.runBankSync()); console.log('Bank accounts synced'); } catch (error) { - console.error('Error syncing bank accounts:', error); + console.error( + 'Error syncing bank accounts:', + formatError(error), + ); } } + + private isRateLimitError(error: unknown): boolean { + if (!error) return false; + + const errorStr = formatError(error); + return errorStr.includes('Rate limit') + || errorStr.includes('rate limited') + || errorStr.includes('rate_limit_exceeded') + || (error instanceof Error + && 'statusCode' in error + && (error as unknown as { statusCode: number }).statusCode === 429); + } } export default ActualAiService; diff --git a/src/llm-service.ts b/src/llm-service.ts index e6eaef9..758c786 100644 --- a/src/llm-service.ts +++ b/src/llm-service.ts @@ -1,9 +1,15 @@ import { generateObject, generateText, LanguageModel } from 'ai'; import { LlmModelFactoryI, LlmServiceI } from './types'; +import { RateLimiter } from './utils/rate-limiter'; +import { PROVIDER_LIMITS } from './utils/provider-limits'; export default class LlmService implements LlmServiceI { private readonly model: LanguageModel; + private readonly rateLimiter: RateLimiter; + + private readonly provider: string; + private isFallbackMode; constructor( @@ -11,35 +17,58 @@ export default class LlmService implements LlmServiceI { ) { this.model = llmModelFactory.create(); this.isFallbackMode = llmModelFactory.isFallbackMode(); + this.provider = llmModelFactory.getProvider(); + this.rateLimiter = new RateLimiter(); + + // Set rate limits for the provider + const limits = PROVIDER_LIMITS[this.provider]; + if (limits) { + this.rateLimiter.setProviderLimit(this.provider, limits.requestsPerMinute); + } } public async ask(prompt: string, categoryIds: string[]): Promise { - if (this.isFallbackMode) { - return this.askUsingFallbackModel(prompt); - } + try { + if (this.isFallbackMode) { + return await this.askUsingFallbackModel(prompt); + } - return this.askWithEnum(prompt, categoryIds); + return await this.askWithEnum(prompt, categoryIds); + } catch (error) { + console.error(`Error during LLM request: ${error instanceof Error ? error.message : String(error)}`); + throw error; + } } public async askWithEnum(prompt: string, categoryIds: string[]): Promise { - const { object } = await generateObject({ - model: this.model, - output: 'enum', - enum: categoryIds, - prompt, - temperature: 0.1, - }); - - return object.replace(/(\r\n|\n|\r|"|')/gm, ''); + return this.rateLimiter.executeWithRateLimiting( + this.provider, + async () => { + const { object } = await generateObject({ + model: this.model, + output: 'enum', + enum: categoryIds, + prompt, + temperature: 0.1, + }); + + return object.replace(/(\r\n|\n|\r|"|')/gm, ''); + }, + ); } public async askUsingFallbackModel(prompt: string): Promise { - const { text } = await generateText({ - model: this.model, - prompt, - temperature: 0.1, - }); + return this.rateLimiter.executeWithRateLimiting( + this.provider, + async () => { + const { text } = await generateText({ + model: this.model, + prompt, + temperature: 0.1, + }); - return text.replace(/(\r\n|\n|\r|"|')/gm, ''); + return text.replace(/(\r\n|\n|\r|"|')/gm, ''); + }, + ); } } diff --git a/src/transaction-service.ts b/src/transaction-service.ts index 714603a..bf5aad2 100644 --- a/src/transaction-service.ts +++ b/src/transaction-service.ts @@ -4,6 +4,7 @@ import { const LEGACY_NOTES_NOT_GUESSED = 'actual-ai could not guess this category'; const LEGACY_NOTES_GUESSED = 'actual-ai guessed this category'; +const BATCH_SIZE = 20; // Process transactions in batches of 20 class TransactionService implements TransactionServiceI { private readonly actualApiService: ActualApiServiceI; @@ -91,40 +92,69 @@ class TransactionService implements TransactionServiceI { && !accountsToSkip.includes(transaction.account), ); - for (let i = 0; i < uncategorizedTransactions.length; i++) { - const transaction = uncategorizedTransactions[i]; - console.log(`${i + 1}/${uncategorizedTransactions.length} Processing transaction ${transaction.imported_payee} / ${transaction.notes} / ${transaction.amount}`); - const prompt = this.promptGenerator.generate(categoryGroups, transaction, payees); - const categoryIds = categories.map((category) => category.id); - categoryIds.push('uncategorized'); - const guess = await this.llmService.ask(prompt, categoryIds); - let guessCategory = categories.find((category) => category.id === guess); - - if (!guessCategory) { - guessCategory = categories.find((category) => category.name === guess); - if (guessCategory) { - console.warn(`${i + 1}/${uncategorizedTransactions.length} LLM guessed category name instead of ID. LLM guess: ${guess}`); - } - } - if (!guessCategory) { - guessCategory = categories.find((category) => guess.includes(category.id)); - if (guessCategory) { - console.warn(`${i + 1}/${uncategorizedTransactions.length} Found category ID in LLM guess, but it wasn't 1:1. LLM guess: ${guess}`); + console.log(`Found ${uncategorizedTransactions.length} transactions to process`); + const categoryIds = categories.map((category) => category.id); + categoryIds.push('uncategorized'); + + // Process transactions in batches to avoid hitting rate limits + for ( + let batchStart = 0; + batchStart < uncategorizedTransactions.length; + batchStart += BATCH_SIZE + ) { + const batchEnd = Math.min(batchStart + BATCH_SIZE, uncategorizedTransactions.length); + console.log(`Processing batch ${batchStart / BATCH_SIZE + 1} (transactions ${batchStart + 1}-${batchEnd})`); + + const batch = uncategorizedTransactions.slice(batchStart, batchEnd); + + for (let i = 0; i < batch.length; i++) { + const transaction = batch[i]; + const globalIndex = batchStart + i; + console.log(`${globalIndex + 1}/${uncategorizedTransactions.length} Processing transaction ${transaction.imported_payee} / ${transaction.notes} / ${transaction.amount}`); + + try { + const prompt = this.promptGenerator.generate(categoryGroups, transaction, payees); + const guess = await this.llmService.ask(prompt, categoryIds); + let guessCategory = categories.find((category) => category.id === guess); + + if (!guessCategory) { + guessCategory = categories.find((category) => category.name === guess); + if (guessCategory) { + console.warn(`${globalIndex + 1}/${uncategorizedTransactions.length} LLM guessed category name instead of ID. LLM guess: ${guess}`); + } + } + if (!guessCategory) { + guessCategory = categories.find((category) => guess.includes(category.id)); + if (guessCategory) { + console.warn(`${globalIndex + 1}/${uncategorizedTransactions.length} Found category ID in LLM guess, but it wasn't 1:1. LLM guess: ${guess}`); + } + } + + if (!guessCategory) { + console.warn(`${globalIndex + 1}/${uncategorizedTransactions.length} LLM could not classify the transaction. LLM guess: ${guess}`); + await this.actualApiService.updateTransactionNotes(transaction.id, this.appendTag(transaction.notes ?? '', this.notGuessedTag)); + continue; + } + console.log(`${globalIndex + 1}/${uncategorizedTransactions.length} Guess: ${guessCategory.name}`); + + await this.actualApiService.updateTransactionNotesAndCategory( + transaction.id, + this.appendTag(transaction.notes ?? '', this.guessedTag), + guessCategory.id, + ); + } catch (error) { + console.error(`Error processing transaction ${globalIndex + 1}/${uncategorizedTransactions.length}:`, error); + // Continue with next transaction } } - if (!guessCategory) { - console.warn(`${i + 1}/${uncategorizedTransactions.length} LLM could not classify the transaction. LLM guess: ${guess}`); - await this.actualApiService.updateTransactionNotes(transaction.id, this.appendTag(transaction.notes ?? '', this.notGuessedTag)); - continue; + // Add a small delay between batches to avoid overwhelming the API + if (batchEnd < uncategorizedTransactions.length) { + console.log('Pausing for 2 seconds before next batch...'); + await new Promise((resolve) => { + setTimeout(resolve, 2000); + }); } - console.log(`${i + 1}/${uncategorizedTransactions.length} Guess: ${guessCategory.name}`); - - await this.actualApiService.updateTransactionNotesAndCategory( - transaction.id, - this.appendTag(transaction.notes ?? '', this.guessedTag), - guessCategory.id, - ); } } } diff --git a/src/utils/error-utils.ts b/src/utils/error-utils.ts new file mode 100644 index 0000000..76a0d14 --- /dev/null +++ b/src/utils/error-utils.ts @@ -0,0 +1,88 @@ +/** + * Checks if an error is a rate limit error + * @param error Any error object or value + * @returns boolean indicating if this is a rate limit error + */ +export const isRateLimitError = (error: unknown): boolean => { + if (!error) return false; + + // Convert to string to handle various error types + const errorStr = String(error); + + // Check for common rate limit indicators + return errorStr.toLowerCase().includes('rate limit') + || errorStr.toLowerCase().includes('rate_limit') + || errorStr.toLowerCase().includes('too many requests') + || (error instanceof Error + && 'statusCode' in error + && (error as any).statusCode === 429); +}; + +/** + * Attempts to extract retry-after information from a rate limit error + * @param error Rate limit error + * @returns Time to wait in milliseconds or undefined if not found + */ +export const extractRetryAfterMs = (error: unknown): number | undefined => { + if (!isRateLimitError(error)) return undefined; + + if (error instanceof Error) { + try { + // Check for retry information in error message (common in provider responses) + const match = /try again in (\d+(\.\d+)?)s/i.exec(error.message); + if (match?.[1]) { + return Math.ceil(parseFloat(match[1]) * 1000); + } + + // Try to get from headers if available + if ('responseHeaders' in error && (error as any).responseHeaders) { + const headers = (error as any).responseHeaders; + if (headers['retry-after'] || headers['Retry-After']) { + const retryAfter = headers['retry-after'] || headers['Retry-After']; + if (!isNaN(Number(retryAfter))) { + return Number(retryAfter) * 1000; + } + } + } + + // Check for reset time in responseBody if it exists + if ('responseBody' in error && typeof (error as any).responseBody === 'string') { + try { + const body = JSON.parse((error as any).responseBody); + if (body.error?.reset_time) { + return body.error.reset_time * 1000; + } + } catch (e) { + // Ignore JSON parse errors + } + } + } catch (e) { + console.warn('Error extracting retry-after information:', e); + } + } + + return undefined; +}; + +/** + * Formats an error into a string + * @param error Any error object or value + * @returns string representation of the error + */ +export const formatError = (error: unknown): string => { + if (error instanceof Error) return error.message; + if (typeof error === 'object' && error !== null) { + try { + return JSON.stringify(error); + } catch { + return '[object Object]'; + } + } + return String(error); +}; + +export default { + isRateLimitError, + extractRetryAfterMs, + formatError, +}; diff --git a/src/utils/provider-limits.ts b/src/utils/provider-limits.ts new file mode 100644 index 0000000..d9ec5db --- /dev/null +++ b/src/utils/provider-limits.ts @@ -0,0 +1,30 @@ +export interface ProviderLimits { + tokensPerMinute: number; + requestsPerMinute: number; +} + +// Default conservative limits - these should be updated based on user's plan +export const PROVIDER_LIMITS: Record = { + openai: { + tokensPerMinute: 60000, + requestsPerMinute: 500, + }, + anthropic: { + tokensPerMinute: 100000, + requestsPerMinute: 400, + }, + google: { + tokensPerMinute: 60000, + requestsPerMinute: 300, + }, + groq: { + tokensPerMinute: 6000, + requestsPerMinute: 100, + }, + ollama: { + tokensPerMinute: 10000, // This is a local model, so limits depend on your hardware + requestsPerMinute: 50, + }, +}; + +export default PROVIDER_LIMITS; diff --git a/src/utils/rate-limiter.ts b/src/utils/rate-limiter.ts new file mode 100644 index 0000000..740bf2e --- /dev/null +++ b/src/utils/rate-limiter.ts @@ -0,0 +1,282 @@ +// Define a custom error type for API rate limiting errors +interface RateLimitError extends Error { + statusCode?: number; + responseHeaders?: Record; +} + +interface RetryParams { + retryAfterMs?: number; + maxRetries: number; + baseDelayMs: number; + maxDelayMs: number; + jitter: boolean; +} + +interface TokenBucket { + limit: number; + remaining: number; + resetTimestamp: number; +} + +export class RateLimiter { + private requestCounts = new Map(); + + private lastRequestTime = new Map(); + + private maxRequestsPerMinute = new Map(); + + private tokenBuckets = new Map(); + + private debugMode = false; + + constructor(debug = false) { + this.debugMode = debug; + } + + public setProviderLimit(provider: string, limit: number): void { + this.maxRequestsPerMinute.set(provider, limit); + } + + public enableDebug(): void { + this.debugMode = true; + } + + public async executeWithRateLimiting( + provider: string, + operation: () => Promise, + retryParams: RetryParams = { + maxRetries: 5, + baseDelayMs: 1000, + maxDelayMs: 60000, + jitter: true, + }, + ): Promise { + let attempt = 0; + let lastError: Error | null = null; + + while (attempt <= retryParams.maxRetries) { + try { + if (attempt > 0) { + console.log(`Retry attempt ${attempt}/${retryParams.maxRetries} for ${provider}...`); + } + + // Wait before proceeding if we need to + await this.waitIfNeeded(provider); + + // Track this request + this.trackRequest(provider); + + return await operation(); + } catch (error) { + lastError = error as Error; + + if (this.isRateLimitError(error)) { + // Update token bucket information if available + this.updateTokenBucketFromError(provider, error); + + // Get retry delay from error or calculate backoff + const retryAfterMs = this.extractRetryAfterMs(error) ?? this.calculateBackoff( + attempt, + retryParams.baseDelayMs, + retryParams.maxDelayMs, + retryParams.jitter, + ); + + // Add additional details in debug mode + if (this.debugMode) { + console.log(`Rate limit details for ${provider}:`, this.getRateLimitDebugInfo(provider, error)); + } + + console.log(`Rate limit hit for ${provider}. Waiting ${retryAfterMs}ms before retry.`); + await this.sleep(retryAfterMs); + attempt += 1; + } else { + // Not a rate limit error, rethrow + throw error; + } + } + } + + // If we've exhausted all retries + throw new Error(`Rate limit retries exceeded (${retryParams.maxRetries}). Last error: ${lastError?.message}`); + } + + private getRateLimitDebugInfo(provider: string, error: unknown): object { + const bucket = this.tokenBuckets.get(provider); + const errorInfo: { message: string; statusCode?: number; headers?: Record } = { + message: '', + }; + + if (error instanceof Error) { + errorInfo.message = error.message; + // Type guard for rate limit errors + const rateLimitError = error as Partial; + if ('statusCode' in error) errorInfo.statusCode = rateLimitError.statusCode; + if ('responseHeaders' in error) errorInfo.headers = rateLimitError.responseHeaders; + } + + return { + provider, + errorInfo, + tokenBucket: bucket ?? 'No token data available', + requestsInLastMinute: this.requestCounts.get(provider) ?? 0, + maxRequestsPerMinute: this.maxRequestsPerMinute.get(provider) ?? 'No limit set', + }; + } + + private updateTokenBucketFromError(provider: string, error: unknown): void { + if (!(error instanceof Error)) return; + + try { + const errorMsg = error.message; + + // Extract Groq token information + // Example: "Limit 100000, Used 99336, Requested 821" + const limitMatch = /Limit (\d+), Used (\d+), Requested (\d+)/.exec(errorMsg); + if (limitMatch) { + const [, limitStr, usedStr] = limitMatch; + const limit = parseInt(limitStr, 10); + const used = parseInt(usedStr, 10); + + // Extract wait time: "Please try again in 2m14.975999999s" + const waitTimeMatch = /try again in ((\d+)m)?(\d+(\.\d+)?)s/i.exec(errorMsg); + let waitTimeMs = 0; + + if (waitTimeMatch) { + const minutes = waitTimeMatch[2] ? parseInt(waitTimeMatch[2], 10) : 0; + const seconds = parseFloat(waitTimeMatch[3]); + waitTimeMs = (minutes * 60 + seconds) * 1000; + } + + const now = Date.now(); + this.tokenBuckets.set(provider, { + limit, + remaining: Math.max(0, limit - used), + resetTimestamp: now + waitTimeMs, + }); + + if (this.debugMode) { + console.log(`Updated token bucket for ${provider}:`, this.tokenBuckets.get(provider)); + } + } + } catch (e) { + console.warn('Error updating token bucket from error:', e); + } + } + + private isRateLimitError(error: unknown): boolean { + if (error instanceof Error) { + // Check for common rate limit status codes and messages + const rateLimitError = error as Partial; + if ('statusCode' in error && rateLimitError.statusCode === 429) { + return true; + } + + // Check for rate limit messages + const errorMessage = error.message.toLowerCase(); + return errorMessage.includes('rate limit') + || errorMessage.includes('too many requests'); + } + return false; + } + + private extractRetryAfterMs(error: unknown): number | undefined { + if (error instanceof Error) { + try { + // Try to extract from Groq error message + const match = /try again in ((\d+)m)?(\d+(\.\d+)?)s/i.exec(error.message); + if (match) { + const minutes = match[2] ? parseInt(match[2], 10) : 0; + const seconds = parseFloat(match[3]); + return Math.ceil((minutes * 60 + seconds) * 1000); + } + + // Try to get from headers if available + const rateLimitError = error as Partial; + if ('responseHeaders' in error && rateLimitError.responseHeaders) { + const headers = rateLimitError.responseHeaders; + if (headers && 'retry-after' in headers) { + const retryAfter = headers['retry-after']; + if (retryAfter && Number.isNaN(Number(retryAfter)) === false) { + return Number(retryAfter) * 1000; + } + } + } + } catch (e) { + console.warn('Error extracting retry-after information:', e); + } + } + return undefined; + } + + private calculateBackoff( + attempt: number, + baseDelay: number, + maxDelay: number, + jitter: boolean, + ): number { + // Exponential backoff: baseDelay * 2^attempt + let delay = Math.min(baseDelay * 2 ** attempt, maxDelay); + + // Add jitter to avoid thundering herd problem + if (jitter) { + delay *= (0.5 + Math.random() * 0.5); + } + + return Math.floor(delay); + } + + private trackRequest(provider: string): void { + const now = Date.now(); + const count = this.requestCounts.get(provider) ?? 0; + const lastTime = this.lastRequestTime.get(provider) ?? 0; + + // Reset counter if more than a minute has passed + if (now - lastTime > 60000) { + this.requestCounts.set(provider, 1); + } else { + this.requestCounts.set(provider, count + 1); + } + + this.lastRequestTime.set(provider, now); + } + + private async waitIfNeeded(provider: string): Promise { + const limit = this.maxRequestsPerMinute.get(provider) ?? 0; + const count = this.requestCounts.get(provider) ?? 0; + const lastTime = this.lastRequestTime.get(provider) ?? 0; + const now = Date.now(); + let waitTime = 0; + + // Check token bucket first - this has priority + const bucket = this.tokenBuckets.get(provider); + if (bucket && bucket.resetTimestamp > now) { + // If we're close to the limit and reset time is in the future + if (bucket.remaining < bucket.limit * 0.10) { + waitTime = bucket.resetTimestamp - now + 1000; // add 1 second buffer + console.log(`Waiting ${waitTime}ms for token bucket to reset for ${provider}`); + await this.sleep(waitTime); + return; + } + } + + // If we have a request limit set and we're approaching it + if (limit && count >= limit * 0.8) { + // If less than a minute has passed since the first request in this window + if (now - lastTime < 60000) { + // Calculate time remaining until the minute is up + waitTime = 60000 - (now - lastTime) + 100; // add 100ms buffer + console.log(`Preemptively waiting ${waitTime}ms to avoid rate limit for ${provider}`); + await this.sleep(waitTime); + } + } + } + + private sleep(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); + } +} + +export default RateLimiter; From 01186f71c4662e1dc1fcddbce3b99813dd4eefd8 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Fri, 28 Feb 2025 17:32:35 -0500 Subject: [PATCH 02/17] Add new category suggestion feature with LLM support - WIP --- .eslintrc.json | 2 +- README.md | 18 ++++ package-lock.json | 11 +-- package.json | 3 +- src/actual-api-service.ts | 9 ++ src/config.ts | 7 ++ src/container.ts | 7 +- src/llm-model-factory.ts | 8 ++ src/llm-service.ts | 57 +++++++++++- src/prompt-generator.ts | 38 +++++++- src/templates/category-suggestion.hbs | 27 ++++++ src/templates/prompt.hbs | 12 ++- src/transaction-service.ts | 125 +++++++++++++++++++++++++- src/types.ts | 24 ++++- 14 files changed, 328 insertions(+), 20 deletions(-) create mode 100644 src/templates/category-suggestion.hbs diff --git a/.eslintrc.json b/.eslintrc.json index 198de32..5f3436e 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -31,7 +31,7 @@ } ], "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": ["error"] + "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }] }, "parserOptions": { "ecmaVersion": 2020, diff --git a/README.md b/README.md index a201a8e..3ebd993 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,10 @@ The app sends requests to the LLM to classify transactions based on their descri #### ✅ Every guessed transaction is marked as guessed in notes, so you can review the classification. +#### 🌱 Suggest and create new categories for transactions that don't fit existing ones + +When enabled, the LLM can suggest entirely new categories for transactions it cannot classify, and optionally create them automatically. + ## 🚀 Usage Sample `docker-compose.yml` file: @@ -56,6 +60,8 @@ services: CLASSIFY_ON_STARTUP: true # Whether to classify transactions on startup (don't wait for cron schedule) SYNC_ACCOUNTS_BEFORE_CLASSIFY: false # Whether to sync accounts before classification LLM_PROVIDER: openai # Can be "openai", "anthropic", "google-generative-ai", "ollama" or "groq" +# SUGGEST_NEW_CATEGORIES: false # Whether to suggest new categories for transactions that can't be classified with existing ones +# DRY_RUN_NEW_CATEGORIES: true # When true, just logs suggested categories without creating them # OPENAI_API_KEY: # optional. required if you want to use the OpenAI API # OPENAI_MODEL: # optional. required if you want to use a specific model, default is "gpt-4o-mini" # OPENAI_BASE_URL: # optional. required if you don't want to use the OpenAI API but OpenAI compatible API, ex: "http://ollama:11424/v1 @@ -120,3 +126,15 @@ loops. 7. `date`: The date of the transaction. This is taken from `transaction.date`. 8. `cleared`: A boolean indicating if the transaction is cleared. This is taken from `transaction.cleared`. 9. `reconciled`: A boolean indicating if the transaction is reconciled. This is taken from `transaction.reconciled`. + +## New Category Suggestions + +When `SUGGEST_NEW_CATEGORIES` is enabled, the system will: + +1. First try to classify transactions using existing categories +2. For transactions that can't be classified, request a new category suggestion from the LLM +3. Check if similar categories already exist +4. If in dry run mode (`DRY_RUN_NEW_CATEGORIES=true`), just log the suggestions +5. If not in dry run mode (`DRY_RUN_NEW_CATEGORIES=false`), create the new categories and assign transactions to them + +This feature is particularly useful when you have transactions that don't fit your current category structure and you want the LLM to help expand your categories intelligently. diff --git a/package-lock.json b/package-lock.json index ff618bf..ed88807 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,8 @@ "node-cron": "^3.0.3", "ollama-ai-provider": "^1.2.0", "ts-node": "^10.9.2", - "typescript": "^5.7.2" + "typescript": "^5.7.2", + "zod": "^3.24.2" }, "devDependencies": { "@jest/globals": "^29.7.0", @@ -7930,10 +7931,10 @@ } }, "node_modules/zod": { - "version": "3.24.1", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.1.tgz", - "integrity": "sha512-muH7gBL9sI1nciMZV67X5fTKKBLtwpZ5VBp1vsOQzj1MhrBZ4wlVCm3gedKZWLp0Oyel8sIGfeiz54Su+OVT+A==", - "peer": true, + "version": "3.24.2", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.2.tgz", + "integrity": "sha512-lY7CDW43ECgW9u1TcT3IoXHflywfVqDYze4waEz812jR/bZ8FHDsl7pFQoSZTz5N+2NqRXs8GBwnAwo3ZNxqhQ==", + "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index bca20bf..78e8abf 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,8 @@ "node-cron": "^3.0.3", "ollama-ai-provider": "^1.2.0", "ts-node": "^10.9.2", - "typescript": "^5.7.2" + "typescript": "^5.7.2", + "zod": "^3.24.2" }, "devDependencies": { "@jest/globals": "^29.7.0", diff --git a/src/actual-api-service.ts b/src/actual-api-service.ts index 4eaafac..8c4fe72 100644 --- a/src/actual-api-service.ts +++ b/src/actual-api-service.ts @@ -123,6 +123,15 @@ class ActualApiService implements ActualApiServiceI { public async runBankSync(): Promise { await this.actualApiClient.runBankSync(); } + + public async createCategory(name: string, groupId: string): Promise { + const result = await this.actualApiClient.createCategory({ + name, + group: groupId, + }); + + return result; + } } export default ActualApiService; diff --git a/src/config.ts b/src/config.ts index b22e0eb..e2cb80a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -2,6 +2,7 @@ import dotenv from 'dotenv'; import fs from 'fs'; const defaultPromptTemplate = fs.readFileSync('./src/templates/prompt.hbs', 'utf8').trim(); +const defaultCategorySuggestionTemplate = fs.readFileSync('./src/templates/category-suggestion.hbs', 'utf8').trim(); dotenv.config(); @@ -28,6 +29,12 @@ export const dataDir = '/tmp/actual-ai/'; export const promptTemplate = process.env.PROMPT_TEMPLATE ?? defaultPromptTemplate; export const notGuessedTag = process.env.NOT_GUESSED_TAG ?? '#actual-ai-miss'; export const guessedTag = process.env.GUESSED_TAG ?? '#actual-ai'; +export const categorySuggestionTemplate = process.env.CATEGORY_SUGGESTION_TEMPLATE + ?? defaultCategorySuggestionTemplate; export const groqApiKey = process.env.GROQ_API_KEY ?? ''; export const groqModel = process.env.GROQ_MODEL ?? 'llama-3.3-70b-versatile'; export const groqBaseURL = process.env.GROQ_BASE_URL ?? 'https://api.groq.com/openai/v1'; + +// Feature Flags +export const suggestNewCategories = process.env.SUGGEST_NEW_CATEGORIES === 'true'; +export const dryRunNewCategories = process.env.DRY_RUN_NEW_CATEGORIES !== 'false'; // Default to true diff --git a/src/container.ts b/src/container.ts index af04ce3..6c30441 100644 --- a/src/container.ts +++ b/src/container.ts @@ -9,6 +9,7 @@ import { anthropicModel, budgetId, dataDir, + dryRunNewCategories, e2ePassword, googleApiKey, googleBaseURL, @@ -26,7 +27,9 @@ import { openaiModel, password, promptTemplate, + categorySuggestionTemplate, serverURL, + suggestNewCategories, syncAccountsBeforeClassify, } from './config'; import ActualAiService from './actual-ai'; @@ -65,7 +68,7 @@ const llmService = new LlmService( llmModelFactory, ); -const promptGenerator = new PromptGenerator(promptTemplate); +const promptGenerator = new PromptGenerator(promptTemplate, categorySuggestionTemplate); const transactionService = new TransactionService( actualApiService, @@ -73,6 +76,8 @@ const transactionService = new TransactionService( promptGenerator, notGuessedTag, guessedTag, + suggestNewCategories, + dryRunNewCategories, ); const actualAi = new ActualAiService( diff --git a/src/llm-model-factory.ts b/src/llm-model-factory.ts index 784d09c..40ea54b 100644 --- a/src/llm-model-factory.ts +++ b/src/llm-model-factory.ts @@ -116,6 +116,14 @@ class LlmModelFactory implements LlmModelFactoryI { public isFallbackMode(): boolean { return this.llmProvider === 'ollama'; } + + public getProvider(): string { + return this.llmProvider; + } + + public getModelProvider(): string { + return this.llmProvider; + } } export default LlmModelFactory; diff --git a/src/llm-service.ts b/src/llm-service.ts index 758c786..dc6a188 100644 --- a/src/llm-service.ts +++ b/src/llm-service.ts @@ -1,9 +1,12 @@ +import { z } from 'zod'; import { generateObject, generateText, LanguageModel } from 'ai'; import { LlmModelFactoryI, LlmServiceI } from './types'; import { RateLimiter } from './utils/rate-limiter'; import { PROVIDER_LIMITS } from './utils/provider-limits'; export default class LlmService implements LlmServiceI { + private readonly llmModelFactory: LlmModelFactoryI; + private readonly model: LanguageModel; private readonly rateLimiter: RateLimiter; @@ -15,35 +18,84 @@ export default class LlmService implements LlmServiceI { constructor( llmModelFactory: LlmModelFactoryI, ) { + this.llmModelFactory = llmModelFactory; this.model = llmModelFactory.create(); this.isFallbackMode = llmModelFactory.isFallbackMode(); this.provider = llmModelFactory.getProvider(); - this.rateLimiter = new RateLimiter(); + this.rateLimiter = new RateLimiter(true); // Set rate limits for the provider const limits = PROVIDER_LIMITS[this.provider]; if (limits) { this.rateLimiter.setProviderLimit(this.provider, limits.requestsPerMinute); + console.log(`Set ${this.provider} rate limits: ${limits.requestsPerMinute} requests/minute, ${limits.tokensPerMinute} tokens/minute`); + } else { + console.warn(`No rate limits configured for provider: ${this.provider}`); } } public async ask(prompt: string, categoryIds: string[]): Promise { try { + console.log(`Making LLM request to ${this.provider}${this.isFallbackMode ? ' (fallback mode)' : ''}`); + if (this.isFallbackMode) { return await this.askUsingFallbackModel(prompt); } return await this.askWithEnum(prompt, categoryIds); } catch (error) { - console.error(`Error during LLM request: ${error instanceof Error ? error.message : String(error)}`); + const errorMsg = error instanceof Error ? error.message : String(error); + console.error(`Error during LLM request to ${this.provider}: ${errorMsg}`); throw error; } } + public async askForCategorySuggestion( + prompt: string, + ): Promise<{ name: string, groupId: string } | null> { + try { + console.log( + `Making LLM request for category suggestion to ${this.provider}${this.isFallbackMode ? ' (fallback mode)' : ''}`, + ); + + const response = await this.rateLimiter.executeWithRateLimiting( + this.provider, + async () => { + const result = await generateObject({ + model: this.model, + prompt, + temperature: 0.2, + output: 'object', + schema: z.object({ + name: z.string(), + groupId: z.string(), + }), + mode: 'json', + }); + return result.object; + }, + ); + + if (response && typeof response === 'object' && 'name' in response && 'groupId' in response) { + return { + name: String(response.name), + groupId: String(response.groupId), + }; + } + + console.warn('LLM response did not contain valid category suggestion format:', response); + return null; + } catch (error) { + console.error('Error while getting category suggestion:', error); + return null; + } + } + public async askWithEnum(prompt: string, categoryIds: string[]): Promise { return this.rateLimiter.executeWithRateLimiting( this.provider, async () => { + console.log(`Sending enum request to ${this.provider} with ${categoryIds.length} options`); const { object } = await generateObject({ model: this.model, output: 'enum', @@ -61,6 +113,7 @@ export default class LlmService implements LlmServiceI { return this.rateLimiter.executeWithRateLimiting( this.provider, async () => { + console.log(`Sending text generation request to ${this.provider}`); const { text } = await generateText({ model: this.model, prompt, diff --git a/src/prompt-generator.ts b/src/prompt-generator.ts index d6f1f14..d415fae 100644 --- a/src/prompt-generator.ts +++ b/src/prompt-generator.ts @@ -7,8 +7,11 @@ import PromptTemplateException from './exceptions/prompt-template-exception'; class PromptGenerator implements PromptGeneratorI { private readonly promptTemplate: string; - constructor(promptTemplate: string) { + private readonly categorySuggestionTemplate: string; + + constructor(promptTemplate: string, categorySuggestionTemplate: string) { this.promptTemplate = promptTemplate; + this.categorySuggestionTemplate = categorySuggestionTemplate; } generate( @@ -42,6 +45,39 @@ class PromptGenerator implements PromptGeneratorI { throw new PromptTemplateException('Error generating prompt. Check syntax of your template.'); } } + + generateCategorySuggestion( + categoryGroups: APICategoryGroupEntity[], + transaction: TransactionEntity, + payees: APIPayeeEntity[], + ): string { + let template; + try { + template = handlebars.compile(this.categorySuggestionTemplate); + } catch (error) { + console.error('Error generating category suggestion prompt.', error); + throw new PromptTemplateException('Error generating category suggestion prompt.'); + } + + const payeeName = payees.find((payee) => payee.id === transaction.payee)?.name; + + try { + return template({ + categoryGroups, + amount: Math.abs(transaction.amount), + type: transaction.amount > 0 ? 'Income' : 'Outcome', + description: transaction.notes, + payee: payeeName, + importedPayee: transaction.imported_payee, + date: transaction.date, + cleared: transaction.cleared, + reconciled: transaction.reconciled, + }); + } catch (error) { + console.error('Error generating category suggestion prompt.', error); + throw new PromptTemplateException('Error generating category suggestion prompt.'); + } + } } export default PromptGenerator; diff --git a/src/templates/category-suggestion.hbs b/src/templates/category-suggestion.hbs new file mode 100644 index 0000000..25ffce5 --- /dev/null +++ b/src/templates/category-suggestion.hbs @@ -0,0 +1,27 @@ +I need to suggest a new category for a transaction that doesn't fit any existing categories. + +Transaction details: +* Amount: {{amount}} +* Type: {{type}} +{{#if description}} +* Description: {{description}} +{{/if}} +{{#if payee}} +* Payee: {{payee}} +{{^}} +* Payee: {{importedPayee}} +{{/if}} + +Available category groups: +{{#each categoryGroups}} +* {{name}} (ID: "{{id}}") +{{/each}} + +RESPOND WITH A JSON OBJECT that suggests a new category with these properties: +1. "name": A short, descriptive name for the new category +2. "groupId": The ID of an existing category group this new category should belong to + +Example response: +{"name": "Online Subscriptions", "groupId": "group-id-here"} + +Choose a specific, not overly general name. Use existing category naming patterns. Select an appropriate existing group ID. \ No newline at end of file diff --git a/src/templates/prompt.hbs b/src/templates/prompt.hbs index ac1bbeb..762a299 100644 --- a/src/templates/prompt.hbs +++ b/src/templates/prompt.hbs @@ -1,10 +1,12 @@ -I want to categorize the given bank transactions into the following categories: +I want to categorize the given bank transaction into one of the following categories: {{#each categoryGroups}} +GROUP: {{name}} (ID: "{{id}}") {{#each categories}} -* {{name}} ({{../name}}) (ID: "{{id}}") +* {{name}} (ID: "{{id}}") {{/each}} {{/each}} -Please categorize the following transaction: + +Transaction details: * Amount: {{amount}} * Type: {{type}} {{#if description}} @@ -15,4 +17,6 @@ Please categorize the following transaction: {{^}} * Payee: {{importedPayee}} {{/if}} -ANSWER BY A CATEGORY ID - DO NOT CREATE ENTIRE SENTENCE - DO NOT WRITE CATEGORY NAME, JUST AN ID. Do not guess, if you don't know the answer, return "uncategorized". + +RESPOND ONLY WITH A CATEGORY ID from the list above. Do not write anything else. +If you're not sure which category to use, respond with "uncategorized". diff --git a/src/transaction-service.ts b/src/transaction-service.ts index bf5aad2..3d0af17 100644 --- a/src/transaction-service.ts +++ b/src/transaction-service.ts @@ -17,18 +17,26 @@ class TransactionService implements TransactionServiceI { private readonly guessedTag: string; + private readonly suggestNewCategories: boolean; + + private readonly dryRunNewCategories: boolean; + constructor( actualApiClient: ActualApiServiceI, llmService: LlmServiceI, promptGenerator: PromptGeneratorI, notGuessedTag: string, guessedTag: string, + suggestNewCategories = false, + dryRunNewCategories = true, ) { this.actualApiService = actualApiClient; this.llmService = llmService; this.promptGenerator = promptGenerator; this.notGuessedTag = notGuessedTag; this.guessedTag = guessedTag; + this.suggestNewCategories = suggestNewCategories; + this.dryRunNewCategories = dryRunNewCategories; } appendTag(notes: string, tag: string): string { @@ -96,10 +104,17 @@ class TransactionService implements TransactionServiceI { const categoryIds = categories.map((category) => category.id); categoryIds.push('uncategorized'); + // Track suggested categories to avoid duplicates and for creating later + const suggestedCategories = new Map(); + // Process transactions in batches to avoid hitting rate limits for ( - let batchStart = 0; - batchStart < uncategorizedTransactions.length; + let batchStart = 0; + batchStart < uncategorizedTransactions.length; batchStart += BATCH_SIZE ) { const batchEnd = Math.min(batchStart + BATCH_SIZE, uncategorizedTransactions.length); @@ -132,7 +147,61 @@ class TransactionService implements TransactionServiceI { if (!guessCategory) { console.warn(`${globalIndex + 1}/${uncategorizedTransactions.length} LLM could not classify the transaction. LLM guess: ${guess}`); - await this.actualApiService.updateTransactionNotes(transaction.id, this.appendTag(transaction.notes ?? '', this.notGuessedTag)); + + // If suggestNewCategories is enabled, try to get a new category suggestion + if (this.suggestNewCategories) { + const newCategoryPrompt = this.promptGenerator.generateCategorySuggestion( + categoryGroups, + transaction, + payees, + ); + const categorySuggestion = await this.llmService.askForCategorySuggestion( + newCategoryPrompt, + ); + + if (categorySuggestion?.name && categorySuggestion.groupId) { + console.log(`${globalIndex + 1}/${uncategorizedTransactions.length} Suggested new category: ${categorySuggestion.name} in group ${categorySuggestion.groupId}`); + + // Check if this category name already exists + const existingCategory = categories.find( + (c) => c.name && c.name.toLowerCase() === categorySuggestion.name.toLowerCase(), + ); + + if (existingCategory) { + console.log(`${globalIndex + 1}/${uncategorizedTransactions.length} Category with similar name already exists: ${existingCategory.name}`); + + // Use existing category instead + await this.actualApiService.updateTransactionNotesAndCategory( + transaction.id, + this.appendTag(transaction.notes ?? '', this.guessedTag), + existingCategory.id, + ); + console.log(`${globalIndex + 1}/${uncategorizedTransactions.length} Used existing category: ${existingCategory.name}`); + } else { + // Add to suggested categories map + const key = `${categorySuggestion.name.toLowerCase()}-${categorySuggestion.groupId}`; + if (suggestedCategories.has(key)) { + suggestedCategories.get(key)?.transactions.push(transaction.id); + } else { + suggestedCategories.set(key, { + name: categorySuggestion.name, + groupId: categorySuggestion.groupId, + transactions: [transaction.id], + }); + } + + // In dry run mode, just mark with notGuessedTag + await this.actualApiService.updateTransactionNotes( + transaction.id, + this.appendTag(transaction.notes ?? '', `${this.notGuessedTag} (Suggested: ${categorySuggestion.name})`), + ); + } + } else { + await this.actualApiService.updateTransactionNotes(transaction.id, this.appendTag(transaction.notes ?? '', this.notGuessedTag)); + } + } else { + await this.actualApiService.updateTransactionNotes(transaction.id, this.appendTag(transaction.notes ?? '', this.notGuessedTag)); + } continue; } console.log(`${globalIndex + 1}/${uncategorizedTransactions.length} Guess: ${guessCategory.name}`); @@ -156,6 +225,56 @@ class TransactionService implements TransactionServiceI { }); } } + + // Create new categories if not in dry run mode + if (this.suggestNewCategories && !this.dryRunNewCategories && suggestedCategories.size > 0) { + console.log(`Creating ${suggestedCategories.size} new categories`); + + // Use Promise.all with map for async operations + await Promise.all( + Array.from(suggestedCategories.entries()).map(async ([_, suggestion]) => { + try { + const newCategoryId = await this.actualApiService.createCategory( + suggestion.name, + suggestion.groupId, + ); + + console.log(`Created new category "${suggestion.name}" with ID ${newCategoryId}`); + + // Use Promise.all with map for nested async operations + await Promise.all( + suggestion.transactions.map(async (transactionId) => { + const transaction = uncategorizedTransactions.find((t) => t.id === transactionId); + if (transaction) { + await this.actualApiService.updateTransactionNotesAndCategory( + transactionId, + this.appendTag(transaction.notes ?? '', this.guessedTag), + newCategoryId, + ); + console.log(`Assigned transaction ${transactionId} to new category ${suggestion.name}`); + } + }), + ); + } catch (error) { + console.error(`Error creating category ${suggestion.name}:`, error); + } + }), + ); + } else if ( + this.suggestNewCategories && this.dryRunNewCategories && suggestedCategories.size > 0 + ) { + // Split the longer line to avoid length error + console.log( + `Dry run: Would create ${suggestedCategories.size} new categories:`, + ); + + // No need for async here, so we can use forEach + Array.from(suggestedCategories.entries()).forEach(([_, suggestion]) => { + console.log( + `- ${suggestion.name} (in group ${suggestion.groupId}) for ${suggestion.transactions.length} transactions`, + ); + }); + } } } diff --git a/src/types.ts b/src/types.ts index 4c003e0..b3e58bd 100644 --- a/src/types.ts +++ b/src/types.ts @@ -7,10 +7,16 @@ import { } from '@actual-app/api/@types/loot-core/server/api-models'; import { TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; +export interface LlmModelI { + ask(prompt: string, possibleAnswers: string[]): Promise; + askFreeform(prompt: string): Promise; +} + export interface LlmModelFactoryI { create(): LanguageModel; - + getProvider(): string; isFallbackMode(): boolean; + getModelProvider(): string; } export interface ActualApiServiceI { @@ -37,6 +43,8 @@ export interface ActualApiServiceI { ): Promise runBankSync(): Promise + + createCategory(name: string, groupId: string): Promise } export interface TransactionServiceI { @@ -47,10 +55,16 @@ export interface TransactionServiceI { export interface ActualAiServiceI { classify(): Promise; + + syncAccounts(): Promise } export interface LlmServiceI { - ask(prompt: string, categoryIds: string[]): Promise; + ask(prompt: string, possibleAnswers: string[]): Promise; + + askForCategorySuggestion( + prompt: string + ): Promise<{ name: string, groupId: string } | null> } export interface PromptGeneratorI { @@ -59,4 +73,10 @@ export interface PromptGeneratorI { transaction: TransactionEntity, payees: APIPayeeEntity[], ): string + + generateCategorySuggestion( + categoryGroups: APICategoryGroupEntity[], + transaction: TransactionEntity, + payees: APIPayeeEntity[], + ): string } From c42d7f5d187ddb37ad5908fdcf63ca6a43ebacef Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sat, 1 Mar 2025 17:15:35 -0500 Subject: [PATCH 03/17] Implement web search tool and improve category suggestion workflow --- .env.example | 4 + README.md | 30 ++++++ package-lock.json | 115 +++++++++++++++++++--- package.json | 1 + src/actual-api-service.ts | 10 ++ src/config.ts | 5 + src/container.ts | 9 ++ src/llm-model-factory.ts | 2 +- src/llm-service.ts | 76 ++++++++++++--- src/prompt-generator.ts | 20 +++- src/templates/category-suggestion.hbs | 34 ++++++- src/transaction-service.ts | 92 +++++++++++------- src/types.ts | 12 ++- src/utils/rate-limiter.ts | 1 - src/utils/tool-service.ts | 134 ++++++++++++++++++++++++++ 15 files changed, 472 insertions(+), 73 deletions(-) create mode 100644 src/utils/tool-service.ts diff --git a/.env.example b/.env.example index c33b9fb..16e701f 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,10 @@ ACTUAL_BUDGET_ID= CLASSIFICATION_SCHEDULE_CRON="0 */4 * * *" CLASSIFY_ON_STARTUP=true SYNC_ACCOUNTS_BEFORE_CLASSIFY=true +SUGGEST_NEW_CATEGORIES=false +DRY_RUN_NEW_CATEGORIES=false +ENABLED_TOOLS=webSearch +VALUESERP_API_KEY= LLM_PROVIDER=openai OPENAI_API_KEY= OPENAI_MODEL=gpt-4o-mini diff --git a/README.md b/README.md index 3ebd993..714f89b 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,10 @@ The app sends requests to the LLM to classify transactions based on their descri When enabled, the LLM can suggest entirely new categories for transactions it cannot classify, and optionally create them automatically. +#### 🌐 Web search for unfamiliar merchants + +Using the ValueSerp API, the system can search the web for information about unfamiliar merchants to help the LLM make better categorization decisions. + ## 🚀 Usage Sample `docker-compose.yml` file: @@ -62,6 +66,8 @@ services: LLM_PROVIDER: openai # Can be "openai", "anthropic", "google-generative-ai", "ollama" or "groq" # SUGGEST_NEW_CATEGORIES: false # Whether to suggest new categories for transactions that can't be classified with existing ones # DRY_RUN_NEW_CATEGORIES: true # When true, just logs suggested categories without creating them +# ENABLED_TOOLS: webSearch # Comma-separated list of tools to enable +# VALUESERP_API_KEY: your_valueserp_api_key # API key for ValueSerp, required if webSearch tool is enabled # OPENAI_API_KEY: # optional. required if you want to use the OpenAI API # OPENAI_MODEL: # optional. required if you want to use a specific model, default is "gpt-4o-mini" # OPENAI_BASE_URL: # optional. required if you don't want to use the OpenAI API but OpenAI compatible API, ex: "http://ollama:11424/v1 @@ -138,3 +144,27 @@ When `SUGGEST_NEW_CATEGORIES` is enabled, the system will: 5. If not in dry run mode (`DRY_RUN_NEW_CATEGORIES=false`), create the new categories and assign transactions to them This feature is particularly useful when you have transactions that don't fit your current category structure and you want the LLM to help expand your categories intelligently. + +## Tools Integration + +The system supports various tools that can be enabled to enhance the LLM's capabilities: + +1. Set `ENABLED_TOOLS` in your environment variables as a comma-separated list of tools to enable +2. Provide any required API keys for the tools you want to use + +Currently supported tools: + +### webSearch + +The webSearch tool uses the ValueSerp API to search for information about merchants that the LLM might not be familiar with, providing additional context for categorization decisions. + +To use this tool: +1. Include `webSearch` in your `ENABLED_TOOLS` list +2. Provide your ValueSerp API key as `VALUESERP_API_KEY` + +This is especially helpful for: +- New or uncommon merchants +- Merchants with ambiguous names +- Specialized services that might be difficult to categorize without additional information + +The search results are included in the prompts sent to the LLM, helping it make more accurate category assignments or suggestions. diff --git a/package-lock.json b/package-lock.json index ed88807..7b74944 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "@ai-sdk/openai": "^1.1.9", "@types/jest": "^29.5.14", "ai": "^4.1.28", + "axios": "^1.8.1", "dotenv": "^16.4.7", "handlebars": "^4.7.8", "node-cron": "^3.0.3", @@ -2233,6 +2234,12 @@ "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", "dev": true }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, "node_modules/available-typed-arrays": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", @@ -2248,6 +2255,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/axios": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.1.tgz", + "integrity": "sha512-NN+fvwH/kV01dYUQ3PTOZns4LWtWhOFCAhQ/pHb88WQ1hNe5V/dvFwc4VJcDL11LT9xSX0QtsR8sWUuyOuOq7g==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/babel-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", @@ -2556,7 +2574,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.1.tgz", "integrity": "sha512-BhYE+WDaywFg2TBWYNXAE+8B1ATnThNBqXHP5nQu0jWJdVvY2hvkpyB3qOmtmDePiS5/BDQ8wASEWGMWRG148g==", - "dev": true, "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" @@ -2750,6 +2767,18 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/compare-versions": { "version": "6.1.1", "resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-6.1.1.tgz", @@ -2974,6 +3003,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/dequal": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", @@ -3048,7 +3086,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "dev": true, "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", @@ -3183,7 +3220,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "dev": true, "engines": { "node": ">= 0.4" } @@ -3192,7 +3228,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, "engines": { "node": ">= 0.4" } @@ -3201,7 +3236,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.0.0.tgz", "integrity": "sha512-MZ4iQ6JwHOBQjahnjwaC1ZtIBH+2ohjamzAO3oaHcXYup7qxjF2fixyH+Q71voWHeOkI2q/TnJao/KfXYIZWbw==", - "dev": true, "dependencies": { "es-errors": "^1.3.0" }, @@ -3213,7 +3247,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "dev": true, "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", @@ -3842,6 +3875,26 @@ "integrity": "sha512-AiwGJM8YcNOaobumgtng+6NHuOqC3A7MixFeDafM3X9cIUM+xUXoS5Vfgf+OihAYe20fxqNM9yPBXJzRtZ/4eA==", "dev": true }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -3851,6 +3904,21 @@ "is-callable": "^1.1.3" } }, + "node_modules/form-data": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz", + "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -3891,7 +3959,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -3947,7 +4014,6 @@ "version": "1.2.7", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.7.tgz", "integrity": "sha512-VW6Pxhsrk0KAOqs3WEd0klDiF/+V7gQOpAvY1jVU/LHmaD/kQO4523aiJuikX/QAKYiW6x8Jh+RJej1almdtCA==", - "dev": true, "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-define-property": "^1.0.1", @@ -3980,7 +4046,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "dev": true, "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" @@ -4118,7 +4183,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -4208,7 +4272,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -4220,7 +4283,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "dev": true, "dependencies": { "has-symbols": "^1.0.3" }, @@ -4235,7 +4297,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "dev": true, "dependencies": { "function-bind": "^1.1.2" }, @@ -5660,7 +5721,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "dev": true, "engines": { "node": ">= 0.4" } @@ -5692,6 +5752,27 @@ "node": ">=8.6" } }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", @@ -6412,6 +6493,12 @@ "node": ">= 6" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, "node_modules/pstree.remy": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", diff --git a/package.json b/package.json index 78e8abf..0a7797d 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "@ai-sdk/openai": "^1.1.9", "@types/jest": "^29.5.14", "ai": "^4.1.28", + "axios": "^1.8.1", "dotenv": "^16.4.7", "handlebars": "^4.7.8", "node-cron": "^3.0.3", diff --git a/src/actual-api-service.ts b/src/actual-api-service.ts index 8c4fe72..6a52434 100644 --- a/src/actual-api-service.ts +++ b/src/actual-api-service.ts @@ -132,6 +132,16 @@ class ActualApiService implements ActualApiServiceI { return result; } + + public async createCategoryGroup(name: string): Promise { + return this.actualApiClient.createCategoryGroup({ + name, + }); + } + + public async updateCategoryGroup(id: string, name: string): Promise { + await this.actualApiClient.updateCategoryGroup(id, { name }); + } } export default ActualApiService; diff --git a/src/config.ts b/src/config.ts index e2cb80a..e099456 100644 --- a/src/config.ts +++ b/src/config.ts @@ -34,7 +34,12 @@ export const categorySuggestionTemplate = process.env.CATEGORY_SUGGESTION_TEMPLA export const groqApiKey = process.env.GROQ_API_KEY ?? ''; export const groqModel = process.env.GROQ_MODEL ?? 'llama-3.3-70b-versatile'; export const groqBaseURL = process.env.GROQ_BASE_URL ?? 'https://api.groq.com/openai/v1'; +export const valueSerpApiKey = process.env.VALUESERP_API_KEY ?? ''; // Feature Flags export const suggestNewCategories = process.env.SUGGEST_NEW_CATEGORIES === 'true'; export const dryRunNewCategories = process.env.DRY_RUN_NEW_CATEGORIES !== 'false'; // Default to true + +// Tools configuration +export const enabledTools = (process.env.ENABLED_TOOLS ?? '').split(',').map((tool) => tool.trim()).filter(Boolean); +export const hasWebSearchTool = enabledTools.includes('webSearch'); diff --git a/src/container.ts b/src/container.ts index 6c30441..da51a95 100644 --- a/src/container.ts +++ b/src/container.ts @@ -31,10 +31,18 @@ import { serverURL, suggestNewCategories, syncAccountsBeforeClassify, + valueSerpApiKey, + enabledTools, } from './config'; import ActualAiService from './actual-ai'; import PromptGenerator from './prompt-generator'; import LlmService from './llm-service'; +import ToolService from './utils/tool-service'; + +// Create tool service if API key is available and tools are enabled +const toolService = valueSerpApiKey && enabledTools.length > 0 + ? new ToolService(valueSerpApiKey) + : undefined; const llmModelFactory = new LlmModelFactory( llmProvider, @@ -66,6 +74,7 @@ const actualApiService = new ActualApiService( const llmService = new LlmService( llmModelFactory, + toolService, ); const promptGenerator = new PromptGenerator(promptTemplate, categorySuggestionTemplate); diff --git a/src/llm-model-factory.ts b/src/llm-model-factory.ts index 40ea54b..3a91d99 100644 --- a/src/llm-model-factory.ts +++ b/src/llm-model-factory.ts @@ -106,7 +106,7 @@ class LlmModelFactory implements LlmModelFactoryI { baseURL: this.groqBaseURL, apiKey: this.groqApiKey, }); - return groq(this.groqModel) as LanguageModel; + return groq(this.groqModel); } default: throw new Error(`Unknown provider: ${this.llmProvider}`); diff --git a/src/llm-service.ts b/src/llm-service.ts index dc6a188..f203ea4 100644 --- a/src/llm-service.ts +++ b/src/llm-service.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; import { generateObject, generateText, LanguageModel } from 'ai'; -import { LlmModelFactoryI, LlmServiceI } from './types'; +import { LlmModelFactoryI, LlmServiceI, ToolServiceI } from './types'; import { RateLimiter } from './utils/rate-limiter'; import { PROVIDER_LIMITS } from './utils/provider-limits'; @@ -13,16 +13,20 @@ export default class LlmService implements LlmServiceI { private readonly provider: string; + private readonly toolService?: ToolServiceI; + private isFallbackMode; constructor( llmModelFactory: LlmModelFactoryI, + toolService?: ToolServiceI, ) { this.llmModelFactory = llmModelFactory; this.model = llmModelFactory.create(); this.isFallbackMode = llmModelFactory.isFallbackMode(); this.provider = llmModelFactory.getProvider(); this.rateLimiter = new RateLimiter(true); + this.toolService = toolService; // Set rate limits for the provider const limits = PROVIDER_LIMITS[this.provider]; @@ -34,6 +38,25 @@ export default class LlmService implements LlmServiceI { } } + public async searchWeb(query: string): Promise { + if (!this.toolService) { + return 'Search functionality is not available.'; + } + + try { + console.log(`Performing web search for: "${query}"`); + if ('search' in this.toolService) { + type SearchFunction = (q: string) => Promise; + const searchFn = this.toolService.search as SearchFunction; + return await searchFn(query); + } + return 'Search tool is not available.'; + } catch (error) { + console.error('Error during web search:', error); + return `Error performing search: ${error instanceof Error ? error.message : String(error)}`; + } + } + public async ask(prompt: string, categoryIds: string[]): Promise { try { console.log(`Making LLM request to ${this.provider}${this.isFallbackMode ? ' (fallback mode)' : ''}`); @@ -52,38 +75,61 @@ export default class LlmService implements LlmServiceI { public async askForCategorySuggestion( prompt: string, - ): Promise<{ name: string, groupId: string } | null> { + ): Promise<{ name: string, groupName: string, groupIsNew: boolean } | null> { try { console.log( `Making LLM request for category suggestion to ${this.provider}${this.isFallbackMode ? ' (fallback mode)' : ''}`, ); - const response = await this.rateLimiter.executeWithRateLimiting( + const categorySchema = z.object({ + name: z.string(), + groupName: z.string(), + groupIsNew: z.boolean(), + }); + + type CategorySuggestion = z.infer; + + const response = await this.rateLimiter.executeWithRateLimiting( this.provider, async () => { - const result = await generateObject({ + const { text, steps } = await generateText({ model: this.model, prompt, temperature: 0.2, - output: 'object', - schema: z.object({ - name: z.string(), - groupId: z.string(), - }), - mode: 'json', + tools: this.toolService?.getTools(), + maxSteps: 3, + system: 'You must use webSearch for unfamiliar payees before suggesting categories', }); - return result.object; + + // Add this debug logging + console.log('Generation steps:', steps.map((step) => ({ + text: step.text, + toolCalls: step.toolCalls, + toolResults: step.toolResults, + }))); + + // Parse the JSON response from the text + try { + const parsedResponse = JSON.parse(text) as unknown; + // Validate against schema + const result = categorySchema.safeParse(parsedResponse); + return result.success ? result.data : null; + } catch (e) { + console.error('Failed to parse JSON response:', e); + return null; + } }, ); - if (response && typeof response === 'object' && 'name' in response && 'groupId' in response) { + if (response) { return { - name: String(response.name), - groupId: String(response.groupId), + name: response.name, + groupName: response.groupName, + groupIsNew: response.groupIsNew, }; } - console.warn('LLM response did not contain valid category suggestion format:', response); + console.warn('LLM response did not contain valid category suggestion format'); return null; } catch (error) { console.error('Error while getting category suggestion:', error); diff --git a/src/prompt-generator.ts b/src/prompt-generator.ts index d415fae..e703e12 100644 --- a/src/prompt-generator.ts +++ b/src/prompt-generator.ts @@ -3,6 +3,7 @@ import { TransactionEntity } from '@actual-app/api/@types/loot-core/types/models import * as handlebars from 'handlebars'; import { PromptGeneratorI } from './types'; import PromptTemplateException from './exceptions/prompt-template-exception'; +import { hasWebSearchTool } from './config'; class PromptGenerator implements PromptGeneratorI { private readonly promptTemplate: string; @@ -28,9 +29,16 @@ class PromptGenerator implements PromptGeneratorI { } const payeeName = payees.find((payee) => payee.id === transaction.payee)?.name; + // Ensure each category group has its categories property + const groupsWithCategories = categoryGroups.map((group) => ({ + ...group, + groupName: group.name, + categories: group.categories || [], + })); + try { return template({ - categoryGroups, + categoryGroups: groupsWithCategories, amount: Math.abs(transaction.amount), type: transaction.amount > 0 ? 'Income' : 'Outcome', description: transaction.notes, @@ -61,9 +69,16 @@ class PromptGenerator implements PromptGeneratorI { const payeeName = payees.find((payee) => payee.id === transaction.payee)?.name; + // Ensure each category group has its categories property + const groupsWithCategories = categoryGroups.map((group) => ({ + ...group, + groupName: group.name, + categories: group.categories || [], + })); + try { return template({ - categoryGroups, + categoryGroups: groupsWithCategories, amount: Math.abs(transaction.amount), type: transaction.amount > 0 ? 'Income' : 'Outcome', description: transaction.notes, @@ -72,6 +87,7 @@ class PromptGenerator implements PromptGeneratorI { date: transaction.date, cleared: transaction.cleared, reconciled: transaction.reconciled, + hasWebSearchTool, }); } catch (error) { console.error('Error generating category suggestion prompt.', error); diff --git a/src/templates/category-suggestion.hbs b/src/templates/category-suggestion.hbs index 25ffce5..f480c10 100644 --- a/src/templates/category-suggestion.hbs +++ b/src/templates/category-suggestion.hbs @@ -1,3 +1,9 @@ +{{#if hasWebSearchTool}} +When suggesting a new category, you MUST use the webSearch tool to research +the business type and common categorizations for this transaction's payee. +Only suggest a category after reviewing the search results. +{{/if}} + I need to suggest a new category for a transaction that doesn't fit any existing categories. Transaction details: @@ -12,16 +18,34 @@ Transaction details: * Payee: {{importedPayee}} {{/if}} -Available category groups: +Existing categories by group: {{#each categoryGroups}} -* {{name}} (ID: "{{id}}") +GROUP: {{name}} (ID: "{{id}}") +{{#each categories}} +* {{name}} +{{/each}} {{/each}} RESPOND WITH A JSON OBJECT that suggests a new category with these properties: 1. "name": A short, descriptive name for the new category -2. "groupId": The ID of an existing category group this new category should belong to +2. "groupName": The name of an existing OR NEW category group this should belong to +3. "groupIsNew": Boolean indicating if this group needs to be created Example response: -{"name": "Online Subscriptions", "groupId": "group-id-here"} +{"name": "Membership fees", "groupName": "Subscriptions", "groupIsNew": true} + +IMPORTANT: Create a specific category that: +- Is appropriate for the transaction details shown above +- Does NOT duplicate or closely resemble any existing category names +- Is specific rather than overly general +- Follows the naming patterns of other categories in the same group + +IMPORTANT: Your response MUST be: +- A SINGLE VALID JSON OBJECT +- No additional text or explanation +- No markdown formatting +- Properly escaped characters +- No trailing commas -Choose a specific, not overly general name. Use existing category naming patterns. Select an appropriate existing group ID. \ No newline at end of file +Example of VALID response: +{"name": "Pet Supplies", "groupName": "Digital Assets", "groupIsNew": true} \ No newline at end of file diff --git a/src/transaction-service.ts b/src/transaction-service.ts index 3d0af17..3841c68 100644 --- a/src/transaction-service.ts +++ b/src/transaction-service.ts @@ -159,45 +159,71 @@ class TransactionService implements TransactionServiceI { newCategoryPrompt, ); - if (categorySuggestion?.name && categorySuggestion.groupId) { - console.log(`${globalIndex + 1}/${uncategorizedTransactions.length} Suggested new category: ${categorySuggestion.name} in group ${categorySuggestion.groupId}`); - - // Check if this category name already exists - const existingCategory = categories.find( - (c) => c.name && c.name.toLowerCase() === categorySuggestion.name.toLowerCase(), - ); - - if (existingCategory) { - console.log(`${globalIndex + 1}/${uncategorizedTransactions.length} Category with similar name already exists: ${existingCategory.name}`); - - // Use existing category instead - await this.actualApiService.updateTransactionNotesAndCategory( - transaction.id, - this.appendTag(transaction.notes ?? '', this.guessedTag), - existingCategory.id, - ); - console.log(`${globalIndex + 1}/${uncategorizedTransactions.length} Used existing category: ${existingCategory.name}`); - } else { - // Add to suggested categories map - const key = `${categorySuggestion.name.toLowerCase()}-${categorySuggestion.groupId}`; - if (suggestedCategories.has(key)) { - suggestedCategories.get(key)?.transactions.push(transaction.id); + if ( + categorySuggestion?.name + && categorySuggestion.groupName + ) { + console.log(`${globalIndex + 1}/${uncategorizedTransactions.length} Suggested new category: ${categorySuggestion.name} in group ${categorySuggestion.groupName}`); + + // Find or create category group + let groupId: string; + if (categorySuggestion.groupIsNew) { + if (this.dryRunNewCategories) { + console.log(`Dry run: Would create new category group "${categorySuggestion.groupName}"`); + groupId = 'dry-run-group-id'; } else { - suggestedCategories.set(key, { - name: categorySuggestion.name, - groupId: categorySuggestion.groupId, - transactions: [transaction.id], - }); + groupId = await this.actualApiService.createCategoryGroup( + categorySuggestion.groupName, + ); + console.log(`Created new category group "${categorySuggestion.groupName}" with ID ${groupId}`); } + } else { + const existingGroup = categoryGroups.find( + (g) => g.name.toLowerCase() === categorySuggestion.groupName.toLowerCase(), + ); + groupId = existingGroup?.id + ?? (this.dryRunNewCategories ? 'dry-run-group-id' : await this.actualApiService.createCategoryGroup( + categorySuggestion.groupName, + )); + } - // In dry run mode, just mark with notGuessedTag - await this.actualApiService.updateTransactionNotes( - transaction.id, - this.appendTag(transaction.notes ?? '', `${this.notGuessedTag} (Suggested: ${categorySuggestion.name})`), + // Then create category and assign transactions... + let newCategoryId: string | null = null; + if (!this.dryRunNewCategories) { + newCategoryId = await this.actualApiService.createCategory( + categorySuggestion.name, + groupId, + ); + console.log(`Created new category "${categorySuggestion.name}" with ID ${newCategoryId}`); + } + + // Handle transaction assignments + if (newCategoryId) { + await Promise.all( + suggestedCategories.get( + categorySuggestion.name.toLowerCase(), + )?.transactions.map(async (transactionId) => { + const uncategorizedTransaction = uncategorizedTransactions.find( + (t) => t.id === transactionId, + ); + if (uncategorizedTransaction) { + await this.actualApiService.updateTransactionNotesAndCategory( + uncategorizedTransaction.id, + this.appendTag(uncategorizedTransaction.notes ?? '', this.guessedTag), + newCategoryId, + ); + console.log(`Assigned transaction ${uncategorizedTransaction.id} to new category ${categorySuggestion?.name}`); + } + }) ?? [], ); } } else { - await this.actualApiService.updateTransactionNotes(transaction.id, this.appendTag(transaction.notes ?? '', this.notGuessedTag)); + // Handle invalid/missing category suggestion + console.log('No valid category suggestion received'); + await this.actualApiService.updateTransactionNotes( + transaction.id, + this.appendTag(transaction.notes ?? '', this.notGuessedTag), + ); } } else { await this.actualApiService.updateTransactionNotes(transaction.id, this.appendTag(transaction.notes ?? '', this.notGuessedTag)); diff --git a/src/types.ts b/src/types.ts index b3e58bd..56c8030 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,4 @@ -import { LanguageModel } from 'ai'; +import { LanguageModel, Tool } from 'ai'; import { APIAccountEntity, APICategoryEntity, @@ -45,6 +45,10 @@ export interface ActualApiServiceI { runBankSync(): Promise createCategory(name: string, groupId: string): Promise + + createCategoryGroup(name: string): Promise + + updateCategoryGroup(id: string, name: string): Promise } export interface TransactionServiceI { @@ -64,7 +68,11 @@ export interface LlmServiceI { askForCategorySuggestion( prompt: string - ): Promise<{ name: string, groupId: string } | null> + ): Promise<{ name: string, groupName: string, groupIsNew: boolean } | null> +} + +export interface ToolServiceI { + getTools(): Record; } export interface PromptGeneratorI { diff --git a/src/utils/rate-limiter.ts b/src/utils/rate-limiter.ts index 740bf2e..5ae0006 100644 --- a/src/utils/rate-limiter.ts +++ b/src/utils/rate-limiter.ts @@ -71,7 +71,6 @@ export class RateLimiter { lastError = error as Error; if (this.isRateLimitError(error)) { - // Update token bucket information if available this.updateTokenBucketFromError(provider, error); // Get retry delay from error or calculate backoff diff --git a/src/utils/tool-service.ts b/src/utils/tool-service.ts new file mode 100644 index 0000000..babe4e0 --- /dev/null +++ b/src/utils/tool-service.ts @@ -0,0 +1,134 @@ +import * as https from 'https'; +import { z } from 'zod'; +import { tool, Tool } from 'ai'; +import { ToolServiceI } from '../types'; +import { enabledTools } from '../config'; + +interface SearchResult { + title: string; + snippet: string; + link: string; +} + +interface OrganicResults { + organic_results?: SearchResult[]; +} + +export default class ToolService implements ToolServiceI { + private readonly valueSerpApiKey: string; + + constructor(valueSerpApiKey: string) { + this.valueSerpApiKey = valueSerpApiKey; + } + + public getTools() { + const tools: Record = {}; + + if (enabledTools.includes('webSearch')) { + tools.webSearch = tool({ + description: 'Essential for researching business types and industry categorizations when existing categories are insufficient. Use when payee is unfamiliar or category context is unclear', + parameters: z.object({ + query: z.string().describe( + 'Combination of payee name and business type with search operators. ' + + 'Example: "StudntLN" (merchant|business|company|payee)', + ), + }), + execute: async ({ query }: { query: string }): Promise => { + if (!this.valueSerpApiKey) return 'Search unavailable'; + const results = await this.performSearch(query); + return this.formatSearchResults(results); + }, + }); + } + + return tools; + } + + private async performSearch(query: string): Promise { + const params = new URLSearchParams({ + api_key: this.valueSerpApiKey, + q: query, + gl: 'us', + hl: 'en', + num: '5', + output: 'json', + }); + + return new Promise((resolve, reject) => { + const options = { + hostname: 'api.valueserp.com', + path: `/search?${params.toString()}`, + method: 'GET', + }; + + const req = https.request(options, (res) => { + let data = ''; + + res.on('data', (chunk) => { + data += chunk; + }); + + res.on('end', () => { + if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) { + try { + const jsonData = JSON.parse(data) as OrganicResults; + resolve(jsonData); + } catch { + reject(new Error('Failed to parse search results')); + } + } else { + reject(new Error(`Search request failed with status code: ${res.statusCode}`)); + } + }); + }); + + req.on('error', (e) => { + reject(e); + }); + + req.end(); + }); + } + + private formatSearchResults(results: OrganicResults): string { + if (!Array.isArray(results?.organic_results)) { + return 'No relevant business information found.'; + } + + if (results.organic_results.length === 0) { + return 'No clear business information found in search results.'; + } + + const processedResults: SearchResult[] = []; + + // Deduplication logic with first occurrence preference + results.organic_results.forEach((result) => { + const isDuplicate = processedResults.some( + (pr) => this.getSimilarity(pr.title, result.title) > 0.8, + ); + if (!isDuplicate) { + processedResults.push(result); + } + }); + + // Format first 3 unique results + const formattedResults = processedResults.slice(0, 3) + .map((result, index) => `[Source ${index + 1}] ${result.title}\n` + + `${result.snippet.replace(/(\r\n|\n|\r)/gm, ' ').substring(0, 150)}...\n` + + `URL: ${result.link}`).join('\n\n'); + + return `SEARCH RESULTS:\n${formattedResults}`; + } + + private getSimilarity(str1: string, str2: string): number { + const words1 = str1.toLowerCase().split(/\W+/).filter((w) => w.length > 3); + const words2 = str2.toLowerCase().split(/\W+/).filter((w) => w.length > 3); + + if (!words1.length || !words2.length) return 0; + + const uniqueWords = Array.from(new Set(words1)); + const matches = uniqueWords.filter((word) => words2.includes(word)); + + return matches.length / Math.max(uniqueWords.length, words2.length); + } +} From 82e5228ce31377b75e8bed98692c4457aa9059a7 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Tue, 4 Mar 2025 23:29:39 -0500 Subject: [PATCH 04/17] Add rule-based transaction categorization support --- .gitignore | 1 + src/actual-api-service.ts | 10 +- src/config.ts | 3 + src/container.ts | 9 +- src/handlebars-helpers.ts | 9 + src/llm-model-factory.ts | 2 +- src/llm-service.ts | 73 ++++++- src/prompt-generator.ts | 88 ++++++-- src/templates/similar-rules.hbs | 38 ++++ src/transaction-service.ts | 91 ++++++++- src/types.ts | 46 ++++- src/utils/error-utils.ts | 48 +++-- src/utils/rule-utils.ts | 59 ++++++ src/utils/tool-service.ts | 3 +- test-handlebars.ts | 49 +++++ tests/prompt-generator.test.ts | 192 ++++++++++++++---- .../in-memory-actual-api-service.ts | 63 +++++- tests/test-doubles/mocked-llm-service.ts | 10 +- tests/test-doubles/mocked-prompt-generator.ts | 23 +++ 19 files changed, 719 insertions(+), 98 deletions(-) create mode 100644 src/handlebars-helpers.ts create mode 100644 src/templates/similar-rules.hbs create mode 100644 src/utils/rule-utils.ts create mode 100644 test-handlebars.ts diff --git a/.gitignore b/.gitignore index bfbd299..10aad4e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ tmp/budgets/ dist/ .env +*.log diff --git a/src/actual-api-service.ts b/src/actual-api-service.ts index 6a52434..78f440c 100644 --- a/src/actual-api-service.ts +++ b/src/actual-api-service.ts @@ -4,7 +4,7 @@ import { APICategoryGroupEntity, APIPayeeEntity, } from '@actual-app/api/@types/loot-core/server/api-models'; -import { TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; +import { TransactionEntity, RuleEntity } from '@actual-app/api/@types/loot-core/types/models'; import { ActualApiServiceI } from './types'; class ActualApiService implements ActualApiServiceI { @@ -108,6 +108,14 @@ class ActualApiService implements ActualApiServiceI { return this.actualApiClient.getTransactions(undefined, undefined, undefined); } + public async getRules(): Promise { + return this.actualApiClient.getRules(); + } + + public async getPayeeRules(payeeId: string): Promise { + return this.actualApiClient.getPayeeRules(payeeId); + } + public async updateTransactionNotes(id: string, notes: string): Promise { await this.actualApiClient.updateTransaction(id, { notes }); } diff --git a/src/config.ts b/src/config.ts index e099456..79581cd 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,6 +3,7 @@ import fs from 'fs'; const defaultPromptTemplate = fs.readFileSync('./src/templates/prompt.hbs', 'utf8').trim(); const defaultCategorySuggestionTemplate = fs.readFileSync('./src/templates/category-suggestion.hbs', 'utf8').trim(); +const defaultSimilarRulesTemplate = fs.readFileSync('./src/templates/similar-rules.hbs', 'utf8').trim(); dotenv.config(); @@ -31,6 +32,8 @@ export const notGuessedTag = process.env.NOT_GUESSED_TAG ?? '#actual-ai-miss'; export const guessedTag = process.env.GUESSED_TAG ?? '#actual-ai'; export const categorySuggestionTemplate = process.env.CATEGORY_SUGGESTION_TEMPLATE ?? defaultCategorySuggestionTemplate; +export const similarRulesTemplate = process.env.SIMILAR_RULES_TEMPLATE + ?? defaultSimilarRulesTemplate; export const groqApiKey = process.env.GROQ_API_KEY ?? ''; export const groqModel = process.env.GROQ_MODEL ?? 'llama-3.3-70b-versatile'; export const groqBaseURL = process.env.GROQ_BASE_URL ?? 'https://api.groq.com/openai/v1'; diff --git a/src/container.ts b/src/container.ts index da51a95..dcae930 100644 --- a/src/container.ts +++ b/src/container.ts @@ -28,6 +28,7 @@ import { password, promptTemplate, categorySuggestionTemplate, + similarRulesTemplate, serverURL, suggestNewCategories, syncAccountsBeforeClassify, @@ -72,13 +73,17 @@ const actualApiService = new ActualApiService( e2ePassword, ); +const promptGenerator = new PromptGenerator( + promptTemplate, + categorySuggestionTemplate, + similarRulesTemplate, +); + const llmService = new LlmService( llmModelFactory, toolService, ); -const promptGenerator = new PromptGenerator(promptTemplate, categorySuggestionTemplate); - const transactionService = new TransactionService( actualApiService, llmService, diff --git a/src/handlebars-helpers.ts b/src/handlebars-helpers.ts new file mode 100644 index 0000000..8009cb6 --- /dev/null +++ b/src/handlebars-helpers.ts @@ -0,0 +1,9 @@ +import * as handlebars from 'handlebars'; + +// Register the 'eq' helper for equality comparison +handlebars.registerHelper('eq', (arg1, arg2) => arg1 === arg2); + +// Register the 'incIndex' helper for incrementing index +handlebars.registerHelper('incIndex', (index: number) => index + 1); + +export default handlebars; diff --git a/src/llm-model-factory.ts b/src/llm-model-factory.ts index 3a91d99..852c1eb 100644 --- a/src/llm-model-factory.ts +++ b/src/llm-model-factory.ts @@ -106,7 +106,7 @@ class LlmModelFactory implements LlmModelFactoryI { baseURL: this.groqBaseURL, apiKey: this.groqApiKey, }); - return groq(this.groqModel); + return groq(this.groqModel) as unknown as LanguageModel; } default: throw new Error(`Unknown provider: ${this.llmProvider}`); diff --git a/src/llm-service.ts b/src/llm-service.ts index f203ea4..fd74a9e 100644 --- a/src/llm-service.ts +++ b/src/llm-service.ts @@ -1,6 +1,9 @@ import { z } from 'zod'; import { generateObject, generateText, LanguageModel } from 'ai'; -import { LlmModelFactoryI, LlmServiceI, ToolServiceI } from './types'; +import { TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; +import { + CategorySuggestion, LlmModelFactoryI, LlmServiceI, ToolServiceI, +} from './types'; import { RateLimiter } from './utils/rate-limiter'; import { PROVIDER_LIMITS } from './utils/provider-limits'; @@ -75,7 +78,7 @@ export default class LlmService implements LlmServiceI { public async askForCategorySuggestion( prompt: string, - ): Promise<{ name: string, groupName: string, groupIsNew: boolean } | null> { + ): Promise { try { console.log( `Making LLM request for category suggestion to ${this.provider}${this.isFallbackMode ? ' (fallback mode)' : ''}`, @@ -87,8 +90,6 @@ export default class LlmService implements LlmServiceI { groupIsNew: z.boolean(), }); - type CategorySuggestion = z.infer; - const response = await this.rateLimiter.executeWithRateLimiting( this.provider, async () => { @@ -101,7 +102,6 @@ export default class LlmService implements LlmServiceI { system: 'You must use webSearch for unfamiliar payees before suggesting categories', }); - // Add this debug logging console.log('Generation steps:', steps.map((step) => ({ text: step.text, toolCalls: step.toolCalls, @@ -137,6 +137,69 @@ export default class LlmService implements LlmServiceI { } } + /** + * Analyze if a transaction is similar to any existing rule and suggest a category + * @param transaction The transaction to analyze + * @param rules List of existing rules in the system + * @param categories List of categories for reference + * @param prompt The prompt to use for finding similar rules + * @returns A suggested category ID if similar rules exist, null otherwise + */ + public async findSimilarRules( + transaction: TransactionEntity, + prompt: string, + ): Promise<{ categoryId: string; ruleName: string } | null> { + try { + console.log( + `Checking if transaction "${transaction.imported_payee}" matches any existing rules`, + ); + + // console.log('Prompt:', prompt.slice(0, 300)); + + return this.rateLimiter.executeWithRateLimiting< + { categoryId: string; ruleName: string } | null>( + this.provider, + async () => { + const { text, steps } = await generateText({ + model: this.model, + prompt, + temperature: 0.1, + tools: this.toolService?.getTools(), + maxSteps: 3, + system: 'You must respond with pong if you receive don\'t have an answer', + }); + + console.log('Generation steps:', steps.map((step) => ({ + text: step.text, + toolCalls: step.toolCalls, + toolResults: step.toolResults, + }))); + + try { + // Parse the JSON response + const response = JSON.parse(text) as { categoryId?: string; ruleName?: string } | null; + + if (response?.categoryId && response.ruleName) { + console.log(`Found similar rule "${response.ruleName}" suggesting category ${response.categoryId}`); + return { + categoryId: response.categoryId, + ruleName: response.ruleName, + }; + } + + return null; + } catch { + console.log('No similar rules found or invalid response'); + return null; + } + }, + ); + } catch (error) { + console.error('Error while finding similar rules:', error); + return null; + } + } + public async askWithEnum(prompt: string, categoryIds: string[]): Promise { return this.rateLimiter.executeWithRateLimiting( this.provider, diff --git a/src/prompt-generator.ts b/src/prompt-generator.ts index e703e12..d192ae9 100644 --- a/src/prompt-generator.ts +++ b/src/prompt-generator.ts @@ -1,18 +1,26 @@ -import { APICategoryGroupEntity, APIPayeeEntity } from '@actual-app/api/@types/loot-core/server/api-models'; -import { TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; -import * as handlebars from 'handlebars'; -import { PromptGeneratorI } from './types'; +import { APICategoryEntity, APICategoryGroupEntity, APIPayeeEntity } from '@actual-app/api/@types/loot-core/server/api-models'; +import { RuleEntity, TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; +import handlebars from './handlebars-helpers'; +import { PromptGeneratorI, RuleDescription } from './types'; import PromptTemplateException from './exceptions/prompt-template-exception'; import { hasWebSearchTool } from './config'; +import { transformRulesToDescriptions } from './utils/rule-utils'; class PromptGenerator implements PromptGeneratorI { private readonly promptTemplate: string; private readonly categorySuggestionTemplate: string; - constructor(promptTemplate: string, categorySuggestionTemplate: string) { + private readonly similarRulesTemplate: string; + + constructor( + promptTemplate: string, + categorySuggestionTemplate = '', + similarRulesTemplate = '', + ) { this.promptTemplate = promptTemplate; this.categorySuggestionTemplate = categorySuggestionTemplate; + this.similarRulesTemplate = similarRulesTemplate; } generate( @@ -23,8 +31,8 @@ class PromptGenerator implements PromptGeneratorI { let template; try { template = handlebars.compile(this.promptTemplate); - } catch (error) { - console.error('Error generating prompt. Check syntax of your template.', error); + } catch { + console.error('Error generating prompt. Check syntax of your template.'); throw new PromptTemplateException('Error generating prompt. Check syntax of your template.'); } const payeeName = payees.find((payee) => payee.id === transaction.payee)?.name; @@ -48,8 +56,8 @@ class PromptGenerator implements PromptGeneratorI { cleared: transaction.cleared, reconciled: transaction.reconciled, }); - } catch (error) { - console.error('Error generating prompt. Check syntax of your template.', error); + } catch { + console.error('Error generating prompt. Check syntax of your template.'); throw new PromptTemplateException('Error generating prompt. Check syntax of your template.'); } } @@ -62,8 +70,8 @@ class PromptGenerator implements PromptGeneratorI { let template; try { template = handlebars.compile(this.categorySuggestionTemplate); - } catch (error) { - console.error('Error generating category suggestion prompt.', error); + } catch { + console.error('Error generating category suggestion prompt.'); throw new PromptTemplateException('Error generating category suggestion prompt.'); } @@ -77,23 +85,69 @@ class PromptGenerator implements PromptGeneratorI { })); try { + const webSearchEnabled = typeof hasWebSearchTool === 'boolean' ? hasWebSearchTool : false; return template({ categoryGroups: groupsWithCategories, + amount: Math.abs(transaction.amount), + type: transaction.amount > 0 ? 'Income' : 'Outcome', + description: transaction.notes ?? '', + payee: payeeName ?? '', + importedPayee: transaction.imported_payee ?? '', + date: transaction.date ?? '', + cleared: transaction.cleared, + reconciled: transaction.reconciled, + hasWebSearchTool: webSearchEnabled, + }); + } catch { + console.error('Error generating category suggestion prompt.'); + throw new PromptTemplateException('Error generating category suggestion prompt.'); + } + } + + generateSimilarRulesPrompt( + transaction: TransactionEntity & { payeeName?: string }, + rulesDescription: RuleDescription[], + ): string { + let template; + try { + template = handlebars.compile(this.similarRulesTemplate); + } catch { + console.error('Error generating similar rules prompt.'); + throw new PromptTemplateException('Error generating similar rules prompt.'); + } + + try { + // Add index to each rule for numbering + const rulesWithIndex = rulesDescription.map((rule, index) => ({ + ...rule, + index, + })); + + // Use payeeName if available, otherwise use imported_payee + const payee = transaction.payeeName ?? transaction.imported_payee; + + return template({ amount: Math.abs(transaction.amount), type: transaction.amount > 0 ? 'Income' : 'Outcome', description: transaction.notes, - payee: payeeName, importedPayee: transaction.imported_payee, + payee, date: transaction.date, - cleared: transaction.cleared, - reconciled: transaction.reconciled, - hasWebSearchTool, + rules: rulesWithIndex, }); } catch (error) { - console.error('Error generating category suggestion prompt.', error); - throw new PromptTemplateException('Error generating category suggestion prompt.'); + console.error('Error generating similar rules prompt:', error); + throw new PromptTemplateException('Error generating similar rules prompt.'); } } + + transformRulesToDescriptions( + rules: RuleEntity[], + categories: (APICategoryEntity | APICategoryGroupEntity)[], + payees: APIPayeeEntity[] = [], + ): RuleDescription[] { + return transformRulesToDescriptions(rules, categories, payees); + } } export default PromptGenerator; diff --git a/src/templates/similar-rules.hbs b/src/templates/similar-rules.hbs new file mode 100644 index 0000000..0eb6e61 --- /dev/null +++ b/src/templates/similar-rules.hbs @@ -0,0 +1,38 @@ +Transaction details: +* Payee: {{importedPayee}} +* Amount: {{amount}} +* Type: {{type}} +{{#if description}} +* Description: {{description}} +{{/if}} +{{#if date}} +* Date: {{date}} +{{/if}} + +Existing rules in the system: +{{#each rules}} +{{incIndex @index}}. Rule: "{{ruleName}}" +- Category: {{categoryName}} +- Conditions: +{{#each conditions}} + - {{field}} {{op}} + {{#if (eq type "id")}} + [{{#each value}}"{{this}}"{{#unless @last}}, {{/unless}}{{/each}}] + {{else}} + "{{value}}" + {{/if}} +{{/each}} +{{/each}} + +Based on the transaction details, determine if it closely matches any of the existing rules. +If there's a match, return a JSON object with the categoryId and ruleName. +If there's no good match, return null. + +Example response for a match: {"categoryId": "abc123", "ruleName": "Rule Name"} +Example response for no match: null + +IMPORTANT: Your response MUST be: +- A SINGLE VALID JSON OBJECT or null +- No additional text or explanation +- No markdown formatting +- Properly escaped characters \ No newline at end of file diff --git a/src/transaction-service.ts b/src/transaction-service.ts index 3841c68..503e03c 100644 --- a/src/transaction-service.ts +++ b/src/transaction-service.ts @@ -81,13 +81,17 @@ class TransactionService implements TransactionServiceI { } async processTransactions(): Promise { - const categoryGroups = await this.actualApiService.getCategoryGroups(); - const categories = await this.actualApiService.getCategories(); - const payees = await this.actualApiService.getPayees(); - const transactions = await this.actualApiService.getTransactions(); - const accounts = await this.actualApiService.getAccounts(); - const accountsToSkip = accounts.filter((account) => account.offbudget) - .map((account) => account.id); + const [categoryGroups, categories, payees, transactions, accounts, rules] = await Promise.all([ + this.actualApiService.getCategoryGroups(), + this.actualApiService.getCategories(), + this.actualApiService.getPayees(), + this.actualApiService.getTransactions(), + this.actualApiService.getAccounts(), + this.actualApiService.getRules(), + ]); + const accountsToSkip = accounts?.filter((account) => account.offbudget) + .map((account) => account.id) ?? []; + console.log(`Found ${rules.length} transaction categorization rules`); const uncategorizedTransactions = transactions.filter( (transaction) => !transaction.category @@ -128,6 +132,79 @@ class TransactionService implements TransactionServiceI { console.log(`${globalIndex + 1}/${uncategorizedTransactions.length} Processing transaction ${transaction.imported_payee} / ${transaction.notes} / ${transaction.amount}`); try { + // Check if there are any rules that might apply to this payee + const payeeRules = transaction.payee + ? await this.actualApiService.getPayeeRules(transaction.payee) + : []; + + // Log if we found any matching rules + if (payeeRules.length > 0) { + console.log(`Found ${payeeRules.length} rules associated with payee ID ${transaction.payee}`); + + // Find a rule that might set a category + const categoryRule = payeeRules.find((rule) => rule.actions.some( + (action) => 'field' in action && action.field === 'category' && action.op === 'set', + )); + + if (categoryRule) { + // Extract the category ID from the rule + const categoryAction = categoryRule.actions.find( + (action) => 'field' in action && action.field === 'category' && action.op === 'set', + ); + + if (categoryAction && 'value' in categoryAction && typeof categoryAction.value === 'string') { + const categoryId = categoryAction.value; + const category = categories.find((c) => c.id === categoryId); + + if (category) { + console.log(`Rule suggests category: ${category.name}`); + await this.actualApiService.updateTransactionNotesAndCategory( + transaction.id, + this.appendTag(transaction.notes ?? '', `${this.guessedTag} (from rule)`), + categoryId, + ); + // Skip to next transaction since we've categorized this one + continue; + } + } + } + } + + // If no direct payee rule was found, check if transaction is similar to any existing rule + if (this.llmService.findSimilarRules) { + const rulesDescription = this.promptGenerator.transformRulesToDescriptions( + rules, + categories, + payees, + ); + + const similarRulesPrompt = this.promptGenerator.generateSimilarRulesPrompt( + transaction, + rulesDescription, + ); + + const similarRuleResult = await this.llmService.findSimilarRules( + transaction, + similarRulesPrompt, + ); + + if (similarRuleResult) { + const { categoryId, ruleName } = similarRuleResult; + const category = categories.find((c) => c.id === categoryId); + + if (category) { + console.log(`Transaction similar to rule "${ruleName}", suggesting category: ${category.name}`); + await this.actualApiService.updateTransactionNotesAndCategory( + transaction.id, + this.appendTag(transaction.notes ?? '', `${this.guessedTag} (similar to rule)`), + categoryId, + ); + // Skip to next transaction since we've categorized this one + continue; + } + } + } + const prompt = this.promptGenerator.generate(categoryGroups, transaction, payees); const guess = await this.llmService.ask(prompt, categoryIds); let guessCategory = categories.find((category) => category.id === guess); diff --git a/src/types.ts b/src/types.ts index 56c8030..e01087b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -5,11 +5,10 @@ import { APICategoryGroupEntity, APIPayeeEntity, } from '@actual-app/api/@types/loot-core/server/api-models'; -import { TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; +import { TransactionEntity, RuleEntity } from '@actual-app/api/@types/loot-core/types/models'; export interface LlmModelI { ask(prompt: string, possibleAnswers: string[]): Promise; - askFreeform(prompt: string): Promise; } export interface LlmModelFactoryI { @@ -34,6 +33,10 @@ export interface ActualApiServiceI { getTransactions(): Promise + getRules(): Promise + + getPayeeRules(payeeId: string): Promise + updateTransactionNotes(id: string, notes: string): Promise updateTransactionNotesAndCategory( @@ -63,12 +66,34 @@ export interface ActualAiServiceI { syncAccounts(): Promise } +export interface RuleDescription { + ruleName: string; + conditions: { + field: string; + op: string; + type?: string; + value: string | string[]; + }[]; + categoryName: string; + categoryId: string; + index?: number; +} + +export interface CategorySuggestion { + name: string; + groupName: string; + groupIsNew: boolean; +} + export interface LlmServiceI { - ask(prompt: string, possibleAnswers: string[]): Promise; + ask(prompt: string, categoryIds: string[]): Promise; - askForCategorySuggestion( + askForCategorySuggestion(prompt: string): Promise; + + findSimilarRules( + transaction: TransactionEntity, prompt: string - ): Promise<{ name: string, groupName: string, groupIsNew: boolean } | null> + ): Promise<{ categoryId: string; ruleName: string } | null>; } export interface ToolServiceI { @@ -87,4 +112,15 @@ export interface PromptGeneratorI { transaction: TransactionEntity, payees: APIPayeeEntity[], ): string + + generateSimilarRulesPrompt( + transaction: TransactionEntity & { payeeName?: string }, + rulesDescription: RuleDescription[], + ): string + + transformRulesToDescriptions( + rules: RuleEntity[], + categories: (APICategoryEntity | APICategoryGroupEntity)[], + payees: APIPayeeEntity[], + ): RuleDescription[] } diff --git a/src/utils/error-utils.ts b/src/utils/error-utils.ts index 76a0d14..6c43329 100644 --- a/src/utils/error-utils.ts +++ b/src/utils/error-utils.ts @@ -1,3 +1,12 @@ +interface RateLimitError extends Error { + statusCode?: number; + responseHeaders?: { + 'retry-after'?: string; + 'Retry-After'?: string; + }; + responseBody?: string; +} + /** * Checks if an error is a rate limit error * @param error Any error object or value @@ -5,17 +14,13 @@ */ export const isRateLimitError = (error: unknown): boolean => { if (!error) return false; - - // Convert to string to handle various error types - const errorStr = String(error); - - // Check for common rate limit indicators + const errorStr = error instanceof Error ? error.message : JSON.stringify(error); return errorStr.toLowerCase().includes('rate limit') || errorStr.toLowerCase().includes('rate_limit') || errorStr.toLowerCase().includes('too many requests') || (error instanceof Error && 'statusCode' in error - && (error as any).statusCode === 429); + && (error as RateLimitError).statusCode === 429); }; /** @@ -28,32 +33,31 @@ export const extractRetryAfterMs = (error: unknown): number | undefined => { if (error instanceof Error) { try { - // Check for retry information in error message (common in provider responses) const match = /try again in (\d+(\.\d+)?)s/i.exec(error.message); if (match?.[1]) { return Math.ceil(parseFloat(match[1]) * 1000); } - - // Try to get from headers if available - if ('responseHeaders' in error && (error as any).responseHeaders) { - const headers = (error as any).responseHeaders; - if (headers['retry-after'] || headers['Retry-After']) { - const retryAfter = headers['retry-after'] || headers['Retry-After']; - if (!isNaN(Number(retryAfter))) { + if ('responseHeaders' in error && (error as RateLimitError).responseHeaders) { + const headers = (error as RateLimitError).responseHeaders; + if (headers?.['retry-after'] || headers?.['Retry-After']) { + const retryAfter = headers['retry-after'] ?? headers['Retry-After']; + if (retryAfter && !Number.isNaN(Number(retryAfter))) { return Number(retryAfter) * 1000; } } } - // Check for reset time in responseBody if it exists - if ('responseBody' in error && typeof (error as any).responseBody === 'string') { - try { - const body = JSON.parse((error as any).responseBody); - if (body.error?.reset_time) { - return body.error.reset_time * 1000; + if ('responseBody' in error) { + const { responseBody } = (error as RateLimitError); + if (typeof responseBody === 'string') { + try { + const body = JSON.parse(responseBody) as { error?: { reset_time?: number } }; + if (body.error?.reset_time) { + return body.error.reset_time * 1000; + } + } catch { + // Ignore JSON parse errors } - } catch (e) { - // Ignore JSON parse errors } } } catch (e) { diff --git a/src/utils/rule-utils.ts b/src/utils/rule-utils.ts new file mode 100644 index 0000000..56e3369 --- /dev/null +++ b/src/utils/rule-utils.ts @@ -0,0 +1,59 @@ +import { APICategoryEntity, APICategoryGroupEntity, APIPayeeEntity } from '@actual-app/api/@types/loot-core/server/api-models'; +import { RuleEntity } from '@actual-app/api/@types/loot-core/types/models'; +import { RuleDescription } from '../types'; + +/** + * Transforms rule entities into a more readable description format + */ +export function transformRulesToDescriptions( + rules: RuleEntity[], + categories: (APICategoryEntity | APICategoryGroupEntity)[], + payees: APIPayeeEntity[] = [], +): RuleDescription[] { + return rules.map((rule) => { + const categoryAction = rule.actions.find( + (action) => 'field' in action && action.field === 'category' && action.op === 'set', + ); + const categoryId = categoryAction?.value as string | undefined; + const category = categories.find((c) => 'id' in c && c.id === categoryId); + + // Improved payee resolution with clean JSON structure + const resolvePayeeValue = (value: string | string[]) => { + const resolveSingle = (id: string) => payees.find((p) => p.id === id)?.name ?? id; + + return Array.isArray(value) + ? value.map(resolveSingle) + : resolveSingle(value); + }; + + return { + conditions: rule.conditions.map((c) => { + // Structure condition as properly typed object + const condition: { + field: string; + op: string; + type?: string; + value: string | string[]; + } = { + field: c.field, + op: c.op, + type: c.type, + value: '', // Default value, will be updated below + }; + + if (c.field === 'payee' && c.type === 'id') { + condition.value = resolvePayeeValue(c.value); + } else { + condition.value = typeof c.value === 'object' ? (c.value as string[]) : String(c.value); + } + + return condition; + }), + categoryName: category && 'name' in category ? category.name : 'unknown', + categoryId: categoryId ?? '', + ruleName: 'name' in rule ? rule.name as string : 'Unnamed rule', + }; + }).filter((r) => r.categoryId); +} + +export default { transformRulesToDescriptions }; diff --git a/src/utils/tool-service.ts b/src/utils/tool-service.ts index babe4e0..0f7ba76 100644 --- a/src/utils/tool-service.ts +++ b/src/utils/tool-service.ts @@ -24,7 +24,7 @@ export default class ToolService implements ToolServiceI { public getTools() { const tools: Record = {}; - if (enabledTools.includes('webSearch')) { + if (Array.isArray(enabledTools) && enabledTools.includes('webSearch')) { tools.webSearch = tool({ description: 'Essential for researching business types and industry categorizations when existing categories are insufficient. Use when payee is unfamiliar or category context is unclear', parameters: z.object({ @@ -35,6 +35,7 @@ export default class ToolService implements ToolServiceI { }), execute: async ({ query }: { query: string }): Promise => { if (!this.valueSerpApiKey) return 'Search unavailable'; + console.log(`Performing web search for ${query}`); const results = await this.performSearch(query); return this.formatSearchResults(results); }, diff --git a/test-handlebars.ts b/test-handlebars.ts new file mode 100644 index 0000000..50ec350 --- /dev/null +++ b/test-handlebars.ts @@ -0,0 +1,49 @@ +import * as fs from 'fs'; +import handlebars from './src/handlebars-helpers'; + +// Load the template +const similarRulesTemplate = fs.readFileSync('./src/templates/similar-rules.hbs', 'utf8').trim(); + +// Compile the template +const template = handlebars.compile(similarRulesTemplate); + +// Test data +const testData = { + amount: 100, + type: 'Outcome', + description: 'Test transaction', + importedPayee: 'Test Payee', + payee: 'Test Payee', + date: '2025-03-05', + rules: [ + { + index: 0, + ruleName: 'Test Rule', + categoryName: 'Test Category', + conditions: [ + { + field: 'payee', + op: 'is', + type: 'id', + value: ['test-id-1', 'test-id-2'], + }, + { + field: 'notes', + op: 'contains', + type: 'string', + value: 'test', + }, + ], + }, + ], +}; + +// Execute the template +try { + const result = template(testData); + console.log('Template rendered successfully:'); + console.log(result); + console.log('\nHandlebars helpers are working correctly!'); +} catch (error) { + console.error('Error rendering template:', error); +} diff --git a/tests/prompt-generator.test.ts b/tests/prompt-generator.test.ts index 4f24814..9679246 100644 --- a/tests/prompt-generator.test.ts +++ b/tests/prompt-generator.test.ts @@ -5,9 +5,52 @@ import GivenActualData from './test-doubles/given/given-actual-data'; import PromptTemplateException from '../src/exceptions/prompt-template-exception'; const promptTemplate = fs.readFileSync('./src/templates/prompt.hbs', 'utf8').trim(); +const categorySuggestionTemplate = fs.readFileSync('./src/templates/category-suggestion.hbs', 'utf8').trim(); +const similarRulesTemplate = fs.readFileSync('./src/templates/similar-rules.hbs', 'utf8').trim(); describe('LlmGenerator', () => { - const promptSet: [TransactionEntity, string, string][] = [ + const expectedAirbnb = 'I want to categorize the given bank transaction into one of the following categories:\n' + + 'GROUP: Usual Expenses (ID: "1")\n' + + '* Groceries (ID: "ff7be77b-40f4-4e9d-aea4-be6b8c431281")\n' + + '* Travel (ID: "541836f1-e756-4473-a5d0-6c1d3f06c7fa")\n' + + 'GROUP: Income (ID: "2")\n' + + '* Salary (ID: "123836f1-e756-4473-a5d0-6c1d3f06c7fa")\n\n' + + 'Transaction details:\n' + + '* Amount: 34169\n' + + '* Type: Outcome\n' + + '* Description: AIRBNB * XXXX1234567 822-307-2000\n' + + '* Payee: Airbnb * XXXX1234567\n\n' + + 'RESPOND ONLY WITH A CATEGORY ID from the list above. Do not write anything else.\n' + + 'If you\'re not sure which category to use, respond with "uncategorized".'; + + const expectedCarrefour = 'I want to categorize the given bank transaction into one of the following categories:\n' + + 'GROUP: Usual Expenses (ID: "1")\n' + + '* Groceries (ID: "ff7be77b-40f4-4e9d-aea4-be6b8c431281")\n' + + '* Travel (ID: "541836f1-e756-4473-a5d0-6c1d3f06c7fa")\n' + + 'GROUP: Income (ID: "2")\n' + + '* Salary (ID: "123836f1-e756-4473-a5d0-6c1d3f06c7fa")\n\n' + + 'Transaction details:\n' + + '* Amount: 1000\n' + + '* Type: Outcome\n' + + '* Payee: Carrefour\n\n' + + 'RESPOND ONLY WITH A CATEGORY ID from the list above. Do not write anything else.\n' + + 'If you\'re not sure which category to use, respond with "uncategorized".'; + + const expectedGoogle = 'I want to categorize the given bank transaction into one of the following categories:\n' + + 'GROUP: Usual Expenses (ID: "1")\n' + + '* Groceries (ID: "ff7be77b-40f4-4e9d-aea4-be6b8c431281")\n' + + '* Travel (ID: "541836f1-e756-4473-a5d0-6c1d3f06c7fa")\n' + + 'GROUP: Income (ID: "2")\n' + + '* Salary (ID: "123836f1-e756-4473-a5d0-6c1d3f06c7fa")\n\n' + + 'Transaction details:\n' + + '* Amount: 2137420\n' + + '* Type: Income\n' + + '* Description: DESCRIPTION\n' + + '* Payee: Google\n\n' + + 'RESPOND ONLY WITH A CATEGORY ID from the list above. Do not write anything else.\n' + + 'If you\'re not sure which category to use, respond with "uncategorized".'; + + const promptSet: [TransactionEntity, string][] = [ [ GivenActualData.createTransaction( '1', @@ -15,17 +58,7 @@ describe('LlmGenerator', () => { 'Airbnb * XXXX1234567', 'AIRBNB * XXXX1234567 822-307-2000', ), - 'I want to categorize the given bank transactions into the following categories:' - + '\n* Groceries (Usual Expenses) (ID: "ff7be77b-40f4-4e9d-aea4-be6b8c431281")' - + '\n* Travel (Usual Expenses) (ID: "541836f1-e756-4473-a5d0-6c1d3f06c7fa")' - + '\n* Salary (Income) (ID: "123836f1-e756-4473-a5d0-6c1d3f06c7fa")' - + '\nPlease categorize the following transaction:' - + '\n* Amount: 34169' - + '\n* Type: Outcome' - + '\n* Description: AIRBNB * XXXX1234567 822-307-2000' - + '\n* Payee: Airbnb * XXXX1234567' - + '\nANSWER BY A CATEGORY ID - DO NOT CREATE ENTIRE SENTENCE - DO NOT WRITE CATEGORY NAME, JUST AN ID. Do not guess, if you don\'t know the answer, return "uncategorized".', - promptTemplate, + expectedAirbnb, ], [ GivenActualData.createTransaction( '1', @@ -34,16 +67,7 @@ describe('LlmGenerator', () => { '', GivenActualData.PAYEE_CARREFOUR, ), - 'I want to categorize the given bank transactions into the following categories:' - + '\n* Groceries (Usual Expenses) (ID: "ff7be77b-40f4-4e9d-aea4-be6b8c431281")' - + '\n* Travel (Usual Expenses) (ID: "541836f1-e756-4473-a5d0-6c1d3f06c7fa")' - + '\n* Salary (Income) (ID: "123836f1-e756-4473-a5d0-6c1d3f06c7fa")' - + '\nPlease categorize the following transaction:' - + '\n* Amount: 1000' - + '\n* Type: Outcome' - + '\n* Payee: Carrefour' - + '\nANSWER BY A CATEGORY ID - DO NOT CREATE ENTIRE SENTENCE - DO NOT WRITE CATEGORY NAME, JUST AN ID. Do not guess, if you don\'t know the answer, return "uncategorized".', - promptTemplate, + expectedCarrefour, ], [ GivenActualData.createTransaction( '1', @@ -52,17 +76,7 @@ describe('LlmGenerator', () => { 'DESCRIPTION', GivenActualData.PAYEE_GOOGLE, ), - 'I want to categorize the given bank transactions into the following categories:' - + '\n* Groceries (Usual Expenses) (ID: "ff7be77b-40f4-4e9d-aea4-be6b8c431281")' - + '\n* Travel (Usual Expenses) (ID: "541836f1-e756-4473-a5d0-6c1d3f06c7fa")' - + '\n* Salary (Income) (ID: "123836f1-e756-4473-a5d0-6c1d3f06c7fa")' - + '\nPlease categorize the following transaction:' - + '\n* Amount: 2137420' - + '\n* Type: Income' - + '\n* Description: DESCRIPTION' - + '\n* Payee: Google' - + '\nANSWER BY A CATEGORY ID - DO NOT CREATE ENTIRE SENTENCE - DO NOT WRITE CATEGORY NAME, JUST AN ID. Do not guess, if you don\'t know the answer, return "uncategorized".', - promptTemplate, + expectedGoogle, ], ]; @@ -73,7 +87,7 @@ describe('LlmGenerator', () => { const categoryGroups = GivenActualData.createSampleCategoryGroups(); const payees = GivenActualData.createSamplePayees(); - const promptGenerator = new PromptGenerator(promptTemplate); + const promptGenerator = new PromptGenerator(promptTemplate, categorySuggestionTemplate); const prompt = promptGenerator.generate(categoryGroups, transaction, payees); expect(prompt).toEqual(expectedPrompt); @@ -84,11 +98,119 @@ describe('LlmGenerator', () => { const payees = GivenActualData.createSamplePayees(); const transaction = GivenActualData.createTransaction('1', 1000, 'Carrefour 2137'); - const promptGenerator = new PromptGenerator('{{#each categories}}'); + const promptGenerator = new PromptGenerator('{{#each categories}}', categorySuggestionTemplate); const t = () => { promptGenerator.generate(categoryGroups, transaction, payees); }; expect(t).toThrow(PromptTemplateException); }); + + it('should generate a category suggestion prompt', () => { + const categoryGroups = GivenActualData.createSampleCategoryGroups(); + const payees = GivenActualData.createSamplePayees(); + const transaction = GivenActualData.createTransaction( + '1', + -1000, + 'Carrefour 2137', + '', + GivenActualData.PAYEE_CARREFOUR, + ); + + const promptGenerator = new PromptGenerator(promptTemplate, categorySuggestionTemplate); + const prompt = promptGenerator.generateCategorySuggestion(categoryGroups, transaction, payees); + + expect(prompt).toContain('I need to suggest a new category for a transaction'); + expect(prompt).toContain('* Payee: Carrefour'); + expect(prompt).toContain('* Amount: 1000'); + expect(prompt).toContain('* Type: Outcome'); + expect(prompt).toContain('RESPOND WITH A JSON OBJECT'); + }); + + it('should throw exception on invalid category suggestion prompt', () => { + const categoryGroups = GivenActualData.createSampleCategoryGroups(); + const payees = GivenActualData.createSamplePayees(); + const transaction = GivenActualData.createTransaction('1', 1000, 'Carrefour 2137'); + const promptGenerator = new PromptGenerator(promptTemplate, '{{#each invalidSyntax}}'); + + const t = () => { + promptGenerator.generateCategorySuggestion(categoryGroups, transaction, payees); + }; + + expect(t).toThrow(PromptTemplateException); + }); + + it('should generate a similar rules prompt', () => { + const transaction = GivenActualData.createTransaction( + '1', + -1000, + 'Carrefour 2137', + '', + GivenActualData.PAYEE_CARREFOUR, + ); + + const rulesDescription = [ + { + ruleName: 'Grocery Rule', + conditions: 'payee contains "Carrefour"', + categoryName: 'Groceries', + categoryId: GivenActualData.CATEGORY_GROCERIES, + }, + { + ruleName: 'Travel Rule', + conditions: 'payee contains "Airbnb"', + categoryName: 'Travel', + categoryId: GivenActualData.CATEGORY_TRAVEL, + }, + ]; + + const promptGenerator = new PromptGenerator( + promptTemplate, + categorySuggestionTemplate, + similarRulesTemplate, + ); + const prompt = promptGenerator.generateSimilarRulesPrompt(transaction, rulesDescription); + + expect(prompt).toContain('Transaction details:'); + expect(prompt).toContain('* Payee: Carrefour 2137'); + expect(prompt).toContain('* Amount: 1000'); + expect(prompt).toContain('* Type: Outcome'); + + // Less strict checks that don't rely on exact formatting + expect(prompt).toContain('Grocery Rule'); + expect(prompt).toContain('payee contains'); + expect(prompt).toContain('Carrefour'); + expect(prompt).toContain('Groceries'); + + expect(prompt).toContain('Travel Rule'); + expect(prompt).toContain('Airbnb'); + expect(prompt).toContain('Travel'); + + expect(prompt).toContain('Based on the transaction details'); + expect(prompt).toContain('If there\'s a match, return a JSON object'); + }); + + it('should throw exception on invalid similar rules prompt', () => { + const transaction = GivenActualData.createTransaction('1', 1000, 'Carrefour 2137'); + const rulesDescription = [ + { + ruleName: 'Grocery Rule', + conditions: 'payee contains "Carrefour"', + categoryName: 'Groceries', + categoryId: GivenActualData.CATEGORY_GROCERIES, + }, + ]; + + const promptGenerator = new PromptGenerator( + promptTemplate, + categorySuggestionTemplate, + '{{#each invalidSyntax}}', + ); + + const t = () => { + promptGenerator.generateSimilarRulesPrompt(transaction, rulesDescription); + }; + + expect(t).toThrow(PromptTemplateException); + }); }); diff --git a/tests/test-doubles/in-memory-actual-api-service.ts b/tests/test-doubles/in-memory-actual-api-service.ts index 46a5699..4710f78 100644 --- a/tests/test-doubles/in-memory-actual-api-service.ts +++ b/tests/test-doubles/in-memory-actual-api-service.ts @@ -4,7 +4,7 @@ import { APICategoryGroupEntity, APIPayeeEntity, } from '@actual-app/api/@types/loot-core/server/api-models'; -import { TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; +import { RuleEntity, TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; import { ActualApiServiceI } from '../../src/types'; export default class InMemoryActualApiService implements ActualApiServiceI { @@ -104,4 +104,65 @@ export default class InMemoryActualApiService implements ActualApiServiceI { public getWasBankSyncRan(): boolean { return this.wasBankSyncRan; } + + async createCategory(name: string, groupId: string): Promise { + const categoryId = `cat-${Date.now()}`; + const newCategory: APICategoryEntity = { + id: categoryId, + name, + group_id: groupId, + is_income: false, + }; + + this.categories.push(newCategory); + + // Update the category group to include this category + const groupIndex = this.categoryGroups.findIndex((group) => group.id === groupId); + if (groupIndex >= 0) { + if (!this.categoryGroups[groupIndex].categories) { + this.categoryGroups[groupIndex].categories = []; + } + this.categoryGroups[groupIndex].categories.push(newCategory); + } + + return Promise.resolve(categoryId); + } + + async createCategoryGroup(name: string): Promise { + const groupId = `group-${Date.now()}`; + const newGroup: APICategoryGroupEntity = { + id: groupId, + name, + is_income: false, + categories: [], + }; + + this.categoryGroups.push(newGroup); + this.categories.push(newGroup); + + return Promise.resolve(groupId); + } + + async updateCategoryGroup(id: string, name: string): Promise { + const groupIndex = this.categoryGroups.findIndex((group) => group.id === id); + if (groupIndex >= 0) { + this.categoryGroups[groupIndex].name = name; + } + + // Also update in the categories array + const categoryIndex = this.categories.findIndex((cat) => cat.id === id); + if (categoryIndex >= 0) { + this.categories[categoryIndex].name = name; + } + + return Promise.resolve(); + } + + async getRules(): Promise { + return Promise.resolve([]); + } + + async getPayeeRules(_payeeId: string): Promise { + return Promise.resolve([]); + } } diff --git a/tests/test-doubles/mocked-llm-service.ts b/tests/test-doubles/mocked-llm-service.ts index 12d9512..c6f0f9e 100644 --- a/tests/test-doubles/mocked-llm-service.ts +++ b/tests/test-doubles/mocked-llm-service.ts @@ -1,4 +1,4 @@ -import { LlmServiceI } from '../../src/types'; +import { CategorySuggestion, LlmServiceI } from '../../src/types'; export default class MockedLlmService implements LlmServiceI { private guess = 'uncategorized'; @@ -7,7 +7,15 @@ export default class MockedLlmService implements LlmServiceI { return Promise.resolve(this.guess); } + async askForCategorySuggestion(): Promise { + return Promise.resolve(null); + } + setGuess(guess: string): void { this.guess = guess; } + + async findSimilarRules(): Promise<{ categoryId: string; ruleName: string } | null> { + return Promise.resolve(null); + } } diff --git a/tests/test-doubles/mocked-prompt-generator.ts b/tests/test-doubles/mocked-prompt-generator.ts index 293e85b..7a67368 100644 --- a/tests/test-doubles/mocked-prompt-generator.ts +++ b/tests/test-doubles/mocked-prompt-generator.ts @@ -1,7 +1,30 @@ +import { APICategoryGroupEntity, APIPayeeEntity } from '@actual-app/api/@types/loot-core/server/api-models'; +import { TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; import { PromptGeneratorI } from '../../src/types'; export default class MockedPromptGenerator implements PromptGeneratorI { generate(): string { return 'mocked prompt'; } + + generateCategorySuggestion( + _categoryGroups: APICategoryGroupEntity[], + _transaction: TransactionEntity, + _payees: APIPayeeEntity[], + ): string { + return 'mocked category suggestion prompt'; + } + + generateSimilarRulesPrompt( + _transaction: TransactionEntity, + _rulesDescription: { + ruleName: string; + conditions: string; + categoryName: string; + categoryId: string; + index?: number; + }[], + ): string { + return 'mocked similar rules prompt'; + } } From 3284daea12b1c13da43aeffd86aa892187ef2a28 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Wed, 5 Mar 2025 23:52:38 -0500 Subject: [PATCH 05/17] Add dry run mode and unified LLM response handling --- README.md | 19 + src/config.ts | 2 +- src/container.ts | 4 +- src/llm-service.ts | 88 +++- src/prompt-generator.ts | 47 +- src/templates/category-suggestion.hbs | 8 +- src/templates/prompt.hbs | 52 ++- src/transaction-service.ts | 616 ++++++++++++++++---------- src/types.ts | 47 +- 9 files changed, 602 insertions(+), 281 deletions(-) diff --git a/README.md b/README.md index 714f89b..7441979 100644 --- a/README.md +++ b/README.md @@ -168,3 +168,22 @@ This is especially helpful for: - Specialized services that might be difficult to categorize without additional information The search results are included in the prompts sent to the LLM, helping it make more accurate category assignments or suggestions. + +## Dry Run Mode + +Enable dry run mode by setting `DRY_RUN=true` (default). In this mode: +- No transactions will be modified +- No categories will be created +- All proposed changes will be logged to console +- System will show what would happen with real execution + +To perform actual changes: +1. Set `DRY_RUN=false` +2. Ensure `SUGGEST_NEW_CATEGORIES=true` if you want new category creation +3. Run the classification process + +Dry run messages will show: +- Which transactions would be categorized +- Which rules would be applied +- What new categories would be created +- How many transactions would be affected by each change diff --git a/src/config.ts b/src/config.ts index 79581cd..9faf228 100644 --- a/src/config.ts +++ b/src/config.ts @@ -41,7 +41,7 @@ export const valueSerpApiKey = process.env.VALUESERP_API_KEY ?? ''; // Feature Flags export const suggestNewCategories = process.env.SUGGEST_NEW_CATEGORIES === 'true'; -export const dryRunNewCategories = process.env.DRY_RUN_NEW_CATEGORIES !== 'false'; // Default to true +export const dryRun = process.env.DRY_RUN !== 'false'; // Default to true unless explicitly false // Tools configuration export const enabledTools = (process.env.ENABLED_TOOLS ?? '').split(',').map((tool) => tool.trim()).filter(Boolean); diff --git a/src/container.ts b/src/container.ts index dcae930..5aea59d 100644 --- a/src/container.ts +++ b/src/container.ts @@ -9,7 +9,7 @@ import { anthropicModel, budgetId, dataDir, - dryRunNewCategories, + dryRun, e2ePassword, googleApiKey, googleBaseURL, @@ -91,7 +91,7 @@ const transactionService = new TransactionService( notGuessedTag, guessedTag, suggestNewCategories, - dryRunNewCategories, + dryRun, ); const actualAi = new ActualAiService( diff --git a/src/llm-service.ts b/src/llm-service.ts index fd74a9e..54f026d 100644 --- a/src/llm-service.ts +++ b/src/llm-service.ts @@ -2,11 +2,18 @@ import { z } from 'zod'; import { generateObject, generateText, LanguageModel } from 'ai'; import { TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; import { - CategorySuggestion, LlmModelFactoryI, LlmServiceI, ToolServiceI, + CategorySuggestion, LlmModelFactoryI, LlmServiceI, ToolServiceI, UnifiedResponse, } from './types'; import { RateLimiter } from './utils/rate-limiter'; import { PROVIDER_LIMITS } from './utils/provider-limits'; +function cleanJsonResponse(text: string): string { + // Remove markdown code fences and any surrounding text + const cleaned = text.replace(/```json\n?|\n?```/g, ''); + // Remove leading/trailing whitespace and non-JSON characters + return cleaned.trim().replace(/^[^{[]*|[^}\]]*$/g, ''); +} + export default class LlmService implements LlmServiceI { private readonly llmModelFactory: LlmModelFactoryI; @@ -18,7 +25,7 @@ export default class LlmService implements LlmServiceI { private readonly toolService?: ToolServiceI; - private isFallbackMode; + private readonly isFallbackMode; constructor( llmModelFactory: LlmModelFactoryI, @@ -233,4 +240,81 @@ export default class LlmService implements LlmServiceI { }, ); } + + public async unifiedAsk(prompt: string): Promise { + return this.rateLimiter.executeWithRateLimiting(this.provider, async () => { + try { + const { text } = await generateText({ + model: this.model, + prompt, + temperature: 0.2, + tools: this.toolService?.getTools(), + maxSteps: 3, + system: 'You must use webSearch for unfamiliar payees before suggesting categories', + }); + + // Move cleanedText declaration outside the try-catch + const cleanedText = cleanJsonResponse(text); + console.log('Cleaned LLM response:', cleanedText); + + try { + // First, try to parse as JSON + let parsed: Partial; + try { + parsed = JSON.parse(cleanedText) as Partial; + } catch { + // If not valid JSON, check if it's a simple ID + const trimmedText = cleanedText.trim().replace(/^"|"$/g, ''); + + if (/^[a-zA-Z0-9_-]+$/.test(trimmedText)) { + console.log(`LLM returned simple ID: "${trimmedText}"`); + return { + type: 'existing', + categoryId: trimmedText, + }; + } + + throw new Error('Response is neither valid JSON nor simple ID'); + } + + // Type guard validation + if (parsed.type === 'existing' && parsed.categoryId) { + return { type: 'existing', categoryId: parsed.categoryId }; + } + if (parsed.type === 'rule' && parsed.categoryId && parsed.ruleName) { + return { + type: 'rule', + categoryId: parsed.categoryId, + ruleName: parsed.ruleName, + }; + } + if (parsed.type === 'new' && parsed.newCategory) { + return { + type: 'new', + newCategory: parsed.newCategory, + }; + } + + // If the response doesn't match expected format but has a categoryId, + // default to treating it as an existing category + if (parsed.categoryId) { + console.log('LLM response missing type but has categoryId, treating as existing category'); + return { + type: 'existing', + categoryId: parsed.categoryId, + }; + } + + console.error('Invalid response structure from LLM:', parsed); + throw new Error('Invalid response format from LLM'); + } catch (parseError) { + console.error('Failed to parse LLM response:', cleanedText, parseError); + throw new Error('Invalid response format from LLM'); + } + } catch (error) { + console.error('LLM response validation failed:', error); + throw new Error('Invalid response format from LLM'); + } + }); + } } diff --git a/src/prompt-generator.ts b/src/prompt-generator.ts index d192ae9..d947216 100644 --- a/src/prompt-generator.ts +++ b/src/prompt-generator.ts @@ -1,7 +1,9 @@ -import { APICategoryEntity, APICategoryGroupEntity, APIPayeeEntity } from '@actual-app/api/@types/loot-core/server/api-models'; +import { APIPayeeEntity } from '@actual-app/api/@types/loot-core/server/api-models'; import { RuleEntity, TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; import handlebars from './handlebars-helpers'; -import { PromptGeneratorI, RuleDescription } from './types'; +import { + PromptGeneratorI, RuleDescription, APICategoryEntity, APICategoryGroupEntity, +} from './types'; import PromptTemplateException from './exceptions/prompt-template-exception'; import { hasWebSearchTool } from './config'; import { transformRulesToDescriptions } from './utils/rule-utils'; @@ -41,7 +43,7 @@ class PromptGenerator implements PromptGeneratorI { const groupsWithCategories = categoryGroups.map((group) => ({ ...group, groupName: group.name, - categories: group.categories || [], + categories: group.categories ?? [], })); try { @@ -81,7 +83,7 @@ class PromptGenerator implements PromptGeneratorI { const groupsWithCategories = categoryGroups.map((group) => ({ ...group, groupName: group.name, - categories: group.categories || [], + categories: group.categories ?? [], })); try { @@ -143,11 +145,46 @@ class PromptGenerator implements PromptGeneratorI { transformRulesToDescriptions( rules: RuleEntity[], - categories: (APICategoryEntity | APICategoryGroupEntity)[], + categories: APICategoryEntity[], payees: APIPayeeEntity[] = [], ): RuleDescription[] { return transformRulesToDescriptions(rules, categories, payees); } + + generateUnifiedPrompt( + categoryGroups: APICategoryGroupEntity[], + transaction: TransactionEntity, + payees: APIPayeeEntity[], + rules: RuleEntity[], + ): string { + const template = handlebars.compile(this.promptTemplate); + const payeeName = payees.find((p) => p.id === transaction.payee)?.name; + + const categories = categoryGroups.flatMap((group) => (group.categories ?? []).map((cat) => ({ + ...cat, + groupName: group.name, + }))); + + const rulesDescription = this.transformRulesToDescriptions( + rules, + categories as APICategoryEntity[], + payees, + ); + + return template({ + categoryGroups: categoryGroups.map((g) => ({ + ...g, + categories: g.categories ?? [], + })), + rules: rulesDescription, + amount: Math.abs(transaction.amount), + type: transaction.amount > 0 ? 'Income' : 'Expense', + description: transaction.notes, + payee: payeeName, + importedPayee: transaction.imported_payee, + date: transaction.date, + }); + } } export default PromptGenerator; diff --git a/src/templates/category-suggestion.hbs b/src/templates/category-suggestion.hbs index f480c10..f3fc207 100644 --- a/src/templates/category-suggestion.hbs +++ b/src/templates/category-suggestion.hbs @@ -48,4 +48,10 @@ IMPORTANT: Your response MUST be: - No trailing commas Example of VALID response: -{"name": "Pet Supplies", "groupName": "Digital Assets", "groupIsNew": true} \ No newline at end of file +{"name": "Pet Supplies", "groupName": "Digital Assets", "groupIsNew": true} + +IMPORTANT: When suggesting categories: +- Use consistent naming for similar expenses +- Prefer existing group names when appropriate +- Use the most specific yet concise names +- Avoid minor variations of the same category \ No newline at end of file diff --git a/src/templates/prompt.hbs b/src/templates/prompt.hbs index 762a299..85db3cd 100644 --- a/src/templates/prompt.hbs +++ b/src/templates/prompt.hbs @@ -1,10 +1,4 @@ -I want to categorize the given bank transaction into one of the following categories: -{{#each categoryGroups}} -GROUP: {{name}} (ID: "{{id}}") -{{#each categories}} -* {{name}} (ID: "{{id}}") -{{/each}} -{{/each}} +I want to categorize the given bank transaction. Transaction details: * Amount: {{amount}} @@ -17,6 +11,46 @@ Transaction details: {{^}} * Payee: {{importedPayee}} {{/if}} +{{#if date}} +* Date: {{date}} +{{/if}} + +Existing categories by group: +{{#each categoryGroups}} +GROUP: {{name}} (ID: "{{id}}") +{{#each categories}} +* {{name}} (ID: "{{id}}") +{{/each}} +{{/each}} + +{{#if rules.length}} +Existing Rules: +{{#each rules}} +{{incIndex @index}}. {{ruleName}} → {{categoryName}} + Conditions: {{#each conditions}}{{field}} {{op}} {{value}}{{#unless @last}}, {{/unless}}{{/each}} +{{/each}} +{{/if}} + +IMPORTANT: You MUST respond with ONLY a valid JSON object using this structure: +{ + "type": "existing"|"new"|"rule", + "categoryId": "string", // Required for existing category or rule match + "ruleName": "string", // Required if matching rule + "newCategory": { // Required if suggesting new category + "name": "string", + "groupName": "string", + "groupIsNew": boolean + } +} + +DO NOT output any text before or after the JSON. Your entire response must be a valid, parsable JSON object. + +Examples: +{"type": "existing", "categoryId": "abc123"} +{"type": "rule", "categoryId": "def456", "ruleName": "Coffee Shop"} +{"type": "new", "newCategory": {"name": "Pet Supplies", "groupName": "Pets", "groupIsNew": true}} + +Extra rules: +* If the transaction is a Credit Card Payment, categorize it as "Transfer" unless it is a fee. +* Flowers can go in the "Gift" category. -RESPOND ONLY WITH A CATEGORY ID from the list above. Do not write anything else. -If you're not sure which category to use, respond with "uncategorized". diff --git a/src/transaction-service.ts b/src/transaction-service.ts index 503e03c..cddfaf3 100644 --- a/src/transaction-service.ts +++ b/src/transaction-service.ts @@ -1,5 +1,13 @@ -import { - ActualApiServiceI, LlmServiceI, PromptGeneratorI, TransactionServiceI, +import type { + CategoryEntity, + TransactionEntity, +} from '@actual-app/api/@types/loot-core/types/models'; +import type { + ActualApiServiceI, + LlmServiceI, + PromptGeneratorI, + TransactionServiceI, + CategorySuggestion, } from './types'; const LEGACY_NOTES_NOT_GUESSED = 'actual-ai could not guess this category'; @@ -19,7 +27,7 @@ class TransactionService implements TransactionServiceI { private readonly suggestNewCategories: boolean; - private readonly dryRunNewCategories: boolean; + private readonly dryRun: boolean; constructor( actualApiClient: ActualApiServiceI, @@ -28,7 +36,7 @@ class TransactionService implements TransactionServiceI { notGuessedTag: string, guessedTag: string, suggestNewCategories = false, - dryRunNewCategories = true, + dryRun = true, ) { this.actualApiService = actualApiClient; this.llmService = llmService; @@ -36,7 +44,7 @@ class TransactionService implements TransactionServiceI { this.notGuessedTag = notGuessedTag; this.guessedTag = guessedTag; this.suggestNewCategories = suggestNewCategories; - this.dryRunNewCategories = dryRunNewCategories; + this.dryRun = dryRun; } appendTag(notes: string, tag: string): string { @@ -81,6 +89,12 @@ class TransactionService implements TransactionServiceI { } async processTransactions(): Promise { + if (this.dryRun) { + console.log('=== DRY RUN MODE ==='); + console.log('No changes will be made to transactions or categories'); + console.log('====================='); + } + const [categoryGroups, categories, payees, transactions, accounts, rules] = await Promise.all([ this.actualApiService.getCategoryGroups(), this.actualApiService.getCategories(), @@ -99,23 +113,28 @@ class TransactionService implements TransactionServiceI { && transaction.starting_balance_flag !== true && transaction.imported_payee !== null && transaction.imported_payee !== '' - && (transaction.notes === null || (!transaction.notes?.includes(this.notGuessedTag))) + // && !transaction.notes?.includes(this.notGuessedTag) && !transaction.is_parent && !accountsToSkip.includes(transaction.account), ); - console.log(`Found ${uncategorizedTransactions.length} transactions to process`); - const categoryIds = categories.map((category) => category.id); - categoryIds.push('uncategorized'); + if (uncategorizedTransactions.length === 0) { + console.log('No uncategorized transactions to process'); + return; + } + + console.log(`Found ${uncategorizedTransactions.length} uncategorized transactions`); - // Track suggested categories to avoid duplicates and for creating later + // Track suggested new categories const suggestedCategories = new Map(); - // Process transactions in batches to avoid hitting rate limits + // Process transactions in batches for ( let batchStart = 0; batchStart < uncategorizedTransactions.length; @@ -126,199 +145,53 @@ class TransactionService implements TransactionServiceI { const batch = uncategorizedTransactions.slice(batchStart, batchEnd); - for (let i = 0; i < batch.length; i++) { - const transaction = batch[i]; - const globalIndex = batchStart + i; - console.log(`${globalIndex + 1}/${uncategorizedTransactions.length} Processing transaction ${transaction.imported_payee} / ${transaction.notes} / ${transaction.amount}`); + await batch.reduce(async (previousPromise, transaction, batchIndex) => { + await previousPromise; + const globalIndex = batchStart + batchIndex; + console.log( + `${globalIndex + 1}/${uncategorizedTransactions.length} Processing transaction '${transaction.imported_payee}'`, + ); try { - // Check if there are any rules that might apply to this payee - const payeeRules = transaction.payee - ? await this.actualApiService.getPayeeRules(transaction.payee) - : []; - - // Log if we found any matching rules - if (payeeRules.length > 0) { - console.log(`Found ${payeeRules.length} rules associated with payee ID ${transaction.payee}`); - - // Find a rule that might set a category - const categoryRule = payeeRules.find((rule) => rule.actions.some( - (action) => 'field' in action && action.field === 'category' && action.op === 'set', - )); - - if (categoryRule) { - // Extract the category ID from the rule - const categoryAction = categoryRule.actions.find( - (action) => 'field' in action && action.field === 'category' && action.op === 'set', - ); - - if (categoryAction && 'value' in categoryAction && typeof categoryAction.value === 'string') { - const categoryId = categoryAction.value; - const category = categories.find((c) => c.id === categoryId); - - if (category) { - console.log(`Rule suggests category: ${category.name}`); - await this.actualApiService.updateTransactionNotesAndCategory( - transaction.id, - this.appendTag(transaction.notes ?? '', `${this.guessedTag} (from rule)`), - categoryId, - ); - // Skip to next transaction since we've categorized this one - continue; - } - } - } - } - - // If no direct payee rule was found, check if transaction is similar to any existing rule - if (this.llmService.findSimilarRules) { - const rulesDescription = this.promptGenerator.transformRulesToDescriptions( - rules, - categories, - payees, - ); + const prompt = this.promptGenerator.generateUnifiedPrompt( + categoryGroups, + transaction, + payees, + rules, + ); - const similarRulesPrompt = this.promptGenerator.generateSimilarRulesPrompt( + const response = await this.llmService.unifiedAsk(prompt); + + if (response.type === 'rule' && response.ruleName && response.categoryId) { + await this.handleRuleMatch(transaction, { + ruleName: response.ruleName, + categoryId: response.categoryId, + }); + } else if (response.type === 'existing' && response.categoryId) { + await this.handleExistingCategory(transaction, { + categoryId: response.categoryId, + }, categories); + } else if (response.type === 'new' && response.newCategory) { + this.trackNewCategory( transaction, - rulesDescription, + response.newCategory, + suggestedCategories, ); - - const similarRuleResult = await this.llmService.findSimilarRules( - transaction, - similarRulesPrompt, + } else { + console.warn(`Unexpected response format: ${JSON.stringify(response)}`); + await this.actualApiService.updateTransactionNotes( + transaction.id, + this.appendTag(transaction.notes ?? '', this.notGuessedTag), ); - - if (similarRuleResult) { - const { categoryId, ruleName } = similarRuleResult; - const category = categories.find((c) => c.id === categoryId); - - if (category) { - console.log(`Transaction similar to rule "${ruleName}", suggesting category: ${category.name}`); - await this.actualApiService.updateTransactionNotesAndCategory( - transaction.id, - this.appendTag(transaction.notes ?? '', `${this.guessedTag} (similar to rule)`), - categoryId, - ); - // Skip to next transaction since we've categorized this one - continue; - } - } - } - - const prompt = this.promptGenerator.generate(categoryGroups, transaction, payees); - const guess = await this.llmService.ask(prompt, categoryIds); - let guessCategory = categories.find((category) => category.id === guess); - - if (!guessCategory) { - guessCategory = categories.find((category) => category.name === guess); - if (guessCategory) { - console.warn(`${globalIndex + 1}/${uncategorizedTransactions.length} LLM guessed category name instead of ID. LLM guess: ${guess}`); - } } - if (!guessCategory) { - guessCategory = categories.find((category) => guess.includes(category.id)); - if (guessCategory) { - console.warn(`${globalIndex + 1}/${uncategorizedTransactions.length} Found category ID in LLM guess, but it wasn't 1:1. LLM guess: ${guess}`); - } - } - - if (!guessCategory) { - console.warn(`${globalIndex + 1}/${uncategorizedTransactions.length} LLM could not classify the transaction. LLM guess: ${guess}`); - - // If suggestNewCategories is enabled, try to get a new category suggestion - if (this.suggestNewCategories) { - const newCategoryPrompt = this.promptGenerator.generateCategorySuggestion( - categoryGroups, - transaction, - payees, - ); - const categorySuggestion = await this.llmService.askForCategorySuggestion( - newCategoryPrompt, - ); - - if ( - categorySuggestion?.name - && categorySuggestion.groupName - ) { - console.log(`${globalIndex + 1}/${uncategorizedTransactions.length} Suggested new category: ${categorySuggestion.name} in group ${categorySuggestion.groupName}`); - - // Find or create category group - let groupId: string; - if (categorySuggestion.groupIsNew) { - if (this.dryRunNewCategories) { - console.log(`Dry run: Would create new category group "${categorySuggestion.groupName}"`); - groupId = 'dry-run-group-id'; - } else { - groupId = await this.actualApiService.createCategoryGroup( - categorySuggestion.groupName, - ); - console.log(`Created new category group "${categorySuggestion.groupName}" with ID ${groupId}`); - } - } else { - const existingGroup = categoryGroups.find( - (g) => g.name.toLowerCase() === categorySuggestion.groupName.toLowerCase(), - ); - groupId = existingGroup?.id - ?? (this.dryRunNewCategories ? 'dry-run-group-id' : await this.actualApiService.createCategoryGroup( - categorySuggestion.groupName, - )); - } - - // Then create category and assign transactions... - let newCategoryId: string | null = null; - if (!this.dryRunNewCategories) { - newCategoryId = await this.actualApiService.createCategory( - categorySuggestion.name, - groupId, - ); - console.log(`Created new category "${categorySuggestion.name}" with ID ${newCategoryId}`); - } - - // Handle transaction assignments - if (newCategoryId) { - await Promise.all( - suggestedCategories.get( - categorySuggestion.name.toLowerCase(), - )?.transactions.map(async (transactionId) => { - const uncategorizedTransaction = uncategorizedTransactions.find( - (t) => t.id === transactionId, - ); - if (uncategorizedTransaction) { - await this.actualApiService.updateTransactionNotesAndCategory( - uncategorizedTransaction.id, - this.appendTag(uncategorizedTransaction.notes ?? '', this.guessedTag), - newCategoryId, - ); - console.log(`Assigned transaction ${uncategorizedTransaction.id} to new category ${categorySuggestion?.name}`); - } - }) ?? [], - ); - } - } else { - // Handle invalid/missing category suggestion - console.log('No valid category suggestion received'); - await this.actualApiService.updateTransactionNotes( - transaction.id, - this.appendTag(transaction.notes ?? '', this.notGuessedTag), - ); - } - } else { - await this.actualApiService.updateTransactionNotes(transaction.id, this.appendTag(transaction.notes ?? '', this.notGuessedTag)); - } - continue; - } - console.log(`${globalIndex + 1}/${uncategorizedTransactions.length} Guess: ${guessCategory.name}`); - - await this.actualApiService.updateTransactionNotesAndCategory( + } catch (error) { + console.error(`Error processing transaction ${globalIndex + 1}:`, error); + await this.actualApiService.updateTransactionNotes( transaction.id, - this.appendTag(transaction.notes ?? '', this.guessedTag), - guessCategory.id, + this.appendTag(transaction.notes ?? '', this.notGuessedTag), ); - } catch (error) { - console.error(`Error processing transaction ${globalIndex + 1}/${uncategorizedTransactions.length}:`, error); - // Continue with next transaction } - } + }, Promise.resolve()); // Add a small delay between batches to avoid overwhelming the API if (batchEnd < uncategorizedTransactions.length) { @@ -330,54 +203,327 @@ class TransactionService implements TransactionServiceI { } // Create new categories if not in dry run mode - if (this.suggestNewCategories && !this.dryRunNewCategories && suggestedCategories.size > 0) { - console.log(`Creating ${suggestedCategories.size} new categories`); - - // Use Promise.all with map for async operations - await Promise.all( - Array.from(suggestedCategories.entries()).map(async ([_, suggestion]) => { - try { - const newCategoryId = await this.actualApiService.createCategory( - suggestion.name, - suggestion.groupId, - ); + if (this.suggestNewCategories && suggestedCategories.size > 0) { + // Optimize categories before applying/reporting + const optimizedCategories = this.optimizeCategorySuggestions(suggestedCategories); + + if (this.dryRun) { + console.log(`\nDRY RUN: Would create ${optimizedCategories.size} new categories after optimization:`); + Array.from(optimizedCategories.entries()).forEach(([_, suggestion]) => { + console.log( + `- ${suggestion.name} in ${suggestion.groupIsNew ? 'new' : 'existing'} group "${suggestion.groupName}"`, + `for ${suggestion.transactions.length} transactions`, + ); + }); + } else { + console.log(`Creating ${optimizedCategories.size} optimized categories`); + + // Use optimized categories instead of original suggestions + await Promise.all( + Array.from(optimizedCategories.entries()).map(async ([_, suggestion]) => { + try { + // First, ensure we have a group ID + let groupId: string; + if (suggestion.groupIsNew) { + groupId = await this.actualApiService.createCategoryGroup(suggestion.groupName); + console.log(`Created new category group "${suggestion.groupName}" with ID ${groupId}`); + } else { + // Find existing group with matching name + const existingGroup = categoryGroups.find( + (g) => g.name.toLowerCase() === suggestion.groupName.toLowerCase(), + ); + if (existingGroup) { + groupId = existingGroup.id; + } else { + // Create group if not found + groupId = await this.actualApiService.createCategoryGroup(suggestion.groupName); + console.log(`Created category group "${suggestion.groupName}" with ID ${groupId}`); + } + } + + const newCategoryId = await this.actualApiService.createCategory( + suggestion.name, + groupId, + ); - console.log(`Created new category "${suggestion.name}" with ID ${newCategoryId}`); + console.log(`Created new category "${suggestion.name}" with ID ${newCategoryId}`); - // Use Promise.all with map for nested async operations - await Promise.all( - suggestion.transactions.map(async (transactionId) => { - const transaction = uncategorizedTransactions.find((t) => t.id === transactionId); - if (transaction) { + // Use Promise.all with map for nested async operations + await Promise.all( + suggestion.transactions.map(async (transaction) => { await this.actualApiService.updateTransactionNotesAndCategory( - transactionId, + transaction.id, this.appendTag(transaction.notes ?? '', this.guessedTag), newCategoryId, ); - console.log(`Assigned transaction ${transactionId} to new category ${suggestion.name}`); - } - }), - ); - } catch (error) { - console.error(`Error creating category ${suggestion.name}:`, error); + console.log(`Assigned transaction ${transaction.id} to new category ${suggestion.name}`); + }), + ); + } catch (error) { + console.error(`Error creating category ${suggestion.name}:`, error); + } + }), + ); + } + } + } + + private async handleRuleMatch( + transaction: TransactionEntity, + response: { categoryId: string; ruleName: string }, + ) { + if (this.dryRun) { + console.log(`DRY RUN: Would assign transaction ${transaction.id} to category ${response.categoryId} via rule ${response.ruleName}`); + return; + } + + await this.actualApiService.updateTransactionNotesAndCategory( + transaction.id, + this.appendTag(transaction.notes ?? '', `${this.guessedTag} (rule: ${response.ruleName})`), + response.categoryId, + ); + } + + private async handleExistingCategory( + transaction: TransactionEntity, + response: { categoryId: string }, + categories: CategoryEntity[], + ) { + const category = categories.find((c) => c.id === response.categoryId); + if (!category) return; + + if (this.dryRun) { + console.log(`DRY RUN: Would assign transaction ${transaction.id} to existing category ${category.name}`); + return; + } + + console.log(`Using existing category: ${category.name}`); + await this.actualApiService.updateTransactionNotesAndCategory( + transaction.id, + this.appendTag(transaction.notes ?? '', this.guessedTag), + response.categoryId, + ); + } + + private trackNewCategory( + transaction: TransactionEntity, + newCategory: CategorySuggestion, + suggestedCategories: Map, + ) { + const categoryKey = `${newCategory.groupName}:${newCategory.name}`; + + const existing = suggestedCategories.get(categoryKey); + if (existing) { + existing.transactions.push(transaction); + } else { + suggestedCategories.set(categoryKey, { + ...newCategory, + transactions: [transaction], + }); + } + } + + // Add this new method to optimize category suggestions + private calculateNameSimilarity(name1: string, name2: string): number { + // Normalize the strings for comparison + const a = name1.toLowerCase().replace(/[^a-z0-9\s]/g, ' ').replace(/\s+/g, ' ').trim(); + const b = name2.toLowerCase().replace(/[^a-z0-9\s]/g, ' ').replace(/\s+/g, ' ').trim(); + + if (a === b) return 1.0; + + // Check for exact word matches + const words1 = new Set(a.split(' ')); + const words2 = new Set(b.split(' ')); + + // Calculate Jaccard similarity for words + const intersection = new Set([...words1].filter((x) => words2.has(x))); + const union = new Set([...words1, ...words2]); + + // Weight for word overlap + const wordSimilarity = intersection.size / union.size; + + // Jaro-Winkler for character-level similarity + const jaro = (s1: string, s2: string) => { + const matchDistance = Math.floor(Math.max(s1.length, s2.length) / 2) - 1; + const s1Matches = new Array(s1.length).fill(false); + const s2Matches = new Array(s2.length).fill(false); + + let matches = 0; + for (let i = 0; i < s1.length; i++) { + const start = Math.max(0, i - matchDistance); + const end = Math.min(i + matchDistance + 1, s2.length); + for (let j = start; j < end; j++) { + if (!s2Matches[j] && s1[i] === s2[j]) { + s1Matches[i] = true; + s2Matches[j] = true; + matches++; + break; } - }), - ); - } else if ( - this.suggestNewCategories && this.dryRunNewCategories && suggestedCategories.size > 0 - ) { - // Split the longer line to avoid length error - console.log( - `Dry run: Would create ${suggestedCategories.size} new categories:`, - ); + } + } - // No need for async here, so we can use forEach - Array.from(suggestedCategories.entries()).forEach(([_, suggestion]) => { - console.log( - `- ${suggestion.name} (in group ${suggestion.groupId}) for ${suggestion.transactions.length} transactions`, - ); + if (matches === 0) return 0; + + let transpositions = 0; + let k = 0; + for (let i = 0; i < s1.length; i++) { + if (s1Matches[i]) { + while (!s2Matches[k]) k++; + if (s1[i] !== s2[k]) transpositions++; + k++; + } + } + + return ( + matches / s1.length + matches / s2.length + (matches - transpositions / 2) / matches) / 3; + }; + + const charSimilarity = jaro(a, b); + + // Combine word-level and character-level similarity + return 0.6 * wordSimilarity + 0.4 * charSimilarity; + } + + private chooseBestCategoryName(names: string[]): string { + if (names.length === 1) return names[0]; + + // Count frequency of words across all names + const wordFrequency = new Map(); + const nameWords = names.map((name) => { + const words = name.toLowerCase().split(/\s+/); + words.forEach((word) => { + wordFrequency.set(word, (wordFrequency.get(word) ?? 0) + 1); }); + return words; + }); + + // Score each name based on word frequency (more common words are better) + const scores = names.map((name, i) => { + const words = nameWords[i]; + const freqScore = words.reduce( + (sum, word) => sum + wordFrequency.get(word)!, + 0, + ) / words.length; + + // Prefer names that are in the sweet spot length (not too short, not too long) + const lengthScore = 1 / (1 + Math.abs(words.length - 2)); + + return { name, score: freqScore * 0.7 + lengthScore * 0.3 }; + }); + + // Sort by score (descending) and return the best + scores.sort((a, b) => b.score - a.score); + return scores[0].name; + } + + private optimizeCategorySuggestions( + suggestedCategories: Map, + ): Map { + console.log('Optimizing category suggestions...'); + + // Convert suggestions to array. + const suggestions = Array.from(suggestedCategories.values()); + + // Cluster suggestions across groups based on name similarity. + const used = new Array(suggestions.length).fill(false); + const clusters: { suggestions: typeof suggestions }[] = []; + for (let i = 0; i < suggestions.length; i++) { + if (used[i]) continue; + const cluster = [suggestions[i]]; + used[i] = true; + for (let j = i + 1; j < suggestions.length; j++) { + if (used[j]) continue; + // Dynamic threshold: shorter names need higher similarity. + const minLength = Math.min(suggestions[i].name.length, suggestions[j].name.length); + const baseThreshold = 0.7; + const dynamicThreshold = baseThreshold + (1 / Math.max(5, minLength)) * 0.3; + const sim = this.calculateNameSimilarity(suggestions[i].name, suggestions[j].name); + if (sim >= dynamicThreshold) { + cluster.push(suggestions[j]); + used[j] = true; + } + } + clusters.push({ suggestions: cluster }); } + + // Create optimized categories from clusters. + const optimizedCategories = new Map(); + clusters.forEach(({ suggestions: cluster }) => { + // Merge transactions and original names. + const mergedTransactions = cluster.flatMap((s) => s.transactions); + const originalNames = cluster.map((s) => s.name); + const bestName = this.chooseBestCategoryName(originalNames); + // Choose representative group name from frequency. + const groupCount = new Map(); + cluster.forEach((s) => { + const grp = s.groupName; + groupCount.set(grp, (groupCount.get(grp) ?? 0) + 1); + }); + let repGroup = cluster[0].groupName; + let maxCount = 0; + groupCount.forEach((cnt, grp) => { + if (cnt > maxCount) { + maxCount = cnt; + repGroup = grp; + } + }); + // Determine groupIsNew: if any in cluster is new, mark true. + const groupIsNew = cluster.some((s) => s.groupIsNew); + optimizedCategories.set(`${repGroup}:${bestName}`, { + name: bestName, + groupName: repGroup, + groupIsNew, + groupId: undefined, + transactions: mergedTransactions, + originalNames, + }); + }); + + console.log(`Optimized from ${suggestions.length} to ${optimizedCategories.size} categories`); + optimizedCategories.forEach((category) => { + if (category.originalNames.length > 1) { + console.log(`Merged categories ${category.originalNames.join(', ')} into "${category.name}"`); + } + }); + + // Return map without originalNames. + return new Map( + Array.from(optimizedCategories.entries()).map(([key, value]) => [ + key, + { + name: value.name, + groupName: value.groupName, + groupIsNew: value.groupIsNew, + groupId: value.groupId, + transactions: value.transactions, + }, + ]), + ); } } diff --git a/src/types.ts b/src/types.ts index e01087b..b9a6720 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,11 +1,16 @@ import { LanguageModel, Tool } from 'ai'; import { APIAccountEntity, - APICategoryEntity, - APICategoryGroupEntity, + APICategoryEntity as ImportedAPICategoryEntity, + APICategoryGroupEntity as ImportedAPICategoryGroupEntity, APIPayeeEntity, } from '@actual-app/api/@types/loot-core/server/api-models'; -import { TransactionEntity, RuleEntity } from '@actual-app/api/@types/loot-core/types/models'; +import { + TransactionEntity, RuleEntity, CategoryEntity, CategoryGroupEntity, +} from '@actual-app/api/@types/loot-core/types/models'; + +export type APICategoryEntity = ImportedAPICategoryEntity | CategoryEntity; +export type APICategoryGroupEntity = ImportedAPICategoryGroupEntity | CategoryGroupEntity; export interface LlmModelI { ask(prompt: string, possibleAnswers: string[]): Promise; @@ -85,15 +90,15 @@ export interface CategorySuggestion { groupIsNew: boolean; } -export interface LlmServiceI { - ask(prompt: string, categoryIds: string[]): Promise; - - askForCategorySuggestion(prompt: string): Promise; +export interface UnifiedResponse { + type: 'existing' | 'new' | 'rule'; + categoryId?: string; + ruleName?: string; + newCategory?: CategorySuggestion; +} - findSimilarRules( - transaction: TransactionEntity, - prompt: string - ): Promise<{ categoryId: string; ruleName: string } | null>; +export interface LlmServiceI { + unifiedAsk(prompt: string): Promise; } export interface ToolServiceI { @@ -101,26 +106,16 @@ export interface ToolServiceI { } export interface PromptGeneratorI { - generate( + generateUnifiedPrompt( categoryGroups: APICategoryGroupEntity[], transaction: TransactionEntity, payees: APIPayeeEntity[], - ): string - - generateCategorySuggestion( - categoryGroups: APICategoryGroupEntity[], - transaction: TransactionEntity, - payees: APIPayeeEntity[], - ): string - - generateSimilarRulesPrompt( - transaction: TransactionEntity & { payeeName?: string }, - rulesDescription: RuleDescription[], - ): string + rules: RuleEntity[], + ): string; transformRulesToDescriptions( rules: RuleEntity[], - categories: (APICategoryEntity | APICategoryGroupEntity)[], + categories: APICategoryEntity[], payees: APIPayeeEntity[], - ): RuleDescription[] + ): RuleDescription[]; } From 0a048cd6794bcb7bfab00a72ef052aa419ddea17 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Fri, 7 Mar 2025 12:23:38 -0500 Subject: [PATCH 06/17] Improve JSON parsing and category logging in transaction processing --- src/llm-service.ts | 18 +++++++++++++++--- src/transaction-service.ts | 8 ++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/llm-service.ts b/src/llm-service.ts index 54f026d..cab7742 100644 --- a/src/llm-service.ts +++ b/src/llm-service.ts @@ -9,9 +9,15 @@ import { PROVIDER_LIMITS } from './utils/provider-limits'; function cleanJsonResponse(text: string): string { // Remove markdown code fences and any surrounding text - const cleaned = text.replace(/```json\n?|\n?```/g, ''); - // Remove leading/trailing whitespace and non-JSON characters - return cleaned.trim().replace(/^[^{[]*|[^}\]]*$/g, ''); + let cleaned = text.replace(/```json\n?|\n?```/g, ''); + cleaned = cleaned.trim(); + + // Remove leading characters up to first JSON structure character + cleaned = cleaned.replace(/^[^{[]*?([{[])/, '$1'); + // Remove trailing characters after last JSON structure character + cleaned = cleaned.replace(/([}\]])[^}\]]*$/, '$1'); + + return cleaned.trim(); } export default class LlmService implements LlmServiceI { @@ -304,6 +310,12 @@ export default class LlmService implements LlmServiceI { categoryId: parsed.categoryId, }; } + if (parsed && typeof parsed === 'string') { + return { + type: 'existing', + categoryId: parsed, + }; + } console.error('Invalid response structure from LLM:', parsed); throw new Error('Invalid response format from LLM'); diff --git a/src/transaction-service.ts b/src/transaction-service.ts index cddfaf3..78cb658 100644 --- a/src/transaction-service.ts +++ b/src/transaction-service.ts @@ -166,7 +166,7 @@ class TransactionService implements TransactionServiceI { await this.handleRuleMatch(transaction, { ruleName: response.ruleName, categoryId: response.categoryId, - }); + }, categories); } else if (response.type === 'existing' && response.categoryId) { await this.handleExistingCategory(transaction, { categoryId: response.categoryId, @@ -271,9 +271,13 @@ class TransactionService implements TransactionServiceI { private async handleRuleMatch( transaction: TransactionEntity, response: { categoryId: string; ruleName: string }, + categories: CategoryEntity[], ) { + const category = categories.find((c) => c.id === response.categoryId); + const categoryName = category ? category.name : 'Unknown Category'; + if (this.dryRun) { - console.log(`DRY RUN: Would assign transaction ${transaction.id} to category ${response.categoryId} via rule ${response.ruleName}`); + console.log(`DRY RUN: Would assign transaction ${transaction.id} to category "${categoryName}" (${response.categoryId}) via rule ${response.ruleName}`); return; } From ecd871c075c016ab451354e8618140204b5dd078 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Fri, 7 Mar 2025 19:19:04 -0500 Subject: [PATCH 07/17] Refactor prompt generation and LLM service & update tests --- src/config.ts | 6 - src/container.ts | 4 - src/llm-service.ts | 248 ++-------------- src/prompt-generator.ts | 130 +------- src/templates/category-suggestion.hbs | 57 ---- src/templates/prompt.hbs | 3 + src/templates/similar-rules.hbs | 38 --- src/transaction-service.ts | 17 +- src/types.ts | 10 +- src/utils/json-utils.ts | 90 ++++++ tests/actual-ai.test.ts | 20 +- tests/prompt-generator.test.ts | 277 +++++++----------- tests/test-doubles/given/given-actual-data.ts | 31 +- .../in-memory-actual-api-service.ts | 8 +- tests/test-doubles/mocked-llm-service.ts | 49 +++- tests/test-doubles/mocked-prompt-generator.ts | 30 +- 16 files changed, 353 insertions(+), 665 deletions(-) delete mode 100644 src/templates/category-suggestion.hbs delete mode 100644 src/templates/similar-rules.hbs create mode 100644 src/utils/json-utils.ts diff --git a/src/config.ts b/src/config.ts index 9faf228..6053b85 100644 --- a/src/config.ts +++ b/src/config.ts @@ -2,8 +2,6 @@ import dotenv from 'dotenv'; import fs from 'fs'; const defaultPromptTemplate = fs.readFileSync('./src/templates/prompt.hbs', 'utf8').trim(); -const defaultCategorySuggestionTemplate = fs.readFileSync('./src/templates/category-suggestion.hbs', 'utf8').trim(); -const defaultSimilarRulesTemplate = fs.readFileSync('./src/templates/similar-rules.hbs', 'utf8').trim(); dotenv.config(); @@ -30,10 +28,6 @@ export const dataDir = '/tmp/actual-ai/'; export const promptTemplate = process.env.PROMPT_TEMPLATE ?? defaultPromptTemplate; export const notGuessedTag = process.env.NOT_GUESSED_TAG ?? '#actual-ai-miss'; export const guessedTag = process.env.GUESSED_TAG ?? '#actual-ai'; -export const categorySuggestionTemplate = process.env.CATEGORY_SUGGESTION_TEMPLATE - ?? defaultCategorySuggestionTemplate; -export const similarRulesTemplate = process.env.SIMILAR_RULES_TEMPLATE - ?? defaultSimilarRulesTemplate; export const groqApiKey = process.env.GROQ_API_KEY ?? ''; export const groqModel = process.env.GROQ_MODEL ?? 'llama-3.3-70b-versatile'; export const groqBaseURL = process.env.GROQ_BASE_URL ?? 'https://api.groq.com/openai/v1'; diff --git a/src/container.ts b/src/container.ts index 5aea59d..5c963c2 100644 --- a/src/container.ts +++ b/src/container.ts @@ -27,8 +27,6 @@ import { openaiModel, password, promptTemplate, - categorySuggestionTemplate, - similarRulesTemplate, serverURL, suggestNewCategories, syncAccountsBeforeClassify, @@ -75,8 +73,6 @@ const actualApiService = new ActualApiService( const promptGenerator = new PromptGenerator( promptTemplate, - categorySuggestionTemplate, - similarRulesTemplate, ); const llmService = new LlmService( diff --git a/src/llm-service.ts b/src/llm-service.ts index cab7742..1ddabc6 100644 --- a/src/llm-service.ts +++ b/src/llm-service.ts @@ -1,24 +1,10 @@ -import { z } from 'zod'; import { generateObject, generateText, LanguageModel } from 'ai'; -import { TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; import { - CategorySuggestion, LlmModelFactoryI, LlmServiceI, ToolServiceI, UnifiedResponse, + LlmModelFactoryI, LlmServiceI, ToolServiceI, UnifiedResponse, } from './types'; import { RateLimiter } from './utils/rate-limiter'; import { PROVIDER_LIMITS } from './utils/provider-limits'; - -function cleanJsonResponse(text: string): string { - // Remove markdown code fences and any surrounding text - let cleaned = text.replace(/```json\n?|\n?```/g, ''); - cleaned = cleaned.trim(); - - // Remove leading characters up to first JSON structure character - cleaned = cleaned.replace(/^[^{[]*?([{[])/, '$1'); - // Remove trailing characters after last JSON structure character - cleaned = cleaned.replace(/([}\]])[^}\]]*$/, '$1'); - - return cleaned.trim(); -} +import { parseLlmResponse } from './utils/json-utils'; export default class LlmService implements LlmServiceI { private readonly llmModelFactory: LlmModelFactoryI; @@ -73,143 +59,50 @@ export default class LlmService implements LlmServiceI { } } - public async ask(prompt: string, categoryIds: string[]): Promise { + public async ask(prompt: string, categoryIds?: string[]): Promise { try { console.log(`Making LLM request to ${this.provider}${this.isFallbackMode ? ' (fallback mode)' : ''}`); + // In fallback mode, return a UnifiedResponse with the string as categoryId if (this.isFallbackMode) { - return await this.askUsingFallbackModel(prompt); + const result = await this.askUsingFallbackModel(prompt); + return { + type: 'existing', + categoryId: result.replace(/(\r\n|\n|\r|"|')/gm, ''), + }; } - return await this.askWithEnum(prompt, categoryIds); - } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); - console.error(`Error during LLM request to ${this.provider}: ${errorMsg}`); - throw error; - } - } - - public async askForCategorySuggestion( - prompt: string, - ): Promise { - try { - console.log( - `Making LLM request for category suggestion to ${this.provider}${this.isFallbackMode ? ' (fallback mode)' : ''}`, - ); - - const categorySchema = z.object({ - name: z.string(), - groupName: z.string(), - groupIsNew: z.boolean(), - }); - - const response = await this.rateLimiter.executeWithRateLimiting( - this.provider, - async () => { - const { text, steps } = await generateText({ - model: this.model, - prompt, - temperature: 0.2, - tools: this.toolService?.getTools(), - maxSteps: 3, - system: 'You must use webSearch for unfamiliar payees before suggesting categories', - }); - - console.log('Generation steps:', steps.map((step) => ({ - text: step.text, - toolCalls: step.toolCalls, - toolResults: step.toolResults, - }))); - - // Parse the JSON response from the text - try { - const parsedResponse = JSON.parse(text) as unknown; - // Validate against schema - const result = categorySchema.safeParse(parsedResponse); - return result.success ? result.data : null; - } catch (e) { - console.error('Failed to parse JSON response:', e); - return null; - } - }, - ); - - if (response) { + // If categoryIds are provided, use enum selection and return as UnifiedResponse + if (categoryIds && categoryIds.length > 0) { + const result = await this.askWithEnum(prompt, categoryIds); return { - name: response.name, - groupName: response.groupName, - groupIsNew: response.groupIsNew, + type: 'existing', + categoryId: result, }; } - console.warn('LLM response did not contain valid category suggestion format'); - return null; - } catch (error) { - console.error('Error while getting category suggestion:', error); - return null; - } - } - - /** - * Analyze if a transaction is similar to any existing rule and suggest a category - * @param transaction The transaction to analyze - * @param rules List of existing rules in the system - * @param categories List of categories for reference - * @param prompt The prompt to use for finding similar rules - * @returns A suggested category ID if similar rules exist, null otherwise - */ - public async findSimilarRules( - transaction: TransactionEntity, - prompt: string, - ): Promise<{ categoryId: string; ruleName: string } | null> { - try { - console.log( - `Checking if transaction "${transaction.imported_payee}" matches any existing rules`, - ); - - // console.log('Prompt:', prompt.slice(0, 300)); - - return this.rateLimiter.executeWithRateLimiting< - { categoryId: string; ruleName: string } | null>( - this.provider, - async () => { - const { text, steps } = await generateText({ + // Otherwise, handle unified response + return this.rateLimiter.executeWithRateLimiting(this.provider, async () => { + try { + const { text } = await generateText({ model: this.model, prompt, - temperature: 0.1, + temperature: 0.2, tools: this.toolService?.getTools(), maxSteps: 3, - system: 'You must respond with pong if you receive don\'t have an answer', + system: 'You must use webSearch for unfamiliar payees before suggesting categories', }); - console.log('Generation steps:', steps.map((step) => ({ - text: step.text, - toolCalls: step.toolCalls, - toolResults: step.toolResults, - }))); - - try { - // Parse the JSON response - const response = JSON.parse(text) as { categoryId?: string; ruleName?: string } | null; - - if (response?.categoryId && response.ruleName) { - console.log(`Found similar rule "${response.ruleName}" suggesting category ${response.categoryId}`); - return { - categoryId: response.categoryId, - ruleName: response.ruleName, - }; - } - - return null; - } catch { - console.log('No similar rules found or invalid response'); - return null; - } - }, - ); + return parseLlmResponse(text); + } catch (error) { + console.error('LLM response validation failed:', error); + throw new Error('Invalid response format from LLM'); + } + }); } catch (error) { - console.error('Error while finding similar rules:', error); - return null; + const errorMsg = error instanceof Error ? error.message : String(error); + console.error(`Error during LLM request to ${this.provider}: ${errorMsg}`); + throw error; } } @@ -246,87 +139,4 @@ export default class LlmService implements LlmServiceI { }, ); } - - public async unifiedAsk(prompt: string): Promise { - return this.rateLimiter.executeWithRateLimiting(this.provider, async () => { - try { - const { text } = await generateText({ - model: this.model, - prompt, - temperature: 0.2, - tools: this.toolService?.getTools(), - maxSteps: 3, - system: 'You must use webSearch for unfamiliar payees before suggesting categories', - }); - - // Move cleanedText declaration outside the try-catch - const cleanedText = cleanJsonResponse(text); - console.log('Cleaned LLM response:', cleanedText); - - try { - // First, try to parse as JSON - let parsed: Partial; - try { - parsed = JSON.parse(cleanedText) as Partial; - } catch { - // If not valid JSON, check if it's a simple ID - const trimmedText = cleanedText.trim().replace(/^"|"$/g, ''); - - if (/^[a-zA-Z0-9_-]+$/.test(trimmedText)) { - console.log(`LLM returned simple ID: "${trimmedText}"`); - return { - type: 'existing', - categoryId: trimmedText, - }; - } - - throw new Error('Response is neither valid JSON nor simple ID'); - } - - // Type guard validation - if (parsed.type === 'existing' && parsed.categoryId) { - return { type: 'existing', categoryId: parsed.categoryId }; - } - if (parsed.type === 'rule' && parsed.categoryId && parsed.ruleName) { - return { - type: 'rule', - categoryId: parsed.categoryId, - ruleName: parsed.ruleName, - }; - } - if (parsed.type === 'new' && parsed.newCategory) { - return { - type: 'new', - newCategory: parsed.newCategory, - }; - } - - // If the response doesn't match expected format but has a categoryId, - // default to treating it as an existing category - if (parsed.categoryId) { - console.log('LLM response missing type but has categoryId, treating as existing category'); - return { - type: 'existing', - categoryId: parsed.categoryId, - }; - } - if (parsed && typeof parsed === 'string') { - return { - type: 'existing', - categoryId: parsed, - }; - } - - console.error('Invalid response structure from LLM:', parsed); - throw new Error('Invalid response format from LLM'); - } catch (parseError) { - console.error('Failed to parse LLM response:', cleanedText, parseError); - throw new Error('Invalid response format from LLM'); - } - } catch (error) { - console.error('LLM response validation failed:', error); - throw new Error('Invalid response format from LLM'); - } - }); - } } diff --git a/src/prompt-generator.ts b/src/prompt-generator.ts index d947216..3688bbe 100644 --- a/src/prompt-generator.ts +++ b/src/prompt-generator.ts @@ -11,24 +11,17 @@ import { transformRulesToDescriptions } from './utils/rule-utils'; class PromptGenerator implements PromptGeneratorI { private readonly promptTemplate: string; - private readonly categorySuggestionTemplate: string; - - private readonly similarRulesTemplate: string; - constructor( promptTemplate: string, - categorySuggestionTemplate = '', - similarRulesTemplate = '', ) { this.promptTemplate = promptTemplate; - this.categorySuggestionTemplate = categorySuggestionTemplate; - this.similarRulesTemplate = similarRulesTemplate; } generate( categoryGroups: APICategoryGroupEntity[], transaction: TransactionEntity, payees: APIPayeeEntity[], + rules: RuleEntity[], ): string { let template; try { @@ -46,50 +39,17 @@ class PromptGenerator implements PromptGeneratorI { categories: group.categories ?? [], })); - try { - return template({ - categoryGroups: groupsWithCategories, - amount: Math.abs(transaction.amount), - type: transaction.amount > 0 ? 'Income' : 'Outcome', - description: transaction.notes, - payee: payeeName, - importedPayee: transaction.imported_payee, - date: transaction.date, - cleared: transaction.cleared, - reconciled: transaction.reconciled, - }); - } catch { - console.error('Error generating prompt. Check syntax of your template.'); - throw new PromptTemplateException('Error generating prompt. Check syntax of your template.'); - } - } - - generateCategorySuggestion( - categoryGroups: APICategoryGroupEntity[], - transaction: TransactionEntity, - payees: APIPayeeEntity[], - ): string { - let template; - try { - template = handlebars.compile(this.categorySuggestionTemplate); - } catch { - console.error('Error generating category suggestion prompt.'); - throw new PromptTemplateException('Error generating category suggestion prompt.'); - } - - const payeeName = payees.find((payee) => payee.id === transaction.payee)?.name; - - // Ensure each category group has its categories property - const groupsWithCategories = categoryGroups.map((group) => ({ - ...group, - groupName: group.name, - categories: group.categories ?? [], - })); + const rulesDescription = this.transformRulesToDescriptions( + rules, + groupsWithCategories, + payees, + ); try { const webSearchEnabled = typeof hasWebSearchTool === 'boolean' ? hasWebSearchTool : false; return template({ categoryGroups: groupsWithCategories, + rules: rulesDescription, amount: Math.abs(transaction.amount), type: transaction.amount > 0 ? 'Income' : 'Outcome', description: transaction.notes ?? '', @@ -101,45 +61,8 @@ class PromptGenerator implements PromptGeneratorI { hasWebSearchTool: webSearchEnabled, }); } catch { - console.error('Error generating category suggestion prompt.'); - throw new PromptTemplateException('Error generating category suggestion prompt.'); - } - } - - generateSimilarRulesPrompt( - transaction: TransactionEntity & { payeeName?: string }, - rulesDescription: RuleDescription[], - ): string { - let template; - try { - template = handlebars.compile(this.similarRulesTemplate); - } catch { - console.error('Error generating similar rules prompt.'); - throw new PromptTemplateException('Error generating similar rules prompt.'); - } - - try { - // Add index to each rule for numbering - const rulesWithIndex = rulesDescription.map((rule, index) => ({ - ...rule, - index, - })); - - // Use payeeName if available, otherwise use imported_payee - const payee = transaction.payeeName ?? transaction.imported_payee; - - return template({ - amount: Math.abs(transaction.amount), - type: transaction.amount > 0 ? 'Income' : 'Outcome', - description: transaction.notes, - importedPayee: transaction.imported_payee, - payee, - date: transaction.date, - rules: rulesWithIndex, - }); - } catch (error) { - console.error('Error generating similar rules prompt:', error); - throw new PromptTemplateException('Error generating similar rules prompt.'); + console.error('Error generating prompt. Check syntax of your template.'); + throw new PromptTemplateException('Error generating prompt. Check syntax of your template.'); } } @@ -150,41 +73,6 @@ class PromptGenerator implements PromptGeneratorI { ): RuleDescription[] { return transformRulesToDescriptions(rules, categories, payees); } - - generateUnifiedPrompt( - categoryGroups: APICategoryGroupEntity[], - transaction: TransactionEntity, - payees: APIPayeeEntity[], - rules: RuleEntity[], - ): string { - const template = handlebars.compile(this.promptTemplate); - const payeeName = payees.find((p) => p.id === transaction.payee)?.name; - - const categories = categoryGroups.flatMap((group) => (group.categories ?? []).map((cat) => ({ - ...cat, - groupName: group.name, - }))); - - const rulesDescription = this.transformRulesToDescriptions( - rules, - categories as APICategoryEntity[], - payees, - ); - - return template({ - categoryGroups: categoryGroups.map((g) => ({ - ...g, - categories: g.categories ?? [], - })), - rules: rulesDescription, - amount: Math.abs(transaction.amount), - type: transaction.amount > 0 ? 'Income' : 'Expense', - description: transaction.notes, - payee: payeeName, - importedPayee: transaction.imported_payee, - date: transaction.date, - }); - } } export default PromptGenerator; diff --git a/src/templates/category-suggestion.hbs b/src/templates/category-suggestion.hbs deleted file mode 100644 index f3fc207..0000000 --- a/src/templates/category-suggestion.hbs +++ /dev/null @@ -1,57 +0,0 @@ -{{#if hasWebSearchTool}} -When suggesting a new category, you MUST use the webSearch tool to research -the business type and common categorizations for this transaction's payee. -Only suggest a category after reviewing the search results. -{{/if}} - -I need to suggest a new category for a transaction that doesn't fit any existing categories. - -Transaction details: -* Amount: {{amount}} -* Type: {{type}} -{{#if description}} -* Description: {{description}} -{{/if}} -{{#if payee}} -* Payee: {{payee}} -{{^}} -* Payee: {{importedPayee}} -{{/if}} - -Existing categories by group: -{{#each categoryGroups}} -GROUP: {{name}} (ID: "{{id}}") -{{#each categories}} -* {{name}} -{{/each}} -{{/each}} - -RESPOND WITH A JSON OBJECT that suggests a new category with these properties: -1. "name": A short, descriptive name for the new category -2. "groupName": The name of an existing OR NEW category group this should belong to -3. "groupIsNew": Boolean indicating if this group needs to be created - -Example response: -{"name": "Membership fees", "groupName": "Subscriptions", "groupIsNew": true} - -IMPORTANT: Create a specific category that: -- Is appropriate for the transaction details shown above -- Does NOT duplicate or closely resemble any existing category names -- Is specific rather than overly general -- Follows the naming patterns of other categories in the same group - -IMPORTANT: Your response MUST be: -- A SINGLE VALID JSON OBJECT -- No additional text or explanation -- No markdown formatting -- Properly escaped characters -- No trailing commas - -Example of VALID response: -{"name": "Pet Supplies", "groupName": "Digital Assets", "groupIsNew": true} - -IMPORTANT: When suggesting categories: -- Use consistent naming for similar expenses -- Prefer existing group names when appropriate -- Use the most specific yet concise names -- Avoid minor variations of the same category \ No newline at end of file diff --git a/src/templates/prompt.hbs b/src/templates/prompt.hbs index 85db3cd..377bff4 100644 --- a/src/templates/prompt.hbs +++ b/src/templates/prompt.hbs @@ -54,3 +54,6 @@ Extra rules: * If the transaction is a Credit Card Payment, categorize it as "Transfer" unless it is a fee. * Flowers can go in the "Gift" category. +{{#if hasWebSearchTool}} +You can use the web search tool to find more information about the transaction. +{{/if}} diff --git a/src/templates/similar-rules.hbs b/src/templates/similar-rules.hbs deleted file mode 100644 index 0eb6e61..0000000 --- a/src/templates/similar-rules.hbs +++ /dev/null @@ -1,38 +0,0 @@ -Transaction details: -* Payee: {{importedPayee}} -* Amount: {{amount}} -* Type: {{type}} -{{#if description}} -* Description: {{description}} -{{/if}} -{{#if date}} -* Date: {{date}} -{{/if}} - -Existing rules in the system: -{{#each rules}} -{{incIndex @index}}. Rule: "{{ruleName}}" -- Category: {{categoryName}} -- Conditions: -{{#each conditions}} - - {{field}} {{op}} - {{#if (eq type "id")}} - [{{#each value}}"{{this}}"{{#unless @last}}, {{/unless}}{{/each}}] - {{else}} - "{{value}}" - {{/if}} -{{/each}} -{{/each}} - -Based on the transaction details, determine if it closely matches any of the existing rules. -If there's a match, return a JSON object with the categoryId and ruleName. -If there's no good match, return null. - -Example response for a match: {"categoryId": "abc123", "ruleName": "Rule Name"} -Example response for no match: null - -IMPORTANT: Your response MUST be: -- A SINGLE VALID JSON OBJECT or null -- No additional text or explanation -- No markdown formatting -- Properly escaped characters \ No newline at end of file diff --git a/src/transaction-service.ts b/src/transaction-service.ts index 78cb658..fb2eddc 100644 --- a/src/transaction-service.ts +++ b/src/transaction-service.ts @@ -27,7 +27,7 @@ class TransactionService implements TransactionServiceI { private readonly suggestNewCategories: boolean; - private readonly dryRun: boolean; + private dryRun = true; constructor( actualApiClient: ActualApiServiceI, @@ -113,7 +113,7 @@ class TransactionService implements TransactionServiceI { && transaction.starting_balance_flag !== true && transaction.imported_payee !== null && transaction.imported_payee !== '' - // && !transaction.notes?.includes(this.notGuessedTag) + && !transaction.notes?.includes(this.notGuessedTag) && !transaction.is_parent && !accountsToSkip.includes(transaction.account), ); @@ -153,14 +153,14 @@ class TransactionService implements TransactionServiceI { ); try { - const prompt = this.promptGenerator.generateUnifiedPrompt( + const prompt = this.promptGenerator.generate( categoryGroups, transaction, payees, rules, ); - const response = await this.llmService.unifiedAsk(prompt); + const response = await this.llmService.ask(prompt); if (response.type === 'rule' && response.ruleName && response.categoryId) { await this.handleRuleMatch(transaction, { @@ -294,7 +294,14 @@ class TransactionService implements TransactionServiceI { categories: CategoryEntity[], ) { const category = categories.find((c) => c.id === response.categoryId); - if (!category) return; + if (!category) { + // Add not guessed tag when category not found + await this.actualApiService.updateTransactionNotes( + transaction.id, + this.appendTag(transaction.notes ?? '', this.notGuessedTag), + ); + return; + } if (this.dryRun) { console.log(`DRY RUN: Would assign transaction ${transaction.id} to existing category ${category.name}`); diff --git a/src/types.ts b/src/types.ts index b9a6720..fb3dd0c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -98,7 +98,7 @@ export interface UnifiedResponse { } export interface LlmServiceI { - unifiedAsk(prompt: string): Promise; + ask(prompt: string, categoryIds?: string[]): Promise; } export interface ToolServiceI { @@ -106,16 +106,10 @@ export interface ToolServiceI { } export interface PromptGeneratorI { - generateUnifiedPrompt( + generate( categoryGroups: APICategoryGroupEntity[], transaction: TransactionEntity, payees: APIPayeeEntity[], rules: RuleEntity[], ): string; - - transformRulesToDescriptions( - rules: RuleEntity[], - categories: APICategoryEntity[], - payees: APIPayeeEntity[], - ): RuleDescription[]; } diff --git a/src/utils/json-utils.ts b/src/utils/json-utils.ts new file mode 100644 index 0000000..e00814a --- /dev/null +++ b/src/utils/json-utils.ts @@ -0,0 +1,90 @@ +import { UnifiedResponse } from '../types'; + +function cleanJsonResponse(text: string): string { + // If the text looks like a UUID or simple ID, return it as is + if (/^[a-zA-Z0-9_-]+$/.test(text.trim())) { + return text.trim(); + } + + // Remove markdown code fences and any surrounding text + let cleaned = text.replace(/```json\n?|\n?```/g, ''); + cleaned = cleaned.trim(); + + // If there are no JSON structure characters, return the trimmed text as is + if (!/[{[]/.test(cleaned) || !/[}\]]/.test(cleaned)) { + return cleaned; + } + + // Remove leading characters up to first JSON structure character + cleaned = cleaned.replace(/^[^{[]*?([{[])/, '$1'); + // Remove trailing characters after last JSON structure character + cleaned = cleaned.replace(/([}\]])[^}\]]*$/, '$1'); + + return cleaned.trim(); +} + +function parseLlmResponse(text: string): UnifiedResponse { + const cleanedText = cleanJsonResponse(text); + console.log('Cleaned LLM response:', cleanedText); + + try { + let parsed: Partial; + try { + parsed = JSON.parse(cleanedText) as Partial; + } catch { + // If not valid JSON, check if it's a simple ID + const trimmedText = cleanedText.trim().replace(/^"|"$/g, ''); + + if (/^[a-zA-Z0-9_-]+$/.test(trimmedText)) { + console.log(`LLM returned simple ID: "${trimmedText}"`); + return { + type: 'existing', + categoryId: trimmedText, + }; + } + + throw new Error('Response is neither valid JSON nor simple ID'); + } + + if (parsed.type === 'existing' && parsed.categoryId) { + return { type: 'existing', categoryId: parsed.categoryId }; + } + if (parsed.type === 'rule' && parsed.categoryId && parsed.ruleName) { + return { + type: 'rule', + categoryId: parsed.categoryId, + ruleName: parsed.ruleName, + }; + } + if (parsed.type === 'new' && parsed.newCategory) { + return { + type: 'new', + newCategory: parsed.newCategory, + }; + } + + // If the response doesn't match expected format but has a categoryId, + // default to treating it as an existing category + if (parsed.categoryId) { + console.log('LLM response missing type but has categoryId, treating as existing category'); + return { + type: 'existing', + categoryId: parsed.categoryId, + }; + } + if (parsed && typeof parsed === 'string') { + return { + type: 'existing', + categoryId: parsed, + }; + } + + console.error('Invalid response structure from LLM:', parsed); + throw new Error('Invalid response format from LLM'); + } catch (parseError) { + console.error('Failed to parse LLM response:', cleanedText, parseError); + throw new Error('Invalid response format from LLM'); + } +} + +export { parseLlmResponse, cleanJsonResponse }; diff --git a/tests/actual-ai.test.ts b/tests/actual-ai.test.ts index ee34d2c..b22b292 100644 --- a/tests/actual-ai.test.ts +++ b/tests/actual-ai.test.ts @@ -1,9 +1,3 @@ -import { - APIAccountEntity, - APICategoryEntity, - APICategoryGroupEntity, - APIPayeeEntity, -} from '@actual-app/api/@types/loot-core/server/api-models'; import TransactionService from '../src/transaction-service'; import InMemoryActualApiService from './test-doubles/in-memory-actual-api-service'; import MockedLlmService from './test-doubles/mocked-llm-service'; @@ -25,21 +19,27 @@ describe('ActualAiService', () => { inMemoryApiService = new InMemoryActualApiService(); mockedLlmService = new MockedLlmService(); mockedPromptGenerator = new MockedPromptGenerator(); - const categoryGroups: APICategoryGroupEntity[] = GivenActualData.createSampleCategoryGroups(); - const categories: APICategoryEntity[] = GivenActualData.createSampleCategories(); - const payees: APIPayeeEntity[] = GivenActualData.createSamplePayees(); - const accounts: APIAccountEntity[] = GivenActualData.createSampleAccounts(); + const categoryGroups = GivenActualData.createSampleCategoryGroups(); + const categories = GivenActualData.createSampleCategories(); + const payees = GivenActualData.createSamplePayees(); + const accounts = GivenActualData.createSampleAccounts(); + const rules = GivenActualData.createSampleRules(); + transactionService = new TransactionService( inMemoryApiService, mockedLlmService, mockedPromptGenerator, NOT_GUESSED_TAG, GUESSED_TAG, + false, + false, ); + inMemoryApiService.setCategoryGroups(categoryGroups); inMemoryApiService.setCategories(categories); inMemoryApiService.setPayees(payees); inMemoryApiService.setAccounts(accounts); + inMemoryApiService.setRules(rules); }); it('It should assign a category to transaction', async () => { diff --git a/tests/prompt-generator.test.ts b/tests/prompt-generator.test.ts index 9679246..697462c 100644 --- a/tests/prompt-generator.test.ts +++ b/tests/prompt-generator.test.ts @@ -1,216 +1,165 @@ -import { TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; +import { TransactionEntity, RuleEntity } from '@actual-app/api/@types/loot-core/types/models'; +import type { APICategoryGroupEntity } from '@actual-app/api/@types/loot-core/server/api-models'; import fs from 'fs'; import PromptGenerator from '../src/prompt-generator'; import GivenActualData from './test-doubles/given/given-actual-data'; import PromptTemplateException from '../src/exceptions/prompt-template-exception'; +import handlebars from '../src/handlebars-helpers'; -const promptTemplate = fs.readFileSync('./src/templates/prompt.hbs', 'utf8').trim(); -const categorySuggestionTemplate = fs.readFileSync('./src/templates/category-suggestion.hbs', 'utf8').trim(); -const similarRulesTemplate = fs.readFileSync('./src/templates/similar-rules.hbs', 'utf8').trim(); - -describe('LlmGenerator', () => { - const expectedAirbnb = 'I want to categorize the given bank transaction into one of the following categories:\n' - + 'GROUP: Usual Expenses (ID: "1")\n' - + '* Groceries (ID: "ff7be77b-40f4-4e9d-aea4-be6b8c431281")\n' - + '* Travel (ID: "541836f1-e756-4473-a5d0-6c1d3f06c7fa")\n' - + 'GROUP: Income (ID: "2")\n' - + '* Salary (ID: "123836f1-e756-4473-a5d0-6c1d3f06c7fa")\n\n' - + 'Transaction details:\n' - + '* Amount: 34169\n' - + '* Type: Outcome\n' - + '* Description: AIRBNB * XXXX1234567 822-307-2000\n' - + '* Payee: Airbnb * XXXX1234567\n\n' - + 'RESPOND ONLY WITH A CATEGORY ID from the list above. Do not write anything else.\n' - + 'If you\'re not sure which category to use, respond with "uncategorized".'; - - const expectedCarrefour = 'I want to categorize the given bank transaction into one of the following categories:\n' - + 'GROUP: Usual Expenses (ID: "1")\n' - + '* Groceries (ID: "ff7be77b-40f4-4e9d-aea4-be6b8c431281")\n' - + '* Travel (ID: "541836f1-e756-4473-a5d0-6c1d3f06c7fa")\n' - + 'GROUP: Income (ID: "2")\n' - + '* Salary (ID: "123836f1-e756-4473-a5d0-6c1d3f06c7fa")\n\n' - + 'Transaction details:\n' - + '* Amount: 1000\n' - + '* Type: Outcome\n' - + '* Payee: Carrefour\n\n' - + 'RESPOND ONLY WITH A CATEGORY ID from the list above. Do not write anything else.\n' - + 'If you\'re not sure which category to use, respond with "uncategorized".'; - - const expectedGoogle = 'I want to categorize the given bank transaction into one of the following categories:\n' - + 'GROUP: Usual Expenses (ID: "1")\n' - + '* Groceries (ID: "ff7be77b-40f4-4e9d-aea4-be6b8c431281")\n' - + '* Travel (ID: "541836f1-e756-4473-a5d0-6c1d3f06c7fa")\n' - + 'GROUP: Income (ID: "2")\n' - + '* Salary (ID: "123836f1-e756-4473-a5d0-6c1d3f06c7fa")\n\n' - + 'Transaction details:\n' - + '* Amount: 2137420\n' - + '* Type: Income\n' - + '* Description: DESCRIPTION\n' - + '* Payee: Google\n\n' - + 'RESPOND ONLY WITH A CATEGORY ID from the list above. Do not write anything else.\n' - + 'If you\'re not sure which category to use, respond with "uncategorized".'; - - const promptSet: [TransactionEntity, string][] = [ +describe('PromptGenerator', () => { + const promptTemplate = fs.readFileSync('./src/templates/prompt.hbs', 'utf8').trim(); + + const promptSet: [TransactionEntity][] = [ [ GivenActualData.createTransaction( '1', -34169, 'Airbnb * XXXX1234567', 'AIRBNB * XXXX1234567 822-307-2000', + undefined, + undefined, + '2021-01-01', ), - expectedAirbnb, - ], [ - GivenActualData.createTransaction( - '1', - -1000, - 'Carrefour 2137', - '', - GivenActualData.PAYEE_CARREFOUR, - ), - expectedCarrefour, - ], [ + ], + [ GivenActualData.createTransaction( - '1', - 2137420, - 'Google Imported', - 'DESCRIPTION', - GivenActualData.PAYEE_GOOGLE, + '2', + -1626, + 'Steam Purc', + 'Steam Purc 16.26_V-miss #actual-ai-miss', + undefined, + undefined, + '2025-02-18', ), - expectedGoogle, ], ]; - it.each(promptSet)('should generate a prompt for categorizing transactions', ( + // Helper function to safely create template data + const loadAndRenderTemplate = ( + templateContent: string, + transaction: TransactionEntity, + categoryGroups: APICategoryGroupEntity[], + ): string => { + const template = handlebars.compile(templateContent); + const payees = GivenActualData.createSamplePayees(); + + // Create a type-safe copy of category groups with only required properties + const safeCategoryGroups = categoryGroups.map((group) => { + // Extract only properties we know exist in APICategoryGroupEntity + const safeGroup: APICategoryGroupEntity = { + id: group.id, + name: group.name, + is_income: group.is_income, + categories: [], + }; + + // Type-safe mapping of categories + const categories = (group.categories ?? []).map((category) => ({ + id: category.id, + name: category.name, + group_id: category.group_id, + is_income: category.is_income, + })); + + safeGroup.categories = categories; + return safeGroup; + }); + + return template({ + categoryGroups: safeCategoryGroups, + amount: Math.abs(transaction.amount), + type: transaction.amount > 0 ? 'Income' : 'Outcome', + description: transaction.notes ?? '', + payee: payees.find((p) => p.id === transaction.payee)?.name ?? '', + importedPayee: transaction.imported_payee ?? '', + date: transaction.date ?? '', + cleared: transaction.cleared ?? false, + reconciled: transaction.reconciled ?? false, + hasWebSearchTool: false, + rules: [], + }); + }; + + it.each(promptSet)('should generate prompts in both modern and legacy formats', ( transaction: TransactionEntity, - expectedPrompt: string, ) => { const categoryGroups = GivenActualData.createSampleCategoryGroups(); - const payees = GivenActualData.createSamplePayees(); - const promptGenerator = new PromptGenerator(promptTemplate, categorySuggestionTemplate); - const prompt = promptGenerator.generate(categoryGroups, transaction, payees); - expect(prompt).toEqual(expectedPrompt); + // Modern format test + const modernTemplate = fs.readFileSync('./src/templates/prompt.hbs', 'utf8').trim(); + const modernPromptGenerator = new PromptGenerator(modernTemplate); + const generatedModern = modernPromptGenerator.generate(categoryGroups, transaction, payees, []); + const expectedModern = loadAndRenderTemplate(modernTemplate, transaction, categoryGroups); + expect(generatedModern.trim()).toEqual(expectedModern.trim()); + + // Legacy format test + const legacyTemplate = ` +I want to categorize the given bank transactions into the following categories: +{{#each categoryGroups}} +{{#each categories}} +* {{name}} ({{../name}}) (ID: "{{id}}") +{{/each}} +{{/each}} +Please categorize the following transaction: +* Amount: {{amount}} +* Type: {{type}} +{{#if description}} +* Description: {{description}} +{{/if}} +{{#if payee}} +* Payee: {{payee}} +{{^}} +* Payee: {{importedPayee}} +{{/if}} +ANSWER BY A CATEGORY ID - DO NOT CREATE ENTIRE SENTENCE - DO NOT WRITE CATEGORY NAME, JUST AN ID. Do not guess, if you don't know the answer, return "uncategorized".`.trim(); + + const legacyPromptGenerator = new PromptGenerator(legacyTemplate); + const generatedLegacy = legacyPromptGenerator.generate(categoryGroups, transaction, payees, []); + const expectedLegacy = loadAndRenderTemplate(legacyTemplate, transaction, categoryGroups); + expect(generatedLegacy.trim()).toEqual(expectedLegacy.trim()); }); it('should throw exception on invalid prompt', () => { const categoryGroups = GivenActualData.createSampleCategoryGroups(); - const payees = GivenActualData.createSamplePayees(); const transaction = GivenActualData.createTransaction('1', 1000, 'Carrefour 2137'); - const promptGenerator = new PromptGenerator('{{#each categories}}', categorySuggestionTemplate); + const promptGenerator = new PromptGenerator('{{#each categories}}'); const t = () => { - promptGenerator.generate(categoryGroups, transaction, payees); + promptGenerator.generate(categoryGroups, transaction, payees, []); }; expect(t).toThrow(PromptTemplateException); }); - it('should generate a category suggestion prompt', () => { - const categoryGroups = GivenActualData.createSampleCategoryGroups(); - const payees = GivenActualData.createSamplePayees(); + it('should include rules in modern format when provided', () => { const transaction = GivenActualData.createTransaction( '1', -1000, 'Carrefour 2137', '', GivenActualData.PAYEE_CARREFOUR, + undefined, + '2021-01-01', ); - const promptGenerator = new PromptGenerator(promptTemplate, categorySuggestionTemplate); - const prompt = promptGenerator.generateCategorySuggestion(categoryGroups, transaction, payees); - - expect(prompt).toContain('I need to suggest a new category for a transaction'); - expect(prompt).toContain('* Payee: Carrefour'); - expect(prompt).toContain('* Amount: 1000'); - expect(prompt).toContain('* Type: Outcome'); - expect(prompt).toContain('RESPOND WITH A JSON OBJECT'); - }); - - it('should throw exception on invalid category suggestion prompt', () => { + const rules: RuleEntity[] = GivenActualData.createSampleRules(); const categoryGroups = GivenActualData.createSampleCategoryGroups(); const payees = GivenActualData.createSamplePayees(); - const transaction = GivenActualData.createTransaction('1', 1000, 'Carrefour 2137'); - const promptGenerator = new PromptGenerator(promptTemplate, '{{#each invalidSyntax}}'); - - const t = () => { - promptGenerator.generateCategorySuggestion(categoryGroups, transaction, payees); - }; - expect(t).toThrow(PromptTemplateException); - }); - - it('should generate a similar rules prompt', () => { - const transaction = GivenActualData.createTransaction( - '1', - -1000, - 'Carrefour 2137', - '', - GivenActualData.PAYEE_CARREFOUR, - ); + const promptGenerator = new PromptGenerator(promptTemplate); + const prompt = promptGenerator.generate(categoryGroups, transaction, payees, rules); - const rulesDescription = [ - { - ruleName: 'Grocery Rule', - conditions: 'payee contains "Carrefour"', - categoryName: 'Groceries', - categoryId: GivenActualData.CATEGORY_GROCERIES, - }, - { - ruleName: 'Travel Rule', - conditions: 'payee contains "Airbnb"', - categoryName: 'Travel', - categoryId: GivenActualData.CATEGORY_TRAVEL, - }, - ]; - - const promptGenerator = new PromptGenerator( - promptTemplate, - categorySuggestionTemplate, - similarRulesTemplate, - ); - const prompt = promptGenerator.generateSimilarRulesPrompt(transaction, rulesDescription); + // Check for rule-specific content + expect(prompt).toContain('Existing Rules:'); + expect(prompt).toContain('1. Unnamed rule → unknown'); + expect(prompt).toContain('2. Unnamed rule → unknown'); + expect(prompt).toContain('Conditions:'); + // Check for transaction details expect(prompt).toContain('Transaction details:'); - expect(prompt).toContain('* Payee: Carrefour 2137'); expect(prompt).toContain('* Amount: 1000'); expect(prompt).toContain('* Type: Outcome'); - - // Less strict checks that don't rely on exact formatting - expect(prompt).toContain('Grocery Rule'); - expect(prompt).toContain('payee contains'); - expect(prompt).toContain('Carrefour'); - expect(prompt).toContain('Groceries'); - - expect(prompt).toContain('Travel Rule'); - expect(prompt).toContain('Airbnb'); - expect(prompt).toContain('Travel'); - - expect(prompt).toContain('Based on the transaction details'); - expect(prompt).toContain('If there\'s a match, return a JSON object'); - }); - - it('should throw exception on invalid similar rules prompt', () => { - const transaction = GivenActualData.createTransaction('1', 1000, 'Carrefour 2137'); - const rulesDescription = [ - { - ruleName: 'Grocery Rule', - conditions: 'payee contains "Carrefour"', - categoryName: 'Groceries', - categoryId: GivenActualData.CATEGORY_GROCERIES, - }, - ]; - - const promptGenerator = new PromptGenerator( - promptTemplate, - categorySuggestionTemplate, - '{{#each invalidSyntax}}', - ); - - const t = () => { - promptGenerator.generateSimilarRulesPrompt(transaction, rulesDescription); - }; - - expect(t).toThrow(PromptTemplateException); + expect(prompt).toContain('* Date: 2021-01-01'); }); }); diff --git a/tests/test-doubles/given/given-actual-data.ts b/tests/test-doubles/given/given-actual-data.ts index 246afbd..a0ba969 100644 --- a/tests/test-doubles/given/given-actual-data.ts +++ b/tests/test-doubles/given/given-actual-data.ts @@ -4,7 +4,7 @@ import { APICategoryGroupEntity, APIPayeeEntity, } from '@actual-app/api/@types/loot-core/server/api-models'; -import { TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; +import { TransactionEntity, RuleEntity } from '@actual-app/api/@types/loot-core/types/models'; export default class GivenActualData { public static CATEGORY_GROCERIES = 'ff7be77b-40f4-4e9d-aea4-be6b8c431281'; @@ -109,4 +109,33 @@ export default class GivenActualData { this.createAccount(GivenActualData.ACCOUNT_OFF_BUDGET, 'Off Budget Account', true, false), ]; } + + static createSampleRules(): RuleEntity[] { + return [ + { + id: '879da987-9879-8798-7987-987987987987', + stage: null, + conditionsOp: 'and', + conditions: [], + actions: [{ + op: 'set', + field: 'category', + value: this.CATEGORY_GROCERIES, + type: 'id', + }], + }, + { + id: '879da987-9879-8798-7987-987987987988', + stage: null, + conditionsOp: 'and', + conditions: [], + actions: [{ + op: 'set', + field: 'category', + value: this.CATEGORY_TRAVEL, + type: 'id', + }], + }, + ]; + } } diff --git a/tests/test-doubles/in-memory-actual-api-service.ts b/tests/test-doubles/in-memory-actual-api-service.ts index 4710f78..429601e 100644 --- a/tests/test-doubles/in-memory-actual-api-service.ts +++ b/tests/test-doubles/in-memory-actual-api-service.ts @@ -20,6 +20,8 @@ export default class InMemoryActualApiService implements ActualApiServiceI { private wasBankSyncRan = false; + private rules: RuleEntity[] = []; + async initializeApi(): Promise { // Initialize the API (mock implementation) } @@ -159,10 +161,14 @@ export default class InMemoryActualApiService implements ActualApiServiceI { } async getRules(): Promise { - return Promise.resolve([]); + return Promise.resolve(this.rules); } async getPayeeRules(_payeeId: string): Promise { return Promise.resolve([]); } + + setRules(rules: RuleEntity[]): void { + this.rules = rules; + } } diff --git a/tests/test-doubles/mocked-llm-service.ts b/tests/test-doubles/mocked-llm-service.ts index c6f0f9e..974164f 100644 --- a/tests/test-doubles/mocked-llm-service.ts +++ b/tests/test-doubles/mocked-llm-service.ts @@ -1,21 +1,46 @@ -import { CategorySuggestion, LlmServiceI } from '../../src/types'; +import { LlmServiceI, UnifiedResponse } from '../../src/types'; +import GivenActualData from './given/given-actual-data'; export default class MockedLlmService implements LlmServiceI { - private guess = 'uncategorized'; + private response: UnifiedResponse = { + type: 'existing', + categoryId: 'uncategorized', + }; - async ask(): Promise { - return Promise.resolve(this.guess); + async ask(): Promise { + return Promise.resolve(this.response); } - async askForCategorySuggestion(): Promise { - return Promise.resolve(null); - } + // For backward compatibility in tests + setGuess(categoryIdOrName: string): void { + const uuidMatch = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i.exec(categoryIdOrName); + const foundUUID = uuidMatch ? uuidMatch[0] : null; - setGuess(guess: string): void { - this.guess = guess; - } + if (foundUUID) { + this.response = { + type: 'existing', + categoryId: foundUUID, + }; + } else { + // Expand category mapping to include all sample categories + const categoryMap: Record = { + Groceries: GivenActualData.CATEGORY_GROCERIES, + Travel: GivenActualData.CATEGORY_TRAVEL, + Salary: GivenActualData.CATEGORY_SALARY, + Insurance: '5c2a6627-f8fa-4f7a-8781-4b039ff133cd', + Repairs: '8955100f-5dc3-42ff-ab22-030136149a20', + 'Shopping & Clothing': '05ba8139-b5f0-4b51-8892-d046527ff6c2', + Subscriptions: '3712d83e-3771-4b77-9059-53601b0b33bc', + Utilities: 'c1277b57-39ac-4757-bb50-16a3290e2612', + 'Car Payment': 'b6874e94-82ae-4737-bbb4-a0ae28c17624', + 'Gas & Parking': '29bc9256-7151-4b1d-987c-069bc26e0454', + 'Drugs & Other Meds': '1602d1ec-ceee-47fa-b144-abca8021e177', + }; - async findSimilarRules(): Promise<{ categoryId: string; ruleName: string } | null> { - return Promise.resolve(null); + this.response = { + type: 'existing', + categoryId: categoryMap[categoryIdOrName] || 'uncategorized', + }; + } } } diff --git a/tests/test-doubles/mocked-prompt-generator.ts b/tests/test-doubles/mocked-prompt-generator.ts index 7a67368..78fa05c 100644 --- a/tests/test-doubles/mocked-prompt-generator.ts +++ b/tests/test-doubles/mocked-prompt-generator.ts @@ -1,30 +1,22 @@ import { APICategoryGroupEntity, APIPayeeEntity } from '@actual-app/api/@types/loot-core/server/api-models'; -import { TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; -import { PromptGeneratorI } from '../../src/types'; +import { RuleEntity, TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; +import { APICategoryEntity, PromptGeneratorI, RuleDescription } from '../../src/types'; export default class MockedPromptGenerator implements PromptGeneratorI { - generate(): string { - return 'mocked prompt'; - } - - generateCategorySuggestion( + generate( _categoryGroups: APICategoryGroupEntity[], _transaction: TransactionEntity, _payees: APIPayeeEntity[], + _rules?: RuleEntity[], ): string { - return 'mocked category suggestion prompt'; + return 'mocked prompt'; } - generateSimilarRulesPrompt( - _transaction: TransactionEntity, - _rulesDescription: { - ruleName: string; - conditions: string; - categoryName: string; - categoryId: string; - index?: number; - }[], - ): string { - return 'mocked similar rules prompt'; + transformRulesToDescriptions( + _rules: RuleEntity[], + _categories: APICategoryEntity[], + _payees: APIPayeeEntity[], + ): RuleDescription[] { + return []; } } From f7b15e6196b83af16bdc24adbedb9ce1bbdb222a Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sat, 8 Mar 2025 00:20:08 -0500 Subject: [PATCH 08/17] Implement feature flag system for dynamic configuration management --- .env.example | 12 +- README.md | 41 ++++-- src/config.ts | 126 +++++++++++++++++- src/container.ts | 8 +- src/llm-service.ts | 1 - src/prompt-generator.ts | 16 +-- src/transaction-service.ts | 56 +++++--- src/utils/tool-service.ts | 4 +- tests/actual-ai.test.ts | 18 ++- tests/prompt-generator.test.ts | 53 +++++++- tests/test-doubles/mocked-config.ts | 9 ++ tests/test-doubles/mocked-llm-service.test.ts | 39 ++++++ tests/test-doubles/mocked-llm-service.ts | 2 +- tests/test-doubles/mocked-prompt-generator.ts | 10 +- 14 files changed, 321 insertions(+), 74 deletions(-) create mode 100644 tests/test-doubles/mocked-config.ts create mode 100644 tests/test-doubles/mocked-llm-service.test.ts diff --git a/.env.example b/.env.example index 16e701f..a20ba88 100644 --- a/.env.example +++ b/.env.example @@ -4,10 +4,16 @@ ACTUAL_BUDGET_ID= CLASSIFICATION_SCHEDULE_CRON="0 */4 * * *" CLASSIFY_ON_STARTUP=true SYNC_ACCOUNTS_BEFORE_CLASSIFY=true -SUGGEST_NEW_CATEGORIES=false -DRY_RUN_NEW_CATEGORIES=false -ENABLED_TOOLS=webSearch + +# Feature flags - can be specified as an array +FEATURES="['webSearch', 'suggestNewCategories', 'rerunMissedTransactions']" +DRY_RUN=true + +# Tools and API keys +# ENABLED_TOOLS=webSearch VALUESERP_API_KEY= + +# LLM configuration LLM_PROVIDER=openai OPENAI_API_KEY= OPENAI_MODEL=gpt-4o-mini diff --git a/README.md b/README.md index 7441979..509bbf4 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,10 @@ When enabled, the LLM can suggest entirely new categories for transactions it ca Using the ValueSerp API, the system can search the web for information about unfamiliar merchants to help the LLM make better categorization decisions. +#### 🔄 Re-run missed transactions + +Re-process transactions previously marked as unclassified. + ## 🚀 Usage Sample `docker-compose.yml` file: @@ -64,9 +68,7 @@ services: CLASSIFY_ON_STARTUP: true # Whether to classify transactions on startup (don't wait for cron schedule) SYNC_ACCOUNTS_BEFORE_CLASSIFY: false # Whether to sync accounts before classification LLM_PROVIDER: openai # Can be "openai", "anthropic", "google-generative-ai", "ollama" or "groq" -# SUGGEST_NEW_CATEGORIES: false # Whether to suggest new categories for transactions that can't be classified with existing ones -# DRY_RUN_NEW_CATEGORIES: true # When true, just logs suggested categories without creating them -# ENABLED_TOOLS: webSearch # Comma-separated list of tools to enable +# FEATURES: '["webSearch", "suggestNewCategories"]' # VALUESERP_API_KEY: your_valueserp_api_key # API key for ValueSerp, required if webSearch tool is enabled # OPENAI_API_KEY: # optional. required if you want to use the OpenAI API # OPENAI_MODEL: # optional. required if you want to use a specific model, default is "gpt-4o-mini" @@ -107,6 +109,23 @@ services: # ANSWER BY A CATEGORY ID - DO NOT CREATE ENTIRE SENTENCE - DO NOT WRITE CATEGORY NAME, JUST AN ID. Do not guess, if you don't know the answer, return "uncategorized". ``` +## Feature Configuration + +You can configure features in using the FEATURES array (recommended): + +The `FEATURES` environment variable accepts a JSON array of feature names to enable: + +``` +FEATURES='["webSearch", "suggestNewCategories"]' +``` + +Available features: +- `webSearch` - Enable web search for merchant information +- `suggestNewCategories` - Allow suggesting new categories for transactions +- `dryRun` - Run in dry run mode (enabled by default) +- `dryRunNewCategories` - Only log suggested categories without creating them (enabled by default) +- `rerunMissedTransactions` - Re-process transactions previously marked as unclassified + ## Customizing the Prompt To create a custom prompt, modify the `PROMPT_TEMPLATE` environment variable to include or exclude variables as needed. @@ -135,13 +154,13 @@ loops. ## New Category Suggestions -When `SUGGEST_NEW_CATEGORIES` is enabled, the system will: +When `suggestNewCategories` feature is enabled, the system will: 1. First try to classify transactions using existing categories 2. For transactions that can't be classified, request a new category suggestion from the LLM 3. Check if similar categories already exist -4. If in dry run mode (`DRY_RUN_NEW_CATEGORIES=true`), just log the suggestions -5. If not in dry run mode (`DRY_RUN_NEW_CATEGORIES=false`), create the new categories and assign transactions to them +4. If in dry run mode (`dryRunNewCategories` is enabled), just log the suggestions +5. If not in dry run mode, create the new categories and assign transactions to them This feature is particularly useful when you have transactions that don't fit your current category structure and you want the LLM to help expand your categories intelligently. @@ -149,7 +168,7 @@ This feature is particularly useful when you have transactions that don't fit yo The system supports various tools that can be enabled to enhance the LLM's capabilities: -1. Set `ENABLED_TOOLS` in your environment variables as a comma-separated list of tools to enable +1. Enable tools by including them in the `FEATURES` array or by setting `ENABLED_TOOLS` 2. Provide any required API keys for the tools you want to use Currently supported tools: @@ -159,7 +178,7 @@ Currently supported tools: The webSearch tool uses the ValueSerp API to search for information about merchants that the LLM might not be familiar with, providing additional context for categorization decisions. To use this tool: -1. Include `webSearch` in your `ENABLED_TOOLS` list +1. Include `webSearch` in your `FEATURES` array or `ENABLED_TOOLS` list 2. Provide your ValueSerp API key as `VALUESERP_API_KEY` This is especially helpful for: @@ -171,15 +190,15 @@ The search results are included in the prompts sent to the LLM, helping it make ## Dry Run Mode -Enable dry run mode by setting `DRY_RUN=true` (default). In this mode: +The `dryRun` feature is enabled by default. In this mode: - No transactions will be modified - No categories will be created - All proposed changes will be logged to console - System will show what would happen with real execution To perform actual changes: -1. Set `DRY_RUN=false` -2. Ensure `SUGGEST_NEW_CATEGORIES=true` if you want new category creation +1. Remove `dryRun` from your FEATURES array or set `DRY_RUN=false` +2. Ensure `suggestNewCategories` is enabled if you want new category creation 3. Run the classification process Dry run messages will show: diff --git a/src/config.ts b/src/config.ts index 6053b85..e668a15 100644 --- a/src/config.ts +++ b/src/config.ts @@ -33,10 +33,124 @@ export const groqModel = process.env.GROQ_MODEL ?? 'llama-3.3-70b-versatile'; export const groqBaseURL = process.env.GROQ_BASE_URL ?? 'https://api.groq.com/openai/v1'; export const valueSerpApiKey = process.env.VALUESERP_API_KEY ?? ''; -// Feature Flags -export const suggestNewCategories = process.env.SUGGEST_NEW_CATEGORIES === 'true'; -export const dryRun = process.env.DRY_RUN !== 'false'; // Default to true unless explicitly false +// Feature Flags System +export interface FeatureFlag { + enabled: boolean; + defaultValue: boolean; + description: string; + options?: string[]; +} -// Tools configuration -export const enabledTools = (process.env.ENABLED_TOOLS ?? '').split(',').map((tool) => tool.trim()).filter(Boolean); -export const hasWebSearchTool = enabledTools.includes('webSearch'); +export type FeatureFlags = Record; + +export const features: FeatureFlags = {}; + +let enabledFeatures: string[] = []; +try { + if (process.env.FEATURES) { + const parsedFeatures = JSON.parse(process.env.FEATURES) as unknown; + if (Array.isArray(parsedFeatures)) { + enabledFeatures = parsedFeatures as string[]; + } else { + console.warn('FEATURES environment variable is not a valid JSON array, ignoring'); + } + } +} catch (e) { + console.warn('Failed to parse FEATURES environment variable, ignoring', e); +} + +// Register standard features with defaults +function registerStandardFeatures() { + // Suggest new categories (disabled by default) + features.suggestNewCategories = { + enabled: enabledFeatures.includes('suggestNewCategories'), + defaultValue: false, + description: 'Suggest new categories for transactions that cannot be classified', + }; + + // Dry run mode (enabled by default) + features.dryRun = { + enabled: enabledFeatures.includes('dryRun'), + defaultValue: true, + description: 'Run in dry mode without actually making changes', + }; + + // Dry run for new categories (enabled by default) + features.dryRunNewCategories = { + enabled: enabledFeatures.includes('dryRunNewCategories'), + defaultValue: true, + description: 'Only log suggested categories without creating them', + }; + + // Rerun missed transactions (disabled by default) + features.rerunMissedTransactions = { + enabled: enabledFeatures.includes('rerunMissedTransactions'), + defaultValue: false, + description: 'Re-process transactions marked as not guessed', + }; +} + +// Register available tools as features +function registerToolFeatures() { + // Parse tools from ENABLED_TOOLS for backward compatibility + const legacyTools = (process.env.ENABLED_TOOLS ?? '').split(',') + .map((tool) => tool.trim()) + .filter(Boolean); + + // Register webSearch tool + features.webSearch = { + enabled: enabledFeatures.includes('webSearch') || legacyTools.includes('webSearch'), + defaultValue: false, + description: 'Enable web search capability for merchant lookup', + options: ['webSearch'], + }; + + // Additional tools can be added here following the same pattern + // features.newTool = { + // enabled: enabledFeatures.includes('newTool'), + // defaultValue: false, + // description: '...' + // }; +} + +registerStandardFeatures(); +registerToolFeatures(); + +export function isFeatureEnabled(featureName: string): boolean { + return features[featureName]?.enabled ?? features[featureName]?.defaultValue ?? false; +} + +export function registerCustomFeatureFlag( + name: string, + enabled: boolean, + defaultValue: boolean, + description: string, + options?: string[], +): void { + features[name] = { + enabled, + defaultValue, + description, + options, + }; +} + +export function toggleFeature(featureName: string, enabled?: boolean): boolean { + if (!features[featureName]) { + console.warn(`Feature flag '${featureName}' does not exist`); + return false; + } + const newValue = enabled ?? !features[featureName].enabled; + features[featureName].enabled = newValue; + return newValue; +} + +export function getEnabledTools(): string[] { + return Object.entries(features) + .filter(([_, config]) => config.options && isFeatureEnabled(config.options[0])) + .flatMap(([_, config]) => config.options ?? []); +} + +export function isToolEnabled(toolName: string): boolean { + return getEnabledTools().includes(toolName); +} diff --git a/src/container.ts b/src/container.ts index 5c963c2..6c68af8 100644 --- a/src/container.ts +++ b/src/container.ts @@ -9,7 +9,6 @@ import { anthropicModel, budgetId, dataDir, - dryRun, e2ePassword, googleApiKey, googleBaseURL, @@ -28,10 +27,9 @@ import { password, promptTemplate, serverURL, - suggestNewCategories, syncAccountsBeforeClassify, valueSerpApiKey, - enabledTools, + getEnabledTools, } from './config'; import ActualAiService from './actual-ai'; import PromptGenerator from './prompt-generator'; @@ -39,7 +37,7 @@ import LlmService from './llm-service'; import ToolService from './utils/tool-service'; // Create tool service if API key is available and tools are enabled -const toolService = valueSerpApiKey && enabledTools.length > 0 +const toolService = valueSerpApiKey && getEnabledTools().length > 0 ? new ToolService(valueSerpApiKey) : undefined; @@ -86,8 +84,6 @@ const transactionService = new TransactionService( promptGenerator, notGuessedTag, guessedTag, - suggestNewCategories, - dryRun, ); const actualAi = new ActualAiService( diff --git a/src/llm-service.ts b/src/llm-service.ts index 1ddabc6..4706569 100644 --- a/src/llm-service.ts +++ b/src/llm-service.ts @@ -90,7 +90,6 @@ export default class LlmService implements LlmServiceI { temperature: 0.2, tools: this.toolService?.getTools(), maxSteps: 3, - system: 'You must use webSearch for unfamiliar payees before suggesting categories', }); return parseLlmResponse(text); diff --git a/src/prompt-generator.ts b/src/prompt-generator.ts index 3688bbe..2a34526 100644 --- a/src/prompt-generator.ts +++ b/src/prompt-generator.ts @@ -2,10 +2,10 @@ import { APIPayeeEntity } from '@actual-app/api/@types/loot-core/server/api-mode import { RuleEntity, TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; import handlebars from './handlebars-helpers'; import { - PromptGeneratorI, RuleDescription, APICategoryEntity, APICategoryGroupEntity, + PromptGeneratorI, APICategoryGroupEntity, } from './types'; import PromptTemplateException from './exceptions/prompt-template-exception'; -import { hasWebSearchTool } from './config'; +import { isToolEnabled } from './config'; import { transformRulesToDescriptions } from './utils/rule-utils'; class PromptGenerator implements PromptGeneratorI { @@ -39,14 +39,14 @@ class PromptGenerator implements PromptGeneratorI { categories: group.categories ?? [], })); - const rulesDescription = this.transformRulesToDescriptions( + const rulesDescription = transformRulesToDescriptions( rules, groupsWithCategories, payees, ); try { - const webSearchEnabled = typeof hasWebSearchTool === 'boolean' ? hasWebSearchTool : false; + const webSearchEnabled = typeof isToolEnabled('webSearch') === 'boolean' ? isToolEnabled('webSearch') : false; return template({ categoryGroups: groupsWithCategories, rules: rulesDescription, @@ -65,14 +65,6 @@ class PromptGenerator implements PromptGeneratorI { throw new PromptTemplateException('Error generating prompt. Check syntax of your template.'); } } - - transformRulesToDescriptions( - rules: RuleEntity[], - categories: APICategoryEntity[], - payees: APIPayeeEntity[] = [], - ): RuleDescription[] { - return transformRulesToDescriptions(rules, categories, payees); - } } export default PromptGenerator; diff --git a/src/transaction-service.ts b/src/transaction-service.ts index fb2eddc..cf70c8c 100644 --- a/src/transaction-service.ts +++ b/src/transaction-service.ts @@ -9,6 +9,7 @@ import type { TransactionServiceI, CategorySuggestion, } from './types'; +import { isFeatureEnabled } from './config'; const LEGACY_NOTES_NOT_GUESSED = 'actual-ai could not guess this category'; const LEGACY_NOTES_GUESSED = 'actual-ai guessed this category'; @@ -25,26 +26,18 @@ class TransactionService implements TransactionServiceI { private readonly guessedTag: string; - private readonly suggestNewCategories: boolean; - - private dryRun = true; - constructor( actualApiClient: ActualApiServiceI, llmService: LlmServiceI, promptGenerator: PromptGeneratorI, notGuessedTag: string, guessedTag: string, - suggestNewCategories = false, - dryRun = true, ) { this.actualApiService = actualApiClient; this.llmService = llmService; this.promptGenerator = promptGenerator; this.notGuessedTag = notGuessedTag; this.guessedTag = guessedTag; - this.suggestNewCategories = suggestNewCategories; - this.dryRun = dryRun; } appendTag(notes: string, tag: string): string { @@ -53,10 +46,14 @@ class TransactionService implements TransactionServiceI { } clearPreviousTags(notes: string): string { - return notes.replace(new RegExp(` ${this.guessedTag}`, 'g'), '') + return notes + .replace(new RegExp(` ${this.guessedTag}`, 'g'), '') .replace(new RegExp(` ${this.notGuessedTag}`, 'g'), '') .replace(new RegExp(` \\| ${LEGACY_NOTES_NOT_GUESSED}`, 'g'), '') .replace(new RegExp(` \\| ${LEGACY_NOTES_GUESSED}`, 'g'), '') + .replace(new RegExp(` ${LEGACY_NOTES_GUESSED}`, 'g'), '') + .replace(new RegExp(` ${LEGACY_NOTES_NOT_GUESSED}`, 'g'), '') + .replace(/-miss(?= #actual-ai)/g, '') // Only remove -miss if it's followed by the tag .trim(); } @@ -76,10 +73,14 @@ class TransactionService implements TransactionServiceI { let newNotes = null; if (transaction.notes?.includes(LEGACY_NOTES_NOT_GUESSED)) { - newNotes = this.appendTag(transaction.notes, this.notGuessedTag); + // Clean up the notes and add the tag + const baseNotes = this.clearPreviousTags(transaction.notes); + newNotes = `${baseNotes} ${this.notGuessedTag}`; } if (transaction.notes?.includes(LEGACY_NOTES_GUESSED)) { - newNotes = this.appendTag(transaction.notes, this.guessedTag); + // Clean up the notes and add the tag + const baseNotes = this.clearPreviousTags(transaction.notes); + newNotes = `${baseNotes} ${this.guessedTag}`; } if (newNotes) { @@ -89,7 +90,7 @@ class TransactionService implements TransactionServiceI { } async processTransactions(): Promise { - if (this.dryRun) { + if (isFeatureEnabled('dryRun')) { console.log('=== DRY RUN MODE ==='); console.log('No changes will be made to transactions or categories'); console.log('====================='); @@ -107,13 +108,17 @@ class TransactionService implements TransactionServiceI { .map((account) => account.id) ?? []; console.log(`Found ${rules.length} transaction categorization rules`); + console.log('rerunMissedTransactions', isFeatureEnabled('rerunMissedTransactions')); + const uncategorizedTransactions = transactions.filter( (transaction) => !transaction.category && (transaction.transfer_id === null || transaction.transfer_id === undefined) && transaction.starting_balance_flag !== true && transaction.imported_payee !== null && transaction.imported_payee !== '' - && !transaction.notes?.includes(this.notGuessedTag) + && ( + isFeatureEnabled('rerunMissedTransactions') ?? !transaction.notes?.includes(this.notGuessedTag) + ) && !transaction.is_parent && !accountsToSkip.includes(transaction.account), ); @@ -203,11 +208,11 @@ class TransactionService implements TransactionServiceI { } // Create new categories if not in dry run mode - if (this.suggestNewCategories && suggestedCategories.size > 0) { + if (isFeatureEnabled('suggestNewCategories') && suggestedCategories.size > 0) { // Optimize categories before applying/reporting const optimizedCategories = this.optimizeCategorySuggestions(suggestedCategories); - if (this.dryRun) { + if (isFeatureEnabled('dryRun')) { console.log(`\nDRY RUN: Would create ${optimizedCategories.size} new categories after optimization:`); Array.from(optimizedCategories.entries()).forEach(([_, suggestion]) => { console.log( @@ -215,6 +220,23 @@ class TransactionService implements TransactionServiceI { `for ${suggestion.transactions.length} transactions`, ); }); + } else if (isFeatureEnabled('dryRunNewCategories')) { + console.log(`\nDRY RUN CATEGORIES: Would create ${optimizedCategories.size} new categories:`); + Array.from(optimizedCategories.entries()).forEach(([_, suggestion]) => { + console.log( + `- ${suggestion.name} in ${suggestion.groupIsNew ? 'new' : 'existing'} group "${suggestion.groupName}"`, + `for ${suggestion.transactions.length} transactions`, + ); + }); + + // Don't create categories but log which transactions would be affected + uncategorizedTransactions.forEach((uncategorizedTransaction) => { + // Skip transactions needing new categories in dry run mode + console.log( + `Skipping categorization for '${uncategorizedTransaction.imported_payee}' ` + + 'as it needs a new category', + ); + }); } else { console.log(`Creating ${optimizedCategories.size} optimized categories`); @@ -276,7 +298,7 @@ class TransactionService implements TransactionServiceI { const category = categories.find((c) => c.id === response.categoryId); const categoryName = category ? category.name : 'Unknown Category'; - if (this.dryRun) { + if (isFeatureEnabled('dryRun')) { console.log(`DRY RUN: Would assign transaction ${transaction.id} to category "${categoryName}" (${response.categoryId}) via rule ${response.ruleName}`); return; } @@ -303,7 +325,7 @@ class TransactionService implements TransactionServiceI { return; } - if (this.dryRun) { + if (isFeatureEnabled('dryRun')) { console.log(`DRY RUN: Would assign transaction ${transaction.id} to existing category ${category.name}`); return; } diff --git a/src/utils/tool-service.ts b/src/utils/tool-service.ts index 0f7ba76..4d53a49 100644 --- a/src/utils/tool-service.ts +++ b/src/utils/tool-service.ts @@ -2,7 +2,7 @@ import * as https from 'https'; import { z } from 'zod'; import { tool, Tool } from 'ai'; import { ToolServiceI } from '../types'; -import { enabledTools } from '../config'; +import { getEnabledTools } from '../config'; interface SearchResult { title: string; @@ -24,7 +24,7 @@ export default class ToolService implements ToolServiceI { public getTools() { const tools: Record = {}; - if (Array.isArray(enabledTools) && enabledTools.includes('webSearch')) { + if (getEnabledTools().includes('webSearch')) { tools.webSearch = tool({ description: 'Essential for researching business types and industry categorizations when existing categories are insufficient. Use when payee is unfamiliar or category context is unclear', parameters: z.object({ diff --git a/tests/actual-ai.test.ts b/tests/actual-ai.test.ts index b22b292..56159fd 100644 --- a/tests/actual-ai.test.ts +++ b/tests/actual-ai.test.ts @@ -5,6 +5,16 @@ import MockedPromptGenerator from './test-doubles/mocked-prompt-generator'; import GivenActualData from './test-doubles/given/given-actual-data'; import ActualAiService from '../src/actual-ai'; +// Mock the config module +jest.mock('../src/config', () => ({ + isFeatureEnabled: (feature: string) => { + if (feature === 'dryRun' || feature === 'dryRunNewCategories') { + return false; + } + return true; + }, +})); + describe('ActualAiService', () => { let sut: ActualAiService; let transactionService: TransactionService; @@ -31,8 +41,6 @@ describe('ActualAiService', () => { mockedPromptGenerator, NOT_GUESSED_TAG, GUESSED_TAG, - false, - false, ); inMemoryApiService.setCategoryGroups(categoryGroups); @@ -244,15 +252,15 @@ describe('ActualAiService', () => { '1', -123, 'Carrefour 1235', - 'Carrefour XXXX1234567 822-307-2000 | actual-ai could not guess this category', + 'Carrefour XXXX1234567 822-307-2000 #actual-ai-miss', ); const transactionGuessed = GivenActualData.createTransaction( '2', -123, 'Carrefour 1234', - 'Carrefour XXXX1234567 822-307-3000 | actual-ai guessed this category', + 'Carrefour XXXX1234567 822-307-3000 actual-ai guessed this category', undefined, - '1', + GivenActualData.ACCOUNT_MAIN, '2021-01-01', false, GivenActualData.CATEGORY_GROCERIES, diff --git a/tests/prompt-generator.test.ts b/tests/prompt-generator.test.ts index 697462c..c5829a7 100644 --- a/tests/prompt-generator.test.ts +++ b/tests/prompt-generator.test.ts @@ -5,6 +5,10 @@ import PromptGenerator from '../src/prompt-generator'; import GivenActualData from './test-doubles/given/given-actual-data'; import PromptTemplateException from '../src/exceptions/prompt-template-exception'; import handlebars from '../src/handlebars-helpers'; +import * as config from '../src/config'; + +// Mock the isToolEnabled function +jest.spyOn(config, 'isToolEnabled').mockReturnValue(false); describe('PromptGenerator', () => { const promptTemplate = fs.readFileSync('./src/templates/prompt.hbs', 'utf8').trim(); @@ -26,7 +30,7 @@ describe('PromptGenerator', () => { '2', -1626, 'Steam Purc', - 'Steam Purc 16.26_V-miss #actual-ai-miss', + 'Steam Purc 16.26_V #actual-ai-miss', undefined, undefined, '2025-02-18', @@ -162,4 +166,51 @@ ANSWER BY A CATEGORY ID - DO NOT CREATE ENTIRE SENTENCE - DO NOT WRITE CATEGORY expect(prompt).toContain('* Type: Outcome'); expect(prompt).toContain('* Date: 2021-01-01'); }); + + // Add test cases for web search tool + describe('web search tool', () => { + it('should include web search tool message when enabled', () => { + jest.spyOn(config, 'isToolEnabled').mockReturnValue(true); + + const transaction = GivenActualData.createTransaction( + '1', + -1000, + 'Carrefour 2137', + '', + GivenActualData.PAYEE_CARREFOUR, + undefined, + '2021-01-01', + ); + + const categoryGroups = GivenActualData.createSampleCategoryGroups(); + const payees = GivenActualData.createSamplePayees(); + + const promptGenerator = new PromptGenerator(promptTemplate); + const prompt = promptGenerator.generate(categoryGroups, transaction, payees, []); + + expect(prompt).toContain('You can use the web search tool to find more information about the transaction.'); + }); + + it('should not include web search tool message when disabled', () => { + jest.spyOn(config, 'isToolEnabled').mockReturnValue(false); + + const transaction = GivenActualData.createTransaction( + '1', + -1000, + 'Carrefour 2137', + '', + GivenActualData.PAYEE_CARREFOUR, + undefined, + '2021-01-01', + ); + + const categoryGroups = GivenActualData.createSampleCategoryGroups(); + const payees = GivenActualData.createSamplePayees(); + + const promptGenerator = new PromptGenerator(promptTemplate); + const prompt = promptGenerator.generate(categoryGroups, transaction, payees, []); + + expect(prompt).not.toContain('You can use the web search tool to find more information about the transaction.'); + }); + }); }); diff --git a/tests/test-doubles/mocked-config.ts b/tests/test-doubles/mocked-config.ts new file mode 100644 index 0000000..2e4587b --- /dev/null +++ b/tests/test-doubles/mocked-config.ts @@ -0,0 +1,9 @@ +// Mock implementation of config to disable dryRun for tests +const isFeatureEnabled = (feature: string): boolean => { + if (feature === 'dryRun' || feature === 'dryRunNewCategories') { + return false; + } + return true; +}; + +export default isFeatureEnabled; diff --git a/tests/test-doubles/mocked-llm-service.test.ts b/tests/test-doubles/mocked-llm-service.test.ts new file mode 100644 index 0000000..3635445 --- /dev/null +++ b/tests/test-doubles/mocked-llm-service.test.ts @@ -0,0 +1,39 @@ +import MockedLlmService from './mocked-llm-service'; +import GivenActualData from './given/given-actual-data'; + +describe('MockedLlmService', () => { + let mockedLlmService: MockedLlmService; + + beforeEach(() => { + mockedLlmService = new MockedLlmService(); + }); + + it('should return the default response when no guess is set', async () => { + const response = await mockedLlmService.ask('test prompt'); + expect(response).toEqual({ + type: 'existing', + categoryId: 'uncategorized', + }); + }); + + it('should set the response when setGuess is called with a UUID', async () => { + const categoryId = GivenActualData.CATEGORY_GROCERIES; + mockedLlmService.setGuess(categoryId); + + const response = await mockedLlmService.ask('test prompt'); + expect(response).toEqual({ + type: 'existing', + categoryId, + }); + }); + + it('should set the response when setGuess is called with a category name', async () => { + mockedLlmService.setGuess('Groceries'); + + const response = await mockedLlmService.ask('test prompt'); + expect(response).toEqual({ + type: 'existing', + categoryId: GivenActualData.CATEGORY_GROCERIES, + }); + }); +}); diff --git a/tests/test-doubles/mocked-llm-service.ts b/tests/test-doubles/mocked-llm-service.ts index 974164f..1461062 100644 --- a/tests/test-doubles/mocked-llm-service.ts +++ b/tests/test-doubles/mocked-llm-service.ts @@ -7,7 +7,7 @@ export default class MockedLlmService implements LlmServiceI { categoryId: 'uncategorized', }; - async ask(): Promise { + async ask(_prompt: string, _categoryIds?: string[]): Promise { return Promise.resolve(this.response); } diff --git a/tests/test-doubles/mocked-prompt-generator.ts b/tests/test-doubles/mocked-prompt-generator.ts index 78fa05c..b34f7a4 100644 --- a/tests/test-doubles/mocked-prompt-generator.ts +++ b/tests/test-doubles/mocked-prompt-generator.ts @@ -1,6 +1,6 @@ import { APICategoryGroupEntity, APIPayeeEntity } from '@actual-app/api/@types/loot-core/server/api-models'; import { RuleEntity, TransactionEntity } from '@actual-app/api/@types/loot-core/types/models'; -import { APICategoryEntity, PromptGeneratorI, RuleDescription } from '../../src/types'; +import { PromptGeneratorI } from '../../src/types'; export default class MockedPromptGenerator implements PromptGeneratorI { generate( @@ -11,12 +11,4 @@ export default class MockedPromptGenerator implements PromptGeneratorI { ): string { return 'mocked prompt'; } - - transformRulesToDescriptions( - _rules: RuleEntity[], - _categories: APICategoryEntity[], - _payees: APIPayeeEntity[], - ): RuleDescription[] { - return []; - } } From d3e9d39683c7afb0a2839c09e13e019fb5f955d4 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sat, 8 Mar 2025 00:21:18 -0500 Subject: [PATCH 09/17] Add VSCode debug configuration for tests --- .vscode/launch.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.vscode/launch.json b/.vscode/launch.json index 83db09b..1af4409 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -16,6 +16,25 @@ "sourceMaps": true, "console": "integratedTerminal", "cwd": "${workspaceFolder}" + }, + { + "type": "node", + "request": "launch", + "name": "Debug Tests", + "runtimeExecutable": "npm", + "runtimeArgs": [ + "run", + "test", + "--", + "--runInBand", + "--watchAll=false" + ], + "skipFiles": [ + "/**" + ], + "sourceMaps": true, + "console": "integratedTerminal", + "cwd": "${workspaceFolder}" } ] } \ No newline at end of file From 359f7881909744a1092053b8939a0efddadb1ff7 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sat, 8 Mar 2025 01:51:02 -0500 Subject: [PATCH 10/17] Refactor feature flag and transaction processing logic with improved test coverage --- src/actual-ai.ts | 1 - src/config.ts | 9 ----- src/llm-service.ts | 3 -- src/transaction-service.ts | 37 +++++++++--------- tests/actual-ai.test.ts | 78 +++++++++++++++++++++++++++++++++----- 5 files changed, 87 insertions(+), 41 deletions(-) diff --git a/src/actual-ai.ts b/src/actual-ai.ts index b41d524..fe751ae 100644 --- a/src/actual-ai.ts +++ b/src/actual-ai.ts @@ -35,7 +35,6 @@ class ActualAiService implements ActualAiServiceI { ); } - // These should run even if sync failed await this.transactionService.migrateToTags(); try { diff --git a/src/config.ts b/src/config.ts index e668a15..2934ca0 100644 --- a/src/config.ts +++ b/src/config.ts @@ -33,7 +33,6 @@ export const groqModel = process.env.GROQ_MODEL ?? 'llama-3.3-70b-versatile'; export const groqBaseURL = process.env.GROQ_BASE_URL ?? 'https://api.groq.com/openai/v1'; export const valueSerpApiKey = process.env.VALUESERP_API_KEY ?? ''; -// Feature Flags System export interface FeatureFlag { enabled: boolean; defaultValue: boolean; @@ -59,30 +58,25 @@ try { console.warn('Failed to parse FEATURES environment variable, ignoring', e); } -// Register standard features with defaults function registerStandardFeatures() { - // Suggest new categories (disabled by default) features.suggestNewCategories = { enabled: enabledFeatures.includes('suggestNewCategories'), defaultValue: false, description: 'Suggest new categories for transactions that cannot be classified', }; - // Dry run mode (enabled by default) features.dryRun = { enabled: enabledFeatures.includes('dryRun'), defaultValue: true, description: 'Run in dry mode without actually making changes', }; - // Dry run for new categories (enabled by default) features.dryRunNewCategories = { enabled: enabledFeatures.includes('dryRunNewCategories'), defaultValue: true, description: 'Only log suggested categories without creating them', }; - // Rerun missed transactions (disabled by default) features.rerunMissedTransactions = { enabled: enabledFeatures.includes('rerunMissedTransactions'), defaultValue: false, @@ -90,14 +84,11 @@ function registerStandardFeatures() { }; } -// Register available tools as features function registerToolFeatures() { - // Parse tools from ENABLED_TOOLS for backward compatibility const legacyTools = (process.env.ENABLED_TOOLS ?? '').split(',') .map((tool) => tool.trim()) .filter(Boolean); - // Register webSearch tool features.webSearch = { enabled: enabledFeatures.includes('webSearch') || legacyTools.includes('webSearch'), defaultValue: false, diff --git a/src/llm-service.ts b/src/llm-service.ts index 4706569..1c6fc0e 100644 --- a/src/llm-service.ts +++ b/src/llm-service.ts @@ -63,7 +63,6 @@ export default class LlmService implements LlmServiceI { try { console.log(`Making LLM request to ${this.provider}${this.isFallbackMode ? ' (fallback mode)' : ''}`); - // In fallback mode, return a UnifiedResponse with the string as categoryId if (this.isFallbackMode) { const result = await this.askUsingFallbackModel(prompt); return { @@ -72,7 +71,6 @@ export default class LlmService implements LlmServiceI { }; } - // If categoryIds are provided, use enum selection and return as UnifiedResponse if (categoryIds && categoryIds.length > 0) { const result = await this.askWithEnum(prompt, categoryIds); return { @@ -81,7 +79,6 @@ export default class LlmService implements LlmServiceI { }; } - // Otherwise, handle unified response return this.rateLimiter.executeWithRateLimiting(this.provider, async () => { try { const { text } = await generateText({ diff --git a/src/transaction-service.ts b/src/transaction-service.ts index cf70c8c..4f175bf 100644 --- a/src/transaction-service.ts +++ b/src/transaction-service.ts @@ -13,7 +13,7 @@ import { isFeatureEnabled } from './config'; const LEGACY_NOTES_NOT_GUESSED = 'actual-ai could not guess this category'; const LEGACY_NOTES_GUESSED = 'actual-ai guessed this category'; -const BATCH_SIZE = 20; // Process transactions in batches of 20 +const BATCH_SIZE = 20; class TransactionService implements TransactionServiceI { private readonly actualApiService: ActualApiServiceI; @@ -47,13 +47,13 @@ class TransactionService implements TransactionServiceI { clearPreviousTags(notes: string): string { return notes - .replace(new RegExp(` ${this.guessedTag}`, 'g'), '') - .replace(new RegExp(` ${this.notGuessedTag}`, 'g'), '') - .replace(new RegExp(` \\| ${LEGACY_NOTES_NOT_GUESSED}`, 'g'), '') - .replace(new RegExp(` \\| ${LEGACY_NOTES_GUESSED}`, 'g'), '') - .replace(new RegExp(` ${LEGACY_NOTES_GUESSED}`, 'g'), '') - .replace(new RegExp(` ${LEGACY_NOTES_NOT_GUESSED}`, 'g'), '') - .replace(/-miss(?= #actual-ai)/g, '') // Only remove -miss if it's followed by the tag + .replace(new RegExp(`\\s*${this.guessedTag}`, 'g'), '') + .replace(new RegExp(`\\s*${this.notGuessedTag}`, 'g'), '') + .replace(new RegExp(`\\s*\\|\\s*${LEGACY_NOTES_NOT_GUESSED}`, 'g'), '') + .replace(new RegExp(`\\s*\\|\\s*${LEGACY_NOTES_GUESSED}`, 'g'), '') + .replace(new RegExp(`\\s*${LEGACY_NOTES_GUESSED}`, 'g'), '') + .replace(new RegExp(`\\s*${LEGACY_NOTES_NOT_GUESSED}`, 'g'), '') + .replace(/-miss(?= #actual-ai)/g, '') .trim(); } @@ -71,19 +71,16 @@ class TransactionService implements TransactionServiceI { const transaction = transactionsToMigrate[i]; console.log(`${i + 1}/${transactionsToMigrate.length} Migrating transaction ${transaction.imported_payee} / ${transaction.notes} / ${transaction.amount}`); - let newNotes = null; + const baseNotes = this.clearPreviousTags(transaction.notes ?? ''); + let newNotes = baseNotes; + if (transaction.notes?.includes(LEGACY_NOTES_NOT_GUESSED)) { - // Clean up the notes and add the tag - const baseNotes = this.clearPreviousTags(transaction.notes); - newNotes = `${baseNotes} ${this.notGuessedTag}`; - } - if (transaction.notes?.includes(LEGACY_NOTES_GUESSED)) { - // Clean up the notes and add the tag - const baseNotes = this.clearPreviousTags(transaction.notes); - newNotes = `${baseNotes} ${this.guessedTag}`; + newNotes = this.appendTag(baseNotes, this.notGuessedTag); + } else if (transaction.notes?.includes(LEGACY_NOTES_GUESSED)) { + newNotes = this.appendTag(baseNotes, this.guessedTag); } - if (newNotes) { + if (newNotes !== transaction.notes) { await this.actualApiService.updateTransactionNotes(transaction.id, newNotes); } } @@ -117,7 +114,9 @@ class TransactionService implements TransactionServiceI { && transaction.imported_payee !== null && transaction.imported_payee !== '' && ( - isFeatureEnabled('rerunMissedTransactions') ?? !transaction.notes?.includes(this.notGuessedTag) + isFeatureEnabled('rerunMissedTransactions') + ? true // Include all if rerun enabled + : !transaction.notes?.includes(this.notGuessedTag) ) && !transaction.is_parent && !accountsToSkip.includes(transaction.account), diff --git a/tests/actual-ai.test.ts b/tests/actual-ai.test.ts index 56159fd..8be7c58 100644 --- a/tests/actual-ai.test.ts +++ b/tests/actual-ai.test.ts @@ -4,16 +4,18 @@ import MockedLlmService from './test-doubles/mocked-llm-service'; import MockedPromptGenerator from './test-doubles/mocked-prompt-generator'; import GivenActualData from './test-doubles/given/given-actual-data'; import ActualAiService from '../src/actual-ai'; +import * as config from '../src/config'; -// Mock the config module -jest.mock('../src/config', () => ({ - isFeatureEnabled: (feature: string) => { - if (feature === 'dryRun' || feature === 'dryRunNewCategories') { - return false; - } - return true; - }, -})); +// Create a reusable mock for isFeatureEnabled +const originalIsFeatureEnabled = config.isFeatureEnabled; +const mockIsFeatureEnabled = jest.spyOn(config, 'isFeatureEnabled'); + +// Default to having rerunMissedTransactions off for most tests +mockIsFeatureEnabled.mockImplementation((feature: string) => { + if (feature === 'rerunMissedTransactions') return false; + if (feature === 'dryRun' || feature === 'dryRunNewCategories') return false; + return originalIsFeatureEnabled(feature); +}); describe('ActualAiService', () => { let sut: ActualAiService; @@ -26,6 +28,13 @@ describe('ActualAiService', () => { const NOT_GUESSED_TAG = '#actual-ai-miss'; beforeEach(() => { + // Reset mock implementation before each test + mockIsFeatureEnabled.mockImplementation((feature: string) => { + if (feature === 'rerunMissedTransactions') return false; + if (feature === 'dryRun' || feature === 'dryRunNewCategories') return false; + return originalIsFeatureEnabled(feature); + }); + inMemoryApiService = new InMemoryActualApiService(); mockedLlmService = new MockedLlmService(); mockedPromptGenerator = new MockedPromptGenerator(); @@ -50,6 +59,10 @@ describe('ActualAiService', () => { inMemoryApiService.setRules(rules); }); + afterEach(() => { + mockIsFeatureEnabled.mockReset(); + }); + it('It should assign a category to transaction', async () => { // Arrange const transaction = GivenActualData.createTransaction( @@ -206,6 +219,13 @@ describe('ActualAiService', () => { inMemoryApiService.setTransactions([transaction]); mockedLlmService.setGuess(GivenActualData.CATEGORY_GROCERIES); + // Ensure rerunMissedTransactions is false for this test + mockIsFeatureEnabled.mockImplementation((feature: string) => { + if (feature === 'rerunMissedTransactions') return false; + if (feature === 'dryRun' || feature === 'dryRunNewCategories') return false; + return originalIsFeatureEnabled(feature); + }); + // Act sut = new ActualAiService( transactionService, @@ -343,6 +363,13 @@ describe('ActualAiService', () => { inMemoryApiService.setTransactions([transaction1, transaction2, transaction3]); mockedLlmService.setGuess(GivenActualData.CATEGORY_GROCERIES); + // Ensure rerunMissedTransactions is false for this test + mockIsFeatureEnabled.mockImplementation((feature: string) => { + if (feature === 'rerunMissedTransactions') return false; + if (feature === 'dryRun' || feature === 'dryRunNewCategories') return false; + return originalIsFeatureEnabled(feature); + }); + // Act sut = new ActualAiService( transactionService, @@ -374,4 +401,37 @@ describe('ActualAiService', () => { // Assert expect(inMemoryApiService.getWasBankSyncRan()).toBe(true); }); + + // Add a new test for when rerunMissedTransactions is true + it('It should process transaction with missed tag when rerunMissedTransactions is true', async () => { + // Arrange + const transaction = GivenActualData.createTransaction( + '1', + -123, + 'Carrefour 1234', + 'Carrefour XXXX1234567 822-307-2000 | #actual-ai-miss', + ); + inMemoryApiService.setTransactions([transaction]); + mockedLlmService.setGuess(GivenActualData.CATEGORY_GROCERIES); + + // Set rerunMissedTransactions to true for this test + mockIsFeatureEnabled.mockImplementation((feature: string) => { + if (feature === 'rerunMissedTransactions') return true; + if (feature === 'dryRun' || feature === 'dryRunNewCategories') return false; + return originalIsFeatureEnabled(feature); + }); + + // Act + sut = new ActualAiService( + transactionService, + inMemoryApiService, + syncAccountsBeforeClassify, + ); + await sut.classify(); + + // Assert + const updatedTransactions = await inMemoryApiService.getTransactions(); + expect(updatedTransactions[0].category).toBe(GivenActualData.CATEGORY_GROCERIES); + expect(updatedTransactions[0].notes).toContain(GUESSED_TAG); + }); }); From 5a81371424176f94e0e8c499ce11dc3e459b2f8a Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sat, 8 Mar 2025 02:00:16 -0500 Subject: [PATCH 11/17] Bump version to 2.0.0 given how much has changed --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7b74944..0e3723a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@sakowicz/actual-ai", - "version": "1.7.8", + "version": "2.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@sakowicz/actual-ai", - "version": "1.7.8", + "version": "2.0.0", "license": "MIT", "dependencies": { "@actual-app/api": "^25.3.1", diff --git a/package.json b/package.json index 0a7797d..2bcf242 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@sakowicz/actual-ai", - "version": "1.8.0", + "version": "2.0.0", "description": "Transaction AI classification for Actual Budget app.", "main": "app.js", "scripts": { From 07aba42c258c4a67d190d6d0aad7a96c2eb70290 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 23 Mar 2025 00:27:39 -0400 Subject: [PATCH 12/17] Add unit tests for RateLimiter --- tests/utils/rate-limiter.test.ts | 117 +++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 tests/utils/rate-limiter.test.ts diff --git a/tests/utils/rate-limiter.test.ts b/tests/utils/rate-limiter.test.ts new file mode 100644 index 0000000..6bebdcf --- /dev/null +++ b/tests/utils/rate-limiter.test.ts @@ -0,0 +1,117 @@ +import RateLimiter from '../../src/utils/rate-limiter'; + +describe('RateLimiter', () => { + let rateLimiter: RateLimiter; + + beforeEach(() => { + rateLimiter = new RateLimiter(); + jest.useFakeTimers(); + // Mock the sleep function to resolve immediately + jest.spyOn(rateLimiter as unknown as { sleep: (ms: number) => Promise }, 'sleep') + .mockImplementation(() => Promise.resolve()); + }); + + afterEach(() => { + jest.useRealTimers(); + jest.restoreAllMocks(); + }); + + describe('setProviderLimit', () => { + it('should set the provider limit', async () => { + rateLimiter.setProviderLimit('test-provider', 10); + + // Create a test function that will be rate limited + const operation = jest.fn().mockResolvedValue('success'); + + // Execute the operation multiple times + for (let i = 0; i < 9; i++) { + await rateLimiter.executeWithRateLimiting('test-provider', operation); + } + + // Verify the operation was called the expected number of times + expect(operation).toHaveBeenCalledTimes(9); + }, 10000); // Increase timeout to 10 seconds + }); + + describe('executeWithRateLimiting', () => { + it('should execute the operation successfully', async () => { + const operation = jest.fn().mockResolvedValue('success'); + const result = await rateLimiter.executeWithRateLimiting('test-provider', operation); + + expect(result).toBe('success'); + expect(operation).toHaveBeenCalledTimes(1); + }); + + it('should retry on rate limit errors', async () => { + // Create a mock operation that fails with a rate limit error on first call + const rateLimitError = new Error('rate limit exceeded'); + Object.assign(rateLimitError, { statusCode: 429 }); + + const operation = jest.fn() + .mockRejectedValueOnce(rateLimitError) + .mockResolvedValueOnce('success'); + + const result = await rateLimiter.executeWithRateLimiting('test-provider', operation, { + maxRetries: 3, + baseDelayMs: 100, + maxDelayMs: 1000, + jitter: false, + }); + + // Fast-forward timer to simulate waiting + jest.advanceTimersByTime(100); + + expect(result).toBe('success'); + expect(operation).toHaveBeenCalledTimes(2); + }, 10000); // Increase timeout to 10 seconds + + it('should throw after max retries', async () => { + // Create a mock operation that always fails with a rate limit error + const rateLimitError = new Error('rate limit exceeded'); + Object.assign(rateLimitError, { statusCode: 429 }); + + const operation = jest.fn().mockRejectedValue(rateLimitError); + + await expect(rateLimiter.executeWithRateLimiting('test-provider', operation, { + maxRetries: 2, + baseDelayMs: 100, + maxDelayMs: 1000, + jitter: false, + })).rejects.toThrow('Rate limit retries exceeded'); + + // Fast-forward timer for each retry + jest.advanceTimersByTime(100); // First retry + jest.advanceTimersByTime(200); // Second retry (exponential backoff) + + expect(operation).toHaveBeenCalledTimes(3); // Initial + 2 retries + }, 10000); // Increase timeout to 10 seconds + + it('should extract retry time from error message', async () => { + // Skip this test for now as there's an issue with the error message in the test environment + const operation = jest.fn().mockResolvedValue('success'); + const result = await rateLimiter.executeWithRateLimiting('test-provider', operation); + expect(result).toBe('success'); + }, 10000); // Increase timeout to 10 seconds + + it('should handle non-rate limit errors', async () => { + const regularError = new Error('regular error'); + const operation = jest.fn().mockRejectedValue(regularError); + + await expect(rateLimiter.executeWithRateLimiting('test-provider', operation)) + .rejects.toThrow('regular error'); + + expect(operation).toHaveBeenCalledTimes(1); + }); + }); + + describe('debug mode', () => { + it('should enable debug mode', () => { + const debugRateLimiter = new RateLimiter(true); + expect(debugRateLimiter).toBeDefined(); + + // Alternative way to enable debug + rateLimiter.enableDebug(); + expect(rateLimiter).toBeDefined(); + }); + }); +}); From c56dd2f117d083038f7d2fae0b93a118e9ba44d0 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 23 Mar 2025 00:44:17 -0400 Subject: [PATCH 13/17] Replace env vars with features for startup options --- .env.example | 4 +--- app.ts | 4 ++-- src/config.ts | 23 ++++++++++++++++++++--- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index a20ba88..62f2a33 100644 --- a/.env.example +++ b/.env.example @@ -2,11 +2,9 @@ ACTUAL_SERVER_URL=http://actual_server:5006 ACTUAL_PASSWORD= ACTUAL_BUDGET_ID= CLASSIFICATION_SCHEDULE_CRON="0 */4 * * *" -CLASSIFY_ON_STARTUP=true -SYNC_ACCOUNTS_BEFORE_CLASSIFY=true # Feature flags - can be specified as an array -FEATURES="['webSearch', 'suggestNewCategories', 'rerunMissedTransactions']" +FEATURES='["freeWebSearch", "suggestNewCategories", "rerunMissedTransactions", "classifyOnStartup", "syncAccountsBeforeClassify"]' DRY_RUN=true # Tools and API keys diff --git a/app.ts b/app.ts index b59c357..26ee8fc 100644 --- a/app.ts +++ b/app.ts @@ -1,5 +1,5 @@ import cron from 'node-cron'; -import { cronSchedule, classifyOnStartup } from './src/config'; +import { cronSchedule, isFeatureEnabled } from './src/config'; import actualAi from './src/container'; if (!cron.validate(cronSchedule)) { @@ -12,7 +12,7 @@ cron.schedule(cronSchedule, async () => { }); console.log('Application started'); -if (classifyOnStartup) { +if (isFeatureEnabled('classifyOnStartup')) { (async () => { await actualAi.classify(); })(); diff --git a/src/config.ts b/src/config.ts index 2934ca0..0a715bb 100644 --- a/src/config.ts +++ b/src/config.ts @@ -10,8 +10,6 @@ export const password = process.env.ACTUAL_PASSWORD ?? ''; export const budgetId = process.env.ACTUAL_BUDGET_ID ?? ''; export const e2ePassword = process.env.ACTUAL_E2E_PASSWORD ?? ''; export const cronSchedule = process.env.CLASSIFICATION_SCHEDULE_CRON ?? ''; -export const classifyOnStartup = process.env.CLASSIFY_ON_STARTUP === 'true'; -export const syncAccountsBeforeClassify = process.env.SYNC_ACCOUNTS_BEFORE_CLASSIFY === 'true'; export const llmProvider = process.env.LLM_PROVIDER ?? 'openai'; export const openaiBaseURL = process.env.OPENAI_BASE_URL ?? 'https://api.openai.com/v1'; export const openaiApiKey = process.env.OPENAI_API_KEY ?? ''; @@ -66,7 +64,7 @@ function registerStandardFeatures() { }; features.dryRun = { - enabled: enabledFeatures.includes('dryRun'), + enabled: enabledFeatures.includes('dryRun') || process.env.DRY_RUN === 'true', defaultValue: true, description: 'Run in dry mode without actually making changes', }; @@ -82,6 +80,18 @@ function registerStandardFeatures() { defaultValue: false, description: 'Re-process transactions marked as not guessed', }; + + features.classifyOnStartup = { + enabled: enabledFeatures.includes('classifyOnStartup') || process.env.CLASSIFY_ON_STARTUP === 'true', + defaultValue: false, + description: 'Run classification when the application starts', + }; + + features.syncAccountsBeforeClassify = { + enabled: enabledFeatures.includes('syncAccountsBeforeClassify') || process.env.SYNC_ACCOUNTS_BEFORE_CLASSIFY === 'true', + defaultValue: false, + description: 'Sync accounts before running classification', + }; } function registerToolFeatures() { @@ -96,6 +106,13 @@ function registerToolFeatures() { options: ['webSearch'], }; + features.freeWebSearch = { + enabled: enabledFeatures.includes('freeWebSearch') || legacyTools.includes('freeWebSearch'), + defaultValue: false, + description: 'Enable free web search capability for merchant lookup (self-hosted alternative to ValueSerp)', + options: ['freeWebSearch'], + }; + // Additional tools can be added here following the same pattern // features.newTool = { // enabled: enabledFeatures.includes('newTool'), From e153b523589b73985ce60f3bf59da9cda6252611 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 23 Mar 2025 01:14:05 -0400 Subject: [PATCH 14/17] Add tests for RateLimiter to enforce rate limits and handle retries with logging. --- src/utils/rate-limiter.ts | 4 +- tests/utils/rate-limiter.test.ts | 187 ++++++++++++++++++++++++++++--- 2 files changed, 175 insertions(+), 16 deletions(-) diff --git a/src/utils/rate-limiter.ts b/src/utils/rate-limiter.ts index 5ae0006..dc51957 100644 --- a/src/utils/rate-limiter.ts +++ b/src/utils/rate-limiter.ts @@ -196,7 +196,7 @@ export class RateLimiter { const headers = rateLimitError.responseHeaders; if (headers && 'retry-after' in headers) { const retryAfter = headers['retry-after']; - if (retryAfter && Number.isNaN(Number(retryAfter)) === false) { + if (retryAfter && !Number.isNaN(Number(retryAfter))) { return Number(retryAfter) * 1000; } } @@ -271,7 +271,7 @@ export class RateLimiter { } } - private sleep(ms: number): Promise { + protected sleep(ms: number): Promise { return new Promise((resolve) => { setTimeout(resolve, ms); }); diff --git a/tests/utils/rate-limiter.test.ts b/tests/utils/rate-limiter.test.ts index 6bebdcf..111ee1b 100644 --- a/tests/utils/rate-limiter.test.ts +++ b/tests/utils/rate-limiter.test.ts @@ -2,6 +2,7 @@ import RateLimiter from '../../src/utils/rate-limiter'; describe('RateLimiter', () => { let rateLimiter: RateLimiter; + let consoleSpy: jest.SpyInstance; beforeEach(() => { rateLimiter = new RateLimiter(); @@ -9,11 +10,13 @@ describe('RateLimiter', () => { // Mock the sleep function to resolve immediately jest.spyOn(rateLimiter as unknown as { sleep: (ms: number) => Promise }, 'sleep') .mockImplementation(() => Promise.resolve()); + consoleSpy = jest.spyOn(console, 'log').mockImplementation(); }); afterEach(() => { jest.useRealTimers(); jest.restoreAllMocks(); + consoleSpy.mockRestore(); }); describe('setProviderLimit', () => { @@ -30,7 +33,28 @@ describe('RateLimiter', () => { // Verify the operation was called the expected number of times expect(operation).toHaveBeenCalledTimes(9); - }, 10000); // Increase timeout to 10 seconds + }); + + it('should enforce rate limits when approaching the limit', async () => { + // Set a low provider limit + rateLimiter.setProviderLimit('test-provider', 5); + + const operation = jest.fn().mockResolvedValue('success'); + + // Execute operations up to 80% of the limit + for (let i = 0; i < 4; i++) { + await rateLimiter.executeWithRateLimiting('test-provider', operation); + } + + expect(operation).toHaveBeenCalledTimes(4); + + // Next operation should trigger preemptive waiting + await rateLimiter.executeWithRateLimiting('test-provider', operation); + + // Verify the sleep was called with correct delay + // This implicitly tests waitIfNeeded when count >= limit * 0.8 + expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('Preemptively waiting')); + }); }); describe('executeWithRateLimiting', () => { @@ -42,8 +66,7 @@ describe('RateLimiter', () => { expect(operation).toHaveBeenCalledTimes(1); }); - it('should retry on rate limit errors', async () => { - // Create a mock operation that fails with a rate limit error on first call + it('should retry on rate limit errors with status code 429', async () => { const rateLimitError = new Error('rate limit exceeded'); Object.assign(rateLimitError, { statusCode: 429 }); @@ -58,15 +81,34 @@ describe('RateLimiter', () => { jitter: false, }); - // Fast-forward timer to simulate waiting jest.advanceTimersByTime(100); expect(result).toBe('success'); expect(operation).toHaveBeenCalledTimes(2); - }, 10000); // Increase timeout to 10 seconds + expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('Rate limit hit for test-provider')); + }); + + it('should retry on rate limit errors with rate limit message', async () => { + const rateLimitError = new Error('too many requests, please try again later'); + + const operation = jest.fn() + .mockRejectedValueOnce(rateLimitError) + .mockResolvedValueOnce('success'); + + const result = await rateLimiter.executeWithRateLimiting('test-provider', operation, { + maxRetries: 3, + baseDelayMs: 100, + maxDelayMs: 1000, + jitter: false, + }); + + jest.advanceTimersByTime(100); + + expect(result).toBe('success'); + expect(operation).toHaveBeenCalledTimes(2); + }); it('should throw after max retries', async () => { - // Create a mock operation that always fails with a rate limit error const rateLimitError = new Error('rate limit exceeded'); Object.assign(rateLimitError, { statusCode: 429 }); @@ -79,19 +121,44 @@ describe('RateLimiter', () => { jitter: false, })).rejects.toThrow('Rate limit retries exceeded'); - // Fast-forward timer for each retry jest.advanceTimersByTime(100); // First retry jest.advanceTimersByTime(200); // Second retry (exponential backoff) expect(operation).toHaveBeenCalledTimes(3); // Initial + 2 retries - }, 10000); // Increase timeout to 10 seconds + }); it('should extract retry time from error message', async () => { - // Skip this test for now as there's an issue with the error message in the test environment - const operation = jest.fn().mockResolvedValue('success'); + const rateLimitError = new Error('Rate limit exceeded. Please try again in 2m14s'); + const operation = jest.fn() + .mockRejectedValueOnce(rateLimitError) + .mockResolvedValueOnce('success'); + + const result = await rateLimiter.executeWithRateLimiting('test-provider', operation); + expect(result).toBe('success'); + + // Should mention waiting with a time close to the parsed value (2m14s = 134000ms) + expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('Rate limit hit for test-provider. Waiting 134000ms')); + }); + + it('should extract retry time from headers', async () => { + const rateLimitError = new Error('rate limit exceeded'); + Object.assign(rateLimitError, { + statusCode: 429, + responseHeaders: { + 'retry-after': '10', + }, + }); + + const operation = jest.fn() + .mockRejectedValueOnce(rateLimitError) + .mockResolvedValueOnce('success'); + const result = await rateLimiter.executeWithRateLimiting('test-provider', operation); expect(result).toBe('success'); - }, 10000); // Increase timeout to 10 seconds + + // Should mention waiting 10 seconds (10000ms) + expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('Rate limit hit for test-provider. Waiting 10000ms')); + }); it('should handle non-rate limit errors', async () => { const regularError = new Error('regular error'); @@ -102,16 +169,108 @@ describe('RateLimiter', () => { expect(operation).toHaveBeenCalledTimes(1); }); + + it('should apply exponential backoff with jitter', async () => { + // Mock Math.random to return a consistent value for testability + const randomSpy = jest.spyOn(Math, 'random').mockReturnValue(0.5); + + const rateLimitError = new Error('rate limit exceeded'); + Object.assign(rateLimitError, { statusCode: 429 }); + + const operation = jest.fn() + .mockRejectedValueOnce(rateLimitError) + .mockRejectedValueOnce(rateLimitError) + .mockResolvedValueOnce('success'); + + await rateLimiter.executeWithRateLimiting('test-provider', operation, { + maxRetries: 3, + baseDelayMs: 1000, + maxDelayMs: 10000, + jitter: true, + }); + + // First retry should have baseDelay with jitter: 1000ms * 0.75 = 750ms + // Second retry should have exponential backoff: 2000ms * 0.75 = 1500ms + expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('Waiting 750ms')); + expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('Waiting 1500ms')); + + randomSpy.mockRestore(); + }); + }); + + describe('token bucket handling', () => { + it('should update token bucket from Groq error message', async () => { + const groqError = new Error('Limit 100000, Used 99336, Requested 821. Please try again in 30s'); + Object.assign(groqError, { statusCode: 429 }); + + const operation = jest.fn() + .mockRejectedValueOnce(groqError) + .mockResolvedValueOnce('success'); + + rateLimiter.enableDebug(); + + const result = await rateLimiter.executeWithRateLimiting('groq', operation); + expect(result).toBe('success'); + + expect(consoleSpy).toHaveBeenCalled(); + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining('Rate limit details for groq:'), + expect.objectContaining({ + tokenBucket: expect.objectContaining({ + limit: 100000, + remaining: 664, + }) as unknown as { + limit: number; + remaining: number; + resetTimestamp?: number; + }, + }), + ); + }); + + it('should wait for token bucket reset when close to limit', async () => { + const groqError = new Error('Limit 100, Used 95, Requested 5. Please try again in 30s'); + Object.assign(groqError, { statusCode: 429 }); + + const operation1 = jest.fn() + .mockRejectedValueOnce(groqError) + .mockResolvedValueOnce('success'); + + await rateLimiter.executeWithRateLimiting('groq', operation1); + + // Now try a second operation that should trigger waiting due to low token bucket + const operation2 = jest.fn().mockResolvedValue('second-success'); + await rateLimiter.executeWithRateLimiting('groq', operation2); + + // Should have logged waiting for token bucket + expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('Waiting') && expect.stringContaining('for token bucket to reset for groq')); + }); }); describe('debug mode', () => { - it('should enable debug mode', () => { + it('should enable debug mode through constructor', () => { const debugRateLimiter = new RateLimiter(true); expect(debugRateLimiter).toBeDefined(); + }); - // Alternative way to enable debug + it('should enable debug mode through method call', () => { rateLimiter.enableDebug(); - expect(rateLimiter).toBeDefined(); + + // Create a rate limit error to trigger debug logging + const rateLimitError = new Error('rate limit exceeded'); + Object.assign(rateLimitError, { statusCode: 429 }); + + const operation = jest.fn() + .mockRejectedValueOnce(rateLimitError) + .mockResolvedValueOnce('success'); + + return rateLimiter.executeWithRateLimiting('test-provider', operation) + .then(() => { + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining('Rate limit details for test-provider:'), + expect.any(Object), + ); + }); }); }); }); From c6e98705a903255cde805f8b350548bd140abd52 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 23 Mar 2025 01:15:47 -0400 Subject: [PATCH 15/17] Remove extra rules from prompt template added by accident --- src/templates/prompt.hbs | 4 ---- test-handlebars.ts | 49 ---------------------------------------- 2 files changed, 53 deletions(-) delete mode 100644 test-handlebars.ts diff --git a/src/templates/prompt.hbs b/src/templates/prompt.hbs index 377bff4..45d0184 100644 --- a/src/templates/prompt.hbs +++ b/src/templates/prompt.hbs @@ -50,10 +50,6 @@ Examples: {"type": "rule", "categoryId": "def456", "ruleName": "Coffee Shop"} {"type": "new", "newCategory": {"name": "Pet Supplies", "groupName": "Pets", "groupIsNew": true}} -Extra rules: -* If the transaction is a Credit Card Payment, categorize it as "Transfer" unless it is a fee. -* Flowers can go in the "Gift" category. - {{#if hasWebSearchTool}} You can use the web search tool to find more information about the transaction. {{/if}} diff --git a/test-handlebars.ts b/test-handlebars.ts deleted file mode 100644 index 50ec350..0000000 --- a/test-handlebars.ts +++ /dev/null @@ -1,49 +0,0 @@ -import * as fs from 'fs'; -import handlebars from './src/handlebars-helpers'; - -// Load the template -const similarRulesTemplate = fs.readFileSync('./src/templates/similar-rules.hbs', 'utf8').trim(); - -// Compile the template -const template = handlebars.compile(similarRulesTemplate); - -// Test data -const testData = { - amount: 100, - type: 'Outcome', - description: 'Test transaction', - importedPayee: 'Test Payee', - payee: 'Test Payee', - date: '2025-03-05', - rules: [ - { - index: 0, - ruleName: 'Test Rule', - categoryName: 'Test Category', - conditions: [ - { - field: 'payee', - op: 'is', - type: 'id', - value: ['test-id-1', 'test-id-2'], - }, - { - field: 'notes', - op: 'contains', - type: 'string', - value: 'test', - }, - ], - }, - ], -}; - -// Execute the template -try { - const result = template(testData); - console.log('Template rendered successfully:'); - console.log(result); - console.log('\nHandlebars helpers are working correctly!'); -} catch (error) { - console.error('Error rendering template:', error); -} From 6e506b1965ce818b1cc6708fb4f54a96c4b8af65 Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 23 Mar 2025 01:16:17 -0400 Subject: [PATCH 16/17] Add FreeWebSearchService and integrate with ToolService. Update types and add tests for search functionality. --- src/types.ts | 19 +++ src/utils/free-web-search-service.ts | 127 ++++++++++++++++++++ src/utils/tool-service.ts | 26 ++++ tests/utils/free-web-search-service.test.ts | 82 +++++++++++++ 4 files changed, 254 insertions(+) create mode 100644 src/utils/free-web-search-service.ts create mode 100644 tests/utils/free-web-search-service.test.ts diff --git a/src/types.ts b/src/types.ts index fb3dd0c..bff0a40 100644 --- a/src/types.ts +++ b/src/types.ts @@ -113,3 +113,22 @@ export interface PromptGeneratorI { rules: RuleEntity[], ): string; } + +export interface SearchResult { + title: string; + snippet: string; + link: string; +} + +export interface LocalSearchServiceI { + initialize(): Promise; + search(query: string): Promise; + addMerchant(merchantData: { + name: string; + description?: string; + category?: string; + website?: string; + tags?: string[]; + }): Promise; + formatSearchResults(results: SearchResult[]): string; +} diff --git a/src/utils/free-web-search-service.ts b/src/utils/free-web-search-service.ts new file mode 100644 index 0000000..de655a1 --- /dev/null +++ b/src/utils/free-web-search-service.ts @@ -0,0 +1,127 @@ +import * as https from 'https'; +import { SearchResult } from '../types'; + +/** + * A free web search service that can be used as an alternative to ValueSerp. + * This service uses the SerpApi.com free API to search the web. + */ +export default class FreeWebSearchService { + /** + * Search the web using a free API + */ + public async search(query: string): Promise { + return this.searchUsingDDG(query); + } + + /** + * Search using DuckDuckGo API + */ + private async searchUsingDDG(query: string): Promise { + const url = `https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}`; + const html = await this.fetchUrl(url); + + // console.debug('[SearchService] DDG raw response:', html); + const results: SearchResult[] = []; + + const rowRegex = /\s*]*>(\d+)\. <\/td>\s*\s*]*href="([^"]+)"[^>]*>([^<]+)<\/a>\s*<\/td>\s*<\/tr>\s*\s*[^<]*<\/td>\s*]*class=['"]result-snippet['"][^>]*>([\s\S]*?)<\/td>\s*<\/tr>/g; + let match; + let count = 0; + while (count < 5) { + match = rowRegex.exec(html); + if (match === null) break; + + const [, , link, title, snippet] = match; + const cleanSnippet = snippet + .replace(/<\/?[^>]+(>|$)/g, '') + .replace(/ /g, ' ') + .replace(/\s+/g, ' ') + .trim(); + + const result = { + title: this.decodeHtmlEntities(title), + snippet: cleanSnippet, + link, + }; + + // console.debug('[SearchService] Parsed DDG result:', result); + results.push(result); + + if (results.length >= 5) break; + + count += 1; + } + + // console.debug('[SearchService] Final DDG results:', results); + return results; + } + + /** + * Fetch a URL and return the response as text + */ + private async fetchUrl(url: string, retries = 3): Promise { + // console.debug('[SearchService] Fetching URL:', url); + return new Promise((resolve, reject) => { + const attempt = () => { + https.get(url, { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + Accept: 'text/html,application/xhtml+xml,application/xml', + 'Accept-Language': 'en-US,en;q=0.9', + }, + }, (res) => { + // console.debug(`[SearchService] HTTP ${res.statusCode} for ${url}`); + if (res.statusCode === 202 && retries > 0) { + setTimeout(attempt, 1000); + return; + } + if (res.statusCode !== 200) { + reject(new Error(`Request failed with status code ${res.statusCode}`)); + return; + } + + let data = ''; + res.on('data', (chunk) => { + data += chunk; + }); + res.on('end', () => { + resolve(data); + }); + }).on('error', (err) => { + reject(err); + }); + }; + attempt(); + }); + } + + /** + * Format search results in a similar way to the ValueSerp service + */ + public formatSearchResults(results: SearchResult[]): string { + if (!results || results.length === 0) { + return 'No relevant business information found.'; + } + + // Format results + const formattedResults = results + .map((result, index) => `[Source ${index + 1}] ${result.title}\n` + + `${result.snippet.substring(0, 150)}...\n` + + `URL: ${result.link}`) + .join('\n\n'); + + return `SEARCH RESULTS:\n${formattedResults}`; + } + + /** + * Decode HTML entities in a string + */ + private decodeHtmlEntities(html: string): string { + return html + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/ /g, ' '); + } +} diff --git a/src/utils/tool-service.ts b/src/utils/tool-service.ts index 4d53a49..161ce69 100644 --- a/src/utils/tool-service.ts +++ b/src/utils/tool-service.ts @@ -3,6 +3,7 @@ import { z } from 'zod'; import { tool, Tool } from 'ai'; import { ToolServiceI } from '../types'; import { getEnabledTools } from '../config'; +import FreeWebSearchService from './free-web-search-service'; interface SearchResult { title: string; @@ -17,8 +18,11 @@ interface OrganicResults { export default class ToolService implements ToolServiceI { private readonly valueSerpApiKey: string; + private readonly freeWebSearchService: FreeWebSearchService; + constructor(valueSerpApiKey: string) { this.valueSerpApiKey = valueSerpApiKey; + this.freeWebSearchService = new FreeWebSearchService(); } public getTools() { @@ -42,6 +46,28 @@ export default class ToolService implements ToolServiceI { }); } + if (getEnabledTools().includes('freeWebSearch')) { + tools.freeWebSearch = tool({ + description: 'Search the web for business information when existing categories are insufficient. Uses free public search APIs. Use when payee is unfamiliar or category context is unclear', + parameters: z.object({ + query: z.string().describe( + 'Combination of payee name and business type. ' + + 'Example: "StudntLN" or "Student Loan"', + ), + }), + execute: async ({ query }: { query: string }): Promise => { + console.log(`Performing free web search for ${query}`); + try { + const results = await this.freeWebSearchService.search(query); + return this.freeWebSearchService.formatSearchResults(results); + } catch (error) { + console.error('Error during free web search:', error); + return 'Web search failed. Please try again later.'; + } + }, + }); + } + return tools; } diff --git a/tests/utils/free-web-search-service.test.ts b/tests/utils/free-web-search-service.test.ts new file mode 100644 index 0000000..17abeaf --- /dev/null +++ b/tests/utils/free-web-search-service.test.ts @@ -0,0 +1,82 @@ +import FreeWebSearchService from '../../src/utils/free-web-search-service'; + +describe('FreeWebSearchService', () => { + let freeWebSearchService: FreeWebSearchService; + + beforeEach(() => { + freeWebSearchService = new FreeWebSearchService(); + // Mock the fetchUrl method to avoid making actual HTTP requests + jest.spyOn(freeWebSearchService as unknown as { fetchUrl: (url: string) => Promise }, 'fetchUrl') + .mockImplementation(() => Promise.resolve(` + + 1.  + Example Result 1 + + + + Sample snippet 1 + + + 2.  + Example Result 2 + + + + Sample snippet 2 + + `)); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + describe('search', () => { + it('should return DDG results with correct structure', async () => { + const results = await freeWebSearchService.search('test query'); + + expect(results).toHaveLength(2); + expect(results[0]).toEqual({ + title: 'Example Result 1', + snippet: 'Sample snippet 1', + link: 'https://example.com/1', + }); + expect(results[1]).toEqual({ + title: 'Example Result 2', + snippet: 'Sample snippet 2', + link: 'https://example.com/2', + }); + }); + }); + + describe('formatSearchResults', () => { + it('should format search results correctly', () => { + const results = [ + { + title: 'Example Result 1', + snippet: 'This is a sample snippet for result 1', + link: 'https://example.com/1', + }, + { + title: 'Example Result 2', + snippet: 'This is a sample snippet for result 2', + link: 'https://example.com/2', + }, + ]; + + const formatted = freeWebSearchService.formatSearchResults(results); + + expect(formatted).toContain('SEARCH RESULTS:'); + expect(formatted).toContain('[Source 1] Example Result 1'); + expect(formatted).toContain('[Source 2] Example Result 2'); + expect(formatted).toContain('URL: https://example.com/1'); + expect(formatted).toContain('URL: https://example.com/2'); + }); + + it('should handle empty results', () => { + const formatted = freeWebSearchService.formatSearchResults([]); + + expect(formatted).toBe('No relevant business information found.'); + }); + }); +}); From dd6029d1875d641c45dd8cdd252893ccf3ddb54d Mon Sep 17 00:00:00 2001 From: Kevin Gatera Date: Sun, 23 Mar 2025 01:16:47 -0400 Subject: [PATCH 17/17] Update .env.example and README for feature flags; remove DRY_RUN and clarify features. Refactor ActualAiService to use feature flag for syncAccountsBeforeClassify. --- .env.example | 1 - README.md | 15 ++++++++++----- src/actual-ai.ts | 7 ++----- src/actual-api-service.ts | 2 ++ src/config.ts | 2 +- src/container.ts | 2 -- tests/actual-ai.test.ts | 21 ++++++--------------- 7 files changed, 21 insertions(+), 29 deletions(-) diff --git a/.env.example b/.env.example index 62f2a33..7cdca71 100644 --- a/.env.example +++ b/.env.example @@ -5,7 +5,6 @@ CLASSIFICATION_SCHEDULE_CRON="0 */4 * * *" # Feature flags - can be specified as an array FEATURES='["freeWebSearch", "suggestNewCategories", "rerunMissedTransactions", "classifyOnStartup", "syncAccountsBeforeClassify"]' -DRY_RUN=true # Tools and API keys # ENABLED_TOOLS=webSearch diff --git a/README.md b/README.md index 509bbf4..c322abc 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,10 @@ When enabled, the LLM can suggest entirely new categories for transactions it ca Using the ValueSerp API, the system can search the web for information about unfamiliar merchants to help the LLM make better categorization decisions. +#### 🔎 Free web search alternative + +A self-hosted alternative to ValueSerp that uses free public search API (DuckDuckGo) to search for merchant information without requiring an API key. + #### 🔄 Re-run missed transactions Re-process transactions previously marked as unclassified. @@ -65,10 +69,8 @@ services: ACTUAL_PASSWORD: your_actual_password ACTUAL_BUDGET_ID: your_actual_budget_id # This is the ID from Settings → Show advanced settings → Sync ID CLASSIFICATION_SCHEDULE_CRON: 0 */4 * * * # How often to run classification. - CLASSIFY_ON_STARTUP: true # Whether to classify transactions on startup (don't wait for cron schedule) - SYNC_ACCOUNTS_BEFORE_CLASSIFY: false # Whether to sync accounts before classification LLM_PROVIDER: openai # Can be "openai", "anthropic", "google-generative-ai", "ollama" or "groq" -# FEATURES: '["webSearch", "suggestNewCategories"]' + FEATURES: '["classifyOnStartup", "syncAccountsBeforeClassify", "freeWebSearch", "suggestNewCategories"]' # VALUESERP_API_KEY: your_valueserp_api_key # API key for ValueSerp, required if webSearch tool is enabled # OPENAI_API_KEY: # optional. required if you want to use the OpenAI API # OPENAI_MODEL: # optional. required if you want to use a specific model, default is "gpt-4o-mini" @@ -116,12 +118,15 @@ You can configure features in using the FEATURES array (recommended): The `FEATURES` environment variable accepts a JSON array of feature names to enable: ``` -FEATURES='["webSearch", "suggestNewCategories"]' +FEATURES='["freeWebSearch", "suggestNewCategories", "classifyOnStartup", "syncAccountsBeforeClassify"]' ``` Available features: - `webSearch` - Enable web search for merchant information +- `freeWebSearch` - Enable free web search for merchant information (self-hosted alternative to ValueSerp) - `suggestNewCategories` - Allow suggesting new categories for transactions +- `classifyOnStartup` - Run classification when the application starts +- `syncAccountsBeforeClassify` - Sync accounts before running classification - `dryRun` - Run in dry run mode (enabled by default) - `dryRunNewCategories` - Only log suggested categories without creating them (enabled by default) - `rerunMissedTransactions` - Re-process transactions previously marked as unclassified @@ -197,7 +202,7 @@ The `dryRun` feature is enabled by default. In this mode: - System will show what would happen with real execution To perform actual changes: -1. Remove `dryRun` from your FEATURES array or set `DRY_RUN=false` +1. Remove `dryRun` from your FEATURES array 2. Ensure `suggestNewCategories` is enabled if you want new category creation 3. Run the classification process diff --git a/src/actual-ai.ts b/src/actual-ai.ts index fe751ae..8e55b22 100644 --- a/src/actual-ai.ts +++ b/src/actual-ai.ts @@ -1,22 +1,19 @@ import { ActualAiServiceI, ActualApiServiceI, TransactionServiceI } from './types'; import suppressConsoleLogsAsync from './utils'; import { formatError } from './utils/error-utils'; +import { isFeatureEnabled } from './config'; class ActualAiService implements ActualAiServiceI { private readonly transactionService: TransactionServiceI; private readonly actualApiService: ActualApiServiceI; - private readonly syncAccountsBeforeClassify: boolean; - constructor( transactionService: TransactionServiceI, actualApiService: ActualApiServiceI, - syncAccountsBeforeClassify: boolean, ) { this.transactionService = transactionService; this.actualApiService = actualApiService; - this.syncAccountsBeforeClassify = syncAccountsBeforeClassify; } public async classify() { @@ -25,7 +22,7 @@ class ActualAiService implements ActualAiServiceI { await this.actualApiService.initializeApi(); try { - if (this.syncAccountsBeforeClassify) { + if (isFeatureEnabled('syncAccountsBeforeClassify')) { await this.syncAccounts(); } } catch (error) { diff --git a/src/actual-api-service.ts b/src/actual-api-service.ts index 78f440c..99514a4 100644 --- a/src/actual-api-service.ts +++ b/src/actual-api-service.ts @@ -109,10 +109,12 @@ class ActualApiService implements ActualApiServiceI { } public async getRules(): Promise { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return return this.actualApiClient.getRules(); } public async getPayeeRules(payeeId: string): Promise { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return return this.actualApiClient.getPayeeRules(payeeId); } diff --git a/src/config.ts b/src/config.ts index 0a715bb..75562be 100644 --- a/src/config.ts +++ b/src/config.ts @@ -64,7 +64,7 @@ function registerStandardFeatures() { }; features.dryRun = { - enabled: enabledFeatures.includes('dryRun') || process.env.DRY_RUN === 'true', + enabled: enabledFeatures.includes('dryRun'), defaultValue: true, description: 'Run in dry mode without actually making changes', }; diff --git a/src/container.ts b/src/container.ts index 6c68af8..6b751c5 100644 --- a/src/container.ts +++ b/src/container.ts @@ -27,7 +27,6 @@ import { password, promptTemplate, serverURL, - syncAccountsBeforeClassify, valueSerpApiKey, getEnabledTools, } from './config'; @@ -89,7 +88,6 @@ const transactionService = new TransactionService( const actualAi = new ActualAiService( transactionService, actualApiService, - syncAccountsBeforeClassify, ); export default actualAi; diff --git a/tests/actual-ai.test.ts b/tests/actual-ai.test.ts index 8be7c58..fd36e66 100644 --- a/tests/actual-ai.test.ts +++ b/tests/actual-ai.test.ts @@ -23,7 +23,6 @@ describe('ActualAiService', () => { let inMemoryApiService: InMemoryActualApiService; let mockedLlmService: MockedLlmService; let mockedPromptGenerator: MockedPromptGenerator; - let syncAccountsBeforeClassify = false; const GUESSED_TAG = '#actual-ai'; const NOT_GUESSED_TAG = '#actual-ai-miss'; @@ -78,7 +77,6 @@ describe('ActualAiService', () => { sut = new ActualAiService( transactionService, inMemoryApiService, - syncAccountsBeforeClassify, ); await sut.classify(); @@ -102,7 +100,6 @@ describe('ActualAiService', () => { sut = new ActualAiService( transactionService, inMemoryApiService, - syncAccountsBeforeClassify, ); await sut.classify(); @@ -128,7 +125,6 @@ describe('ActualAiService', () => { sut = new ActualAiService( transactionService, inMemoryApiService, - syncAccountsBeforeClassify, ); await sut.classify(); @@ -152,7 +148,6 @@ describe('ActualAiService', () => { sut = new ActualAiService( transactionService, inMemoryApiService, - syncAccountsBeforeClassify, ); await sut.classify(); @@ -176,7 +171,6 @@ describe('ActualAiService', () => { sut = new ActualAiService( transactionService, inMemoryApiService, - syncAccountsBeforeClassify, ); await sut.classify(); @@ -199,7 +193,6 @@ describe('ActualAiService', () => { sut = new ActualAiService( transactionService, inMemoryApiService, - syncAccountsBeforeClassify, ); await sut.classify(); @@ -230,7 +223,6 @@ describe('ActualAiService', () => { sut = new ActualAiService( transactionService, inMemoryApiService, - syncAccountsBeforeClassify, ); await sut.classify(); @@ -257,7 +249,6 @@ describe('ActualAiService', () => { sut = new ActualAiService( transactionService, inMemoryApiService, - syncAccountsBeforeClassify, ); await sut.classify(); @@ -291,7 +282,6 @@ describe('ActualAiService', () => { sut = new ActualAiService( transactionService, inMemoryApiService, - syncAccountsBeforeClassify, ); await sut.classify(); @@ -321,7 +311,6 @@ describe('ActualAiService', () => { sut = new ActualAiService( transactionService, inMemoryApiService, - syncAccountsBeforeClassify, ); await sut.classify(); @@ -374,7 +363,6 @@ describe('ActualAiService', () => { sut = new ActualAiService( transactionService, inMemoryApiService, - syncAccountsBeforeClassify, ); await sut.classify(); @@ -388,13 +376,17 @@ describe('ActualAiService', () => { it('It should run bank sync when flag is set', async () => { // Arrange - syncAccountsBeforeClassify = true; + mockIsFeatureEnabled.mockImplementation((feature: string) => { + if (feature === 'syncAccountsBeforeClassify') return true; + if (feature === 'rerunMissedTransactions') return false; + if (feature === 'dryRun' || feature === 'dryRunNewCategories') return false; + return originalIsFeatureEnabled(feature); + }); // Act sut = new ActualAiService( transactionService, inMemoryApiService, - syncAccountsBeforeClassify, ); await sut.classify(); @@ -425,7 +417,6 @@ describe('ActualAiService', () => { sut = new ActualAiService( transactionService, inMemoryApiService, - syncAccountsBeforeClassify, ); await sut.classify();