@@ -3,10 +3,13 @@ import { getEncoding } from 'js-tiktoken';
3
3
import fs from 'node:fs/promises' ;
4
4
import path from 'node:path' ;
5
5
import pc from 'picocolors' ;
6
+ import readline from 'node:readline/promises' ;
7
+ import { stdin as input , stdout as output } from 'node:process' ;
6
8
7
9
// Environment variables for chunk configuration, with defaults
8
10
const CHUNK_SIZE = Number ( process . env . CHUNK_SIZE || '100000' ) ;
9
11
const CHUNK_OVERLAP = Number ( process . env . CHUNK_OVERLAP || '50000' ) ;
12
+ const costPerToken = 3e-6 ; // 3$ per million tokens
10
13
11
14
export async function generateWithLLM (
12
15
repoContent : string ,
@@ -47,7 +50,7 @@ function formatTokenCount(count: number): string {
47
50
return pc . red ( formatted ) ;
48
51
}
49
52
50
- function chunkText ( text : string , chunkSize : number , overlap : number ) : string [ ] {
53
+ async function chunkText ( text : string , chunkSize : number , overlap : number ) : Promise < string [ ] > {
51
54
console . log ( pc . cyan ( '\n┌─────────────────────────────────────────┐' ) ) ;
52
55
console . log ( pc . cyan ( '│ CONTENT CHUNKING │' ) ) ;
53
56
console . log ( pc . cyan ( '└─────────────────────────────────────────┘\n' ) ) ;
@@ -63,7 +66,26 @@ function chunkText(text: string, chunkSize: number, overlap: number): string[] {
63
66
console . log ( `● Chunk size: ${ formatTokenCount ( chunkSize ) } tokens` ) ;
64
67
console . log ( `● Chunk overlap: ${ formatTokenCount ( overlap ) } tokens\n` ) ;
65
68
66
- console . log ( pc . cyan ( 'Chunking document...' ) ) ;
69
+ // Calculate and display the estimated cost
70
+ const estimatedCost = ( totalTokens * costPerToken ) . toFixed ( 4 ) ;
71
+ console . log ( pc . yellow ( `● Estimated cost: $${ estimatedCost } (${ formatTokenCount ( totalTokens ) } tokens × $${ costPerToken } per token)` ) ) ;
72
+
73
+ // Create a user dialog to confirm proceeding
74
+ const rl = readline . createInterface ( { input, output } ) ;
75
+
76
+ try {
77
+ const answer = await rl . question ( pc . yellow ( '\nProceed with processing? (y/n): ' ) ) ;
78
+ const proceed = answer . toLowerCase ( ) === 'y' || answer . toLowerCase ( ) === 'yes' ;
79
+
80
+ if ( ! proceed ) {
81
+ console . log ( pc . red ( '\nOperation cancelled by user.' ) ) ;
82
+ process . exit ( 0 ) ;
83
+ }
84
+ } finally {
85
+ rl . close ( ) ;
86
+ }
87
+
88
+ console . log ( pc . cyan ( '\nChunking document...' ) ) ;
67
89
68
90
let i = 0 ;
69
91
while ( i < tokens . length ) {
@@ -102,7 +124,7 @@ async function generateWithClaude(repoContent: string, guidelines: string, outpu
102
124
} ) ;
103
125
104
126
// Create chunks of the repository content
105
- const chunks = chunkText ( repoContent , CHUNK_SIZE , CHUNK_OVERLAP ) ;
127
+ const chunks = await chunkText ( repoContent , CHUNK_SIZE , CHUNK_OVERLAP ) ;
106
128
107
129
// Display token counts for each chunk in a table format
108
130
console . log ( pc . cyan ( '\n┌─────────────────────────────────────────┐' ) ) ;
0 commit comments