From 5b6fee6e5e0dc300c6b68877e80c6c8f90cfda05 Mon Sep 17 00:00:00 2001 From: Anthony Campolo <12433465+ajcwebdev@users.noreply.github.com> Date: Sat, 2 Nov 2024 01:19:23 -0500 Subject: [PATCH 1/5] make types stricter --- package.json | 19 ++++----- src/autoshow.ts | 72 +++++++++++++++++++------------ src/commands/processFile.ts | 2 +- src/commands/processRSS.ts | 35 ++++++++++----- src/commands/processVideo.ts | 2 +- src/llms/chatgpt.ts | 30 ++++++------- src/llms/claude.ts | 4 +- src/llms/cohere.ts | 4 +- src/llms/fireworks.ts | 4 +- src/llms/gemini.ts | 4 +- src/llms/groq.ts | 4 +- src/llms/mistral.ts | 27 ++++++------ src/llms/ollama.ts | 4 +- src/llms/prompt.ts | 26 +++++++----- src/llms/together.ts | 4 +- src/transcription/assembly.ts | 27 ++++++------ src/transcription/deepgram.ts | 22 ++++++---- src/transcription/whisper.ts | 46 +++++++++++--------- src/utils/runTranscription.ts | 5 +-- tsconfig.json | 80 +++++++++++++++++++++++++++++------ 20 files changed, 261 insertions(+), 160 deletions(-) diff --git a/package.json b/package.json index eddfb4f..fa1bd94 100644 --- a/package.json +++ b/package.json @@ -25,16 +25,15 @@ "setup-python": "bash ./scripts/setup-python.sh", "autoshow": "npm run tsx:base -- src/autoshow.ts", "as": "npm run tsx:base -- src/autoshow.ts", - "v": "npm run tsx:base -- src/autoshow.ts --video", - "u": "npm run tsx:base -- src/autoshow.ts --urls", - "urls": "npm run tsx:base -- src/autoshow.ts --urls content/urls.md", - "p": "npm run tsx:base -- src/autoshow.ts --playlist", - "f": "npm run tsx:base -- src/autoshow.ts --file", - "r": "npm run tsx:base -- src/autoshow.ts --rss", - "rss-info": "npm run tsx:base -- src/autoshow.ts --info --rss", - "info": "npm run tsx:base -- src/autoshow.ts --info", - "last2": "npm run tsx:base -- src/autoshow.ts --last 2 --rss", - "last3": "npm run tsx:base -- src/autoshow.ts --last 3 --rss", + "v": "npm run as -- --video", + "u": "npm run as -- --urls", + "p": "npm run as -- --playlist", + "f": "npm run as -- --file", + "r": "npm run as -- --rss", + "rss-info": "npm run as -- --info --rss", + "info": "npm run as -- --info", + "last2": "npm run as -- --last 2 --rss", + "last3": "npm run as -- --last 3 --rss", "docker": "docker compose run --remove-orphans --rm autoshow --whisperDocker", "docker-up": "docker compose up --build -d --remove-orphans --no-start", "ds": "docker compose images && docker compose ls", diff --git a/src/autoshow.ts b/src/autoshow.ts index ccc9f52..4dd30c2 100644 --- a/src/autoshow.ts +++ b/src/autoshow.ts @@ -27,6 +27,24 @@ import type { ProcessingOptions, HandlerFunction, LLMServices, TranscriptService // Initialize the command-line interface using Commander.js const program = new Command() +// Define valid action types +type ValidAction = 'video' | 'playlist' | 'channel' | 'urls' | 'file' | 'rss' + +// Define the process handlers with strict typing +const PROCESS_HANDLERS: Record = { + video: processVideo, + playlist: processPlaylist, + channel: processChannel, + urls: processURLs, + file: processFile, + rss: processRSS, +} + +// Type guard to check if a string is a valid action +function isValidAction(action: string | undefined): action is ValidAction { + return Boolean(action && action in PROCESS_HANDLERS) +} + /** * Defines the command-line interface options and descriptions. * Sets up all available commands and their respective flags @@ -98,16 +116,6 @@ program.action(async (options: ProcessingOptions) => { l(opts(JSON.stringify(options, null, 2))) l(``) - // Define mapping of action types to their handler functions - const PROCESS_HANDLERS: Record = { - video: processVideo, - playlist: processPlaylist, - channel: processChannel, - urls: processURLs, - file: processFile, - rss: processRSS, - } - // Extract interactive mode flag const { interactive } = options @@ -136,24 +144,34 @@ program.action(async (options: ProcessingOptions) => { options.whisper = 'large-v3-turbo' } - // Execute the appropriate handler if an action was specified - if (action) { - try { - // Process the content using the selected handler - await PROCESS_HANDLERS[action]( - options, - options[action as keyof ProcessingOptions] as string, - llmServices, - transcriptServices - ) - l(final(`\n================================================================================================`)) - l(final(` ${action} Processing Completed Successfully.`)) - l(final(`================================================================================================\n`)) - exit(0) - } catch (error) { - err(`Error processing ${action}:`, (error as Error).message) - exit(1) + // Validate action + if (!isValidAction(action)) { + err(`Invalid or missing action`) + exit(1) + } + + try { + // Get input value with proper typing + const input = options[action] + + // Ensure we have a valid input value + if (!input || typeof input !== 'string') { + throw new Error(`No valid input provided for ${action} processing`) } + + // Get handler with proper typing + const handler = PROCESS_HANDLERS[action] + + // Process the content using the selected handler + await handler(options, input, llmServices, transcriptServices) + + l(final(`\n================================================================================================`)) + l(final(` ${action} Processing Completed Successfully.`)) + l(final(`================================================================================================\n`)) + exit(0) + } catch (error) { + err(`Error processing ${action}:`, (error as Error).message) + exit(1) } }) diff --git a/src/commands/processFile.ts b/src/commands/processFile.ts index ffca77a..7b9c1ff 100644 --- a/src/commands/processFile.ts +++ b/src/commands/processFile.ts @@ -49,7 +49,7 @@ export async function processFile( await downloadAudio(options, filePath, filename) // Convert the audio to text using the specified transcription service - await runTranscription(options, finalPath, frontMatter, transcriptServices) + await runTranscription(options, finalPath, transcriptServices) // Process the transcript with a language model if one was specified await runLLM(options, finalPath, frontMatter, llmServices) diff --git a/src/commands/processRSS.ts b/src/commands/processRSS.ts index 52b7b8d..aef4534 100644 --- a/src/commands/processRSS.ts +++ b/src/commands/processRSS.ts @@ -106,6 +106,7 @@ function extractFeedItems(feed: any): RSSItem[] { const { title: channelTitle, link: channelLink, image: channelImageObject, item: feedItems } = feed.rss.channel const channelImage = channelImageObject?.url || '' const feedItemsArray = Array.isArray(feedItems) ? feedItems : [feedItems] + const defaultDate = new Date().toISOString().substring(0, 10) const items: RSSItem[] = feedItemsArray .filter((item) => { @@ -113,15 +114,29 @@ function extractFeedItems(feed: any): RSSItem[] { const audioVideoTypes = ['audio/', 'video/'] return audioVideoTypes.some((type) => item.enclosure.type.startsWith(type)) }) - .map((item) => ({ - showLink: item.enclosure.url, - channel: channelTitle, - channelURL: channelLink, - title: item.title, - description: '', - publishDate: new Date(item.pubDate).toISOString().split('T')[0], - coverImage: item['itunes:image']?.href || channelImage || '', - })) + .map((item) => { + // Ensure publishDate is always a valid string + let publishDate: string + try { + // Try to parse the date, fall back to current date if invalid + const date = item.pubDate ? new Date(item.pubDate) : new Date() + publishDate = date.toISOString().substring(0, 10) + } catch { + // If date parsing fails, use current date + publishDate = defaultDate + } + + return { + showLink: item.enclosure.url || '', + channel: channelTitle || '', + channelURL: channelLink || '', + title: item.title || '', + description: item.description || '', + publishDate, + coverImage: item['itunes:image']?.href || channelImage || '', + audioURL: item.enclosure?.url || '' + } + }) if (items.length === 0) { err('Error: No audio/video items found in the RSS feed.') @@ -205,7 +220,7 @@ async function processItem( try { const { frontMatter, finalPath, filename } = await generateMarkdown(options, item) await downloadAudio(options, item.showLink, filename) - await runTranscription(options, finalPath, frontMatter, transcriptServices) + await runTranscription(options, finalPath, transcriptServices) await runLLM(options, finalPath, frontMatter, llmServices) if (!options.noCleanUp) { diff --git a/src/commands/processVideo.ts b/src/commands/processVideo.ts index d5874d0..443f133 100644 --- a/src/commands/processVideo.ts +++ b/src/commands/processVideo.ts @@ -47,7 +47,7 @@ export async function processVideo( await downloadAudio(options, url, filename) // Convert the audio to text using the specified transcription service - await runTranscription(options, finalPath, frontMatter, transcriptServices) + await runTranscription(options, finalPath, transcriptServices) // Process the transcript with a language model if one was specified await runLLM(options, finalPath, frontMatter, llmServices) diff --git a/src/llms/chatgpt.ts b/src/llms/chatgpt.ts index 9c5948f..d87fecf 100644 --- a/src/llms/chatgpt.ts +++ b/src/llms/chatgpt.ts @@ -4,7 +4,6 @@ import { writeFile } from 'node:fs/promises' import { env } from 'node:process' import { OpenAI } from 'openai' import { l, wait, err, GPT_MODELS } from '../globals.js' - import type { LLMFunction, ChatGPTModelType } from '../types.js' /** @@ -21,12 +20,12 @@ export const callChatGPT: LLMFunction = async ( model: string = 'GPT_4o_MINI' ): Promise => { // Check for API key - if (!env.OPENAI_API_KEY) { + if (!env['OPENAI_API_KEY']) { throw new Error('OPENAI_API_KEY environment variable is not set. Please set it to your OpenAI API key.') } // Initialize the OpenAI client with the API key from environment variables - const openai = new OpenAI({ apiKey: env.OPENAI_API_KEY }) + const openai = new OpenAI({ apiKey: env['OPENAI_API_KEY'] }) try { // Select the actual model to use, defaulting to GPT_4o_MINI if not specified @@ -38,20 +37,21 @@ export const callChatGPT: LLMFunction = async ( max_tokens: 4000, // Maximum number of tokens in the response messages: [{ role: 'user', content: promptAndTranscript }], // The input message (transcript content) }) - - // Destructure the response to get relevant information - const { - choices: [{ message: { content }, finish_reason }], // The generated content and finish reason - usage, // Token usage information - model: usedModel // The actual model used - } = response + + // Check if we have a valid response + const firstChoice = response.choices[0] + if (!firstChoice || !firstChoice.message?.content) { + throw new Error('No valid response received from the API') + } + + // Get the content and other details safely + const content = firstChoice.message.content + const finish_reason = firstChoice.finish_reason ?? 'unknown' + const usedModel = response.model + const usage = response.usage // Write the generated content to the output file - if (content !== null) { - await writeFile(tempPath, content) - } else { - throw new Error('No content generated from the API') - } + await writeFile(tempPath, content) l(wait(` - Finish Reason: ${finish_reason}\n - ChatGPT Model: ${usedModel}`)) diff --git a/src/llms/claude.ts b/src/llms/claude.ts index eb3997c..89298a0 100644 --- a/src/llms/claude.ts +++ b/src/llms/claude.ts @@ -21,12 +21,12 @@ export const callClaude: LLMFunction = async ( model: string = 'CLAUDE_3_HAIKU' ): Promise => { // Check if the ANTHROPIC_API_KEY environment variable is set - if (!env.ANTHROPIC_API_KEY) { + if (!env['ANTHROPIC_API_KEY']) { throw new Error('ANTHROPIC_API_KEY environment variable is not set. Please set it to your Anthropic API key.') } // Initialize the Anthropic client with the API key from environment variables - const anthropic = new Anthropic({ apiKey: env.ANTHROPIC_API_KEY }) + const anthropic = new Anthropic({ apiKey: env['ANTHROPIC_API_KEY'] }) try { // Select the actual model to use, defaulting to CLAUDE_3_HAIKU if not specified diff --git a/src/llms/cohere.ts b/src/llms/cohere.ts index 5ae9b50..2d07cce 100644 --- a/src/llms/cohere.ts +++ b/src/llms/cohere.ts @@ -21,12 +21,12 @@ export const callCohere: LLMFunction = async ( model: string = 'COMMAND_R' ): Promise => { // Check if the COHERE_API_KEY environment variable is set - if (!env.COHERE_API_KEY) { + if (!env['COHERE_API_KEY']) { throw new Error('COHERE_API_KEY environment variable is not set. Please set it to your Cohere API key.') } // Initialize the Cohere client with the API key from environment variables - const cohere = new CohereClient({ token: env.COHERE_API_KEY }) + const cohere = new CohereClient({ token: env['COHERE_API_KEY'] }) try { // Select the actual model to use, defaulting to COMMAND_R if not specified diff --git a/src/llms/fireworks.ts b/src/llms/fireworks.ts index 1a2c45e..c12d22d 100644 --- a/src/llms/fireworks.ts +++ b/src/llms/fireworks.ts @@ -19,7 +19,7 @@ export const callFireworks: LLMFunction = async ( model: string = 'LLAMA_3_2_3B' ): Promise => { // Check if the FIREWORKS_API_KEY environment variable is set - if (!env.FIREWORKS_API_KEY) { + if (!env['FIREWORKS_API_KEY']) { throw new Error('FIREWORKS_API_KEY environment variable is not set. Please set it to your Fireworks API key.') } @@ -41,7 +41,7 @@ export const callFireworks: LLMFunction = async ( const response = await fetch('https://api.fireworks.ai/inference/v1/chat/completions', { method: 'POST', headers: { - Authorization: `Bearer ${env.FIREWORKS_API_KEY}`, + Authorization: `Bearer ${env['FIREWORKS_API_KEY']}`, 'Content-Type': 'application/json', }, body: JSON.stringify(requestBody), diff --git a/src/llms/gemini.ts b/src/llms/gemini.ts index 5afde09..9c9d995 100644 --- a/src/llms/gemini.ts +++ b/src/llms/gemini.ts @@ -27,12 +27,12 @@ export const callGemini: LLMFunction = async ( model: string = 'GEMINI_1_5_FLASH' ): Promise => { // Check if the GEMINI_API_KEY environment variable is set - if (!env.GEMINI_API_KEY) { + if (!env['GEMINI_API_KEY']) { throw new Error('GEMINI_API_KEY environment variable is not set. Please set it to your Gemini API key.') } // Initialize the Google Generative AI client - const genAI = new GoogleGenerativeAI(env.GEMINI_API_KEY) + const genAI = new GoogleGenerativeAI(env['GEMINI_API_KEY']) // Select the actual model to use, defaulting to GEMINI_1_5_FLASH if not specified const actualModel = GEMINI_MODELS[model as GeminiModelType] || GEMINI_MODELS.GEMINI_1_5_FLASH diff --git a/src/llms/groq.ts b/src/llms/groq.ts index 3cf698e..75c8b28 100644 --- a/src/llms/groq.ts +++ b/src/llms/groq.ts @@ -16,7 +16,7 @@ const GROQ_API_URL = 'https://api.groq.com/openai/v1/chat/completions' */ export const callGroq = async (promptAndTranscript: string, tempPath: string, model: string = 'MIXTRAL_8X7B_32768'): Promise => { // Ensure that the API key is set - if (!env.GROQ_API_KEY) { + if (!env['GROQ_API_KEY']) { throw new Error('GROQ_API_KEY environment variable is not set. Please set it to your Groq API key.') } @@ -39,7 +39,7 @@ export const callGroq = async (promptAndTranscript: string, tempPath: string, mo const response = await fetch(GROQ_API_URL, { method: 'POST', headers: { - Authorization: `Bearer ${env.GROQ_API_KEY}`, + Authorization: `Bearer ${env['GROQ_API_KEY']}`, 'Content-Type': 'application/json', }, body: JSON.stringify(requestBody), diff --git a/src/llms/mistral.ts b/src/llms/mistral.ts index 44a9886..a777cfc 100644 --- a/src/llms/mistral.ts +++ b/src/llms/mistral.ts @@ -4,7 +4,6 @@ import { writeFile } from 'node:fs/promises' import { env } from 'node:process' import { Mistral } from '@mistralai/mistralai' import { l, wait, err, MISTRAL_MODELS } from '../globals.js' - import type { LLMFunction, MistralModelType } from '../types.js' /** @@ -21,11 +20,12 @@ export const callMistral: LLMFunction = async ( model: string = 'MISTRAL_NEMO' ): Promise => { // Check if the MISTRAL_API_KEY environment variable is set - if (!env.MISTRAL_API_KEY) { + if (!env['MISTRAL_API_KEY']) { throw new Error('MISTRAL_API_KEY environment variable is not set. Please set it to your Mistral API key.') } + // Initialize Mistral client with API key from environment variables - const mistral = new Mistral({ apiKey: env.MISTRAL_API_KEY }) + const mistral = new Mistral({ apiKey: env['MISTRAL_API_KEY'] }) try { // Select the actual model to use, defaulting to MISTRAL_NEMO if the specified model is not found @@ -39,29 +39,30 @@ export const callMistral: LLMFunction = async ( messages: [{ role: 'user', content: promptAndTranscript }], }) - // Safely access the response properties + // Safely access the response properties with proper null checks if (!response.choices || response.choices.length === 0) { throw new Error("No choices returned from Mistral API") } - const content = response.choices[0].message.content - const finishReason = response.choices[0].finishReason - const { promptTokens, completionTokens, totalTokens } = response.usage ?? {} - - // Check if content was generated - if (!content) { - throw new Error("No content generated from Mistral") + const firstChoice = response.choices[0] + if (!firstChoice?.message?.content) { + throw new Error("Invalid response format from Mistral API") } + + const content = firstChoice.message.content + const finishReason = firstChoice.finishReason ?? 'unknown' + const usage = response.usage ?? { promptTokens: 0, completionTokens: 0, totalTokens: 0 } // Write the generated content to the specified output file await writeFile(tempPath, content) + // Log finish reason, used model, and token usage l(wait(`\n Finish Reason: ${finishReason}\n Model Used: ${actualModel}`)) - l(wait(` Token Usage:\n - ${promptTokens} prompt tokens\n - ${completionTokens} completion tokens\n - ${totalTokens} total tokens`)) + l(wait(` Token Usage:\n - ${usage.promptTokens} prompt tokens\n - ${usage.completionTokens} completion tokens\n - ${usage.totalTokens} total tokens`)) } catch (error) { // Log any errors that occur during the process - err(`Error in callMistral: ${error instanceof Error ? (error as Error).message : String(error)}`) + err(`Error in callMistral: ${error instanceof Error ? error.message : String(error)}`) throw error // Re-throw the error for handling by the caller } } \ No newline at end of file diff --git a/src/llms/ollama.ts b/src/llms/ollama.ts index cf92c53..25ae3d9 100644 --- a/src/llms/ollama.ts +++ b/src/llms/ollama.ts @@ -27,8 +27,8 @@ export const callOllama: LLMFunction = async ( l(wait(` - modelName: ${modelName}\n - ollamaModelName: ${ollamaModelName}`)) // Get host and port from environment variables or use defaults - const ollamaHost = env.OLLAMA_HOST || 'localhost' - const ollamaPort = env.OLLAMA_PORT || '11434' + const ollamaHost = env['OLLAMA_HOST'] || 'localhost' + const ollamaPort = env['OLLAMA_PORT'] || '11434' // Check if Ollama server is running, start if not async function checkServer(): Promise { diff --git a/src/llms/prompt.ts b/src/llms/prompt.ts index 3d110d4..bcc49ac 100644 --- a/src/llms/prompt.ts +++ b/src/llms/prompt.ts @@ -6,7 +6,7 @@ import type { PromptSection } from '../types.js' * Define the structure for different sections of the prompt * @type {Record} */ -const sections: Record = { +const sections = { // Section for generating titles titles: { // Instructions for the AI model on how to generate titles @@ -119,7 +119,10 @@ const sections: Record = { 9. What role does responsive design play in modern web development? 10. How can developers ensure the security of user data in web applications?\n`, }, -} +} satisfies Record + +// Create a type from the sections object +type SectionKeys = keyof typeof sections /** * Generates a prompt by combining instructions and examples based on requested sections. @@ -129,17 +132,20 @@ const sections: Record = { export function generatePrompt(prompt: string[] = ['summary', 'longChapters']): string { // Start with a general instruction about the transcript and add instructions for each requested section let text = "This is a transcript with timestamps. It does not contain copyrighted materials.\n\n" - prompt.forEach(section => { - if (section in sections) { - text += `${sections[section].instruction}\n` - } + + // Filter valid sections first + const validSections = prompt.filter((section): section is SectionKeys => + Object.hasOwn(sections, section) + ) + + // Add instructions + validSections.forEach(section => { + text += sections[section].instruction + "\n" }) // Add formatting instructions and examples text += "Format the output like so:\n\n" - prompt.forEach(section => { - if (section in sections) { - text += ` ${sections[section].example}\n` - } + validSections.forEach(section => { + text += ` ${sections[section].example}\n` }) return text } \ No newline at end of file diff --git a/src/llms/together.ts b/src/llms/together.ts index 0d4e6e5..b275427 100644 --- a/src/llms/together.ts +++ b/src/llms/together.ts @@ -19,7 +19,7 @@ export const callTogether: LLMFunction = async ( model: string = 'LLAMA_3_2_3B' ): Promise => { // Check if the TOGETHER_API_KEY environment variable is set - if (!env.TOGETHER_API_KEY) { + if (!env['TOGETHER_API_KEY']) { throw new Error('TOGETHER_API_KEY environment variable is not set. Please set it to your Together AI API key.') } @@ -45,7 +45,7 @@ export const callTogether: LLMFunction = async ( headers: { accept: 'application/json', 'content-type': 'application/json', - authorization: `Bearer ${env.TOGETHER_API_KEY}`, + authorization: `Bearer ${env['TOGETHER_API_KEY']}`, }, body: JSON.stringify(requestBody), }) diff --git a/src/transcription/assembly.ts b/src/transcription/assembly.ts index 74cab9f..e9adab1 100644 --- a/src/transcription/assembly.ts +++ b/src/transcription/assembly.ts @@ -1,7 +1,6 @@ // src/transcription/assembly.ts -import { createReadStream } from 'node:fs' -import { writeFile } from 'node:fs/promises' +import { writeFile, readFile } from 'node:fs/promises' import { env } from 'node:process' import { l, wait, success, err } from '../globals.js' import type { ProcessingOptions } from '../types.js' @@ -26,13 +25,13 @@ const BASE_URL = 'https://api.assemblyai.com/v2' */ export async function callAssembly(options: ProcessingOptions, finalPath: string): Promise { l(wait('\n Using AssemblyAI for transcription...')) - // Check if the ASSEMBLY_API_KEY environment variable is set - if (!env.ASSEMBLY_API_KEY) { + + if (!env['ASSEMBLY_API_KEY']) { throw new Error('ASSEMBLY_API_KEY environment variable is not set. Please set it to your AssemblyAI API key.') } const headers = { - 'Authorization': env.ASSEMBLY_API_KEY, + 'Authorization': env['ASSEMBLY_API_KEY'], 'Content-Type': 'application/json' } @@ -43,16 +42,15 @@ export async function callAssembly(options: ProcessingOptions, finalPath: string // Step 1: Upload the audio file l(wait('\n Uploading audio file to AssemblyAI...')) const uploadUrl = `${BASE_URL}/upload` - const fileStream = createReadStream(audioFilePath) + const fileBuffer = await readFile(audioFilePath) const uploadResponse = await fetch(uploadUrl, { method: 'POST', headers: { - 'Authorization': env.ASSEMBLY_API_KEY, + 'Authorization': env['ASSEMBLY_API_KEY'], 'Content-Type': 'application/octet-stream', }, - body: fileStream, - duplex: 'half', + body: fileBuffer }) if (!uploadResponse.ok) { @@ -119,9 +117,15 @@ export async function callAssembly(options: ProcessingOptions, finalPath: string `${speakerLabels ? `Speaker ${utt.speaker} ` : ''}(${formatTime(utt.start)}): ${utt.text}` ).join('\n') } else if (transcript.words && transcript.words.length > 0) { + // Check if words array exists and has content + const firstWord = transcript.words[0] + if (!firstWord) { + throw new Error('No words found in transcript') + } + // If only words are available, group them into lines with timestamps let currentLine = '' - let currentTimestamp = formatTime(transcript.words[0].start) + let currentTimestamp = formatTime(firstWord.start) transcript.words.forEach((word: AssemblyAIWord) => { if (currentLine.length + word.text.length > 80) { @@ -152,8 +156,7 @@ export async function callAssembly(options: ProcessingOptions, finalPath: string return txtContent } catch (error) { - // Log any errors that occur during the transcription process err(`Error processing the transcription: ${(error as Error).message}`) - throw error // Re-throw the error for handling in the calling function + throw error } } \ No newline at end of file diff --git a/src/transcription/deepgram.ts b/src/transcription/deepgram.ts index 9998416..7eb92c0 100644 --- a/src/transcription/deepgram.ts +++ b/src/transcription/deepgram.ts @@ -3,23 +3,19 @@ import { writeFile, readFile } from 'node:fs/promises' import { env } from 'node:process' import { l, wait, err } from '../globals.js' -import type { ProcessingOptions, DeepgramResponse } from '../types.js' +import type { DeepgramResponse } from '../types.js' /** * Main function to handle transcription using Deepgram API. - * @param {ProcessingOptions} options - Additional processing options. * @param {string} finalPath - The identifier used for naming output files. * @returns {Promise} - Returns the formatted transcript content. * @throws {Error} - If an error occurs during transcription. */ -export async function callDeepgram(options: ProcessingOptions, finalPath: string): Promise { +export async function callDeepgram(finalPath: string): Promise { l(wait('\n Using Deepgram for transcription...\n')) - // l(`Options received in callDeepgram:\n`) - // l(options) - // l(`finalPath:`, finalPath) // Check if the DEEPGRAM_API_KEY environment variable is set - if (!env.DEEPGRAM_API_KEY) { + if (!env['DEEPGRAM_API_KEY']) { throw new Error('DEEPGRAM_API_KEY environment variable is not set. Please set it to your Deepgram API key.') } @@ -40,7 +36,7 @@ export async function callDeepgram(options: ProcessingOptions, finalPath: string const response = await fetch(apiUrl, { method: 'POST', headers: { - 'Authorization': `Token ${env.DEEPGRAM_API_KEY}`, + 'Authorization': `Token ${env['DEEPGRAM_API_KEY']}`, 'Content-Type': 'audio/wav' }, body: audioBuffer @@ -52,8 +48,16 @@ export async function callDeepgram(options: ProcessingOptions, finalPath: string const result = await response.json() as DeepgramResponse + // Add null checks and provide default values for the Deepgram response + const channel = result.results?.channels?.[0] + const alternative = channel?.alternatives?.[0] + + if (!alternative?.words) { + throw new Error('No transcription results found in Deepgram response') + } + // Extract the words array from the Deepgram API response - const txtContent = result.results.channels[0].alternatives[0].words + const txtContent = alternative.words // Use reduce to iterate over the words array and build the formatted transcript .reduce((acc, { word, start }, i, arr) => { // Determine if a timestamp should be added diff --git a/src/transcription/whisper.ts b/src/transcription/whisper.ts index a8e7471..fa03aff 100644 --- a/src/transcription/whisper.ts +++ b/src/transcription/whisper.ts @@ -11,6 +11,9 @@ import { existsSync } from 'node:fs' import { l, err, wait, success, WHISPER_MODELS, WHISPER_PYTHON_MODELS, execPromise } from '../globals.js' import type { ProcessingOptions, WhisperModelType, WhisperTranscriptServices } from '../types.js' +// Updated function signatures for the runner functions +type WhisperRunner = (finalPath: string, whisperModel: string) => Promise + /** * Main function to handle transcription using Whisper. * @param options - Additional processing options. @@ -69,7 +72,7 @@ export async function callWhisper( l(wait(`\n - whisperModel: ${whisperModel}`)) // Run the appropriate transcription method - await config.runner(options, finalPath, whisperModel) + await config.runner(finalPath, whisperModel) // Read the generated transcript file (assuming it's always `${finalPath}.txt`) const txtContent = await readFile(`${finalPath}.txt`, 'utf8') @@ -88,7 +91,7 @@ export async function callWhisper( * @param finalPath - Base path for files. * @param whisperModel - The Whisper model to use. */ -async function runWhisperCpp(options: ProcessingOptions, finalPath: string, whisperModel: string): Promise { +const runWhisperCpp: WhisperRunner = async (finalPath, whisperModel) => { const modelGGMLName = WHISPER_MODELS[whisperModel as WhisperModelType] l(wait(` - modelGGMLName: ${modelGGMLName}`)) @@ -123,7 +126,7 @@ async function runWhisperCpp(options: ProcessingOptions, finalPath: string, whis * @param finalPath - Base path for files. * @param whisperModel - The Whisper model to use. */ -async function runWhisperDocker(options: ProcessingOptions, finalPath: string, whisperModel: string): Promise { +const runWhisperDocker: WhisperRunner = async (finalPath, whisperModel) => { const modelGGMLName = WHISPER_MODELS[whisperModel as WhisperModelType] const CONTAINER_NAME = 'autoshow-whisper-1' const modelPathContainer = `/app/models/${modelGGMLName}` @@ -159,7 +162,7 @@ async function runWhisperDocker(options: ProcessingOptions, finalPath: string, w * @param finalPath - Base path for files. * @param whisperModel - The Whisper model to use. */ -async function runWhisperPython(options: ProcessingOptions, finalPath: string, whisperModel: string): Promise { +const runWhisperPython: WhisperRunner = async (finalPath, whisperModel) => { // Check if ffmpeg is installed try { await execPromise('ffmpeg -version') @@ -210,7 +213,7 @@ async function runWhisperPython(options: ProcessingOptions, finalPath: string, w * @param finalPath - Base path for files. * @param whisperModel - The Whisper model to use. */ -async function runWhisperDiarization(options: ProcessingOptions, finalPath: string, whisperModel: string): Promise { +const runWhisperDiarization: WhisperRunner = async (finalPath, whisperModel) => { // Check if the virtual environment exists const venvPythonPath = 'whisper-diarization/venv/bin/python' if (!existsSync(venvPythonPath)) { @@ -264,22 +267,23 @@ function srtToTxt(srtContent: string): string { return blocks .map(block => { const lines = block.split('\n').filter(line => line.trim() !== '') - if (lines.length >= 2) { - const timestampLine = lines[1] - const textLines = lines.slice(2) - const match = timestampLine.match(/(\d{2}):(\d{2}):(\d{2}),\d{3}/) - if (match) { - const hours = parseInt(match[1], 10) - const minutes = parseInt(match[2], 10) - const seconds = match[3] - const totalMinutes = hours * 60 + minutes - const timestamp = `[${String(totalMinutes).padStart(2, '0')}:${seconds}]` - const text = textLines.join(' ') - return `${timestamp} ${text}` - } - } - return null + if (lines.length < 2) return null + + const timestampLine = lines[1] + const textLines = lines.slice(2) + + const match = timestampLine?.match(/(\d{2}):(\d{2}):(\d{2}),\d{3}/) + if (!match?.[1] || !match?.[2] || !match?.[3]) return null + + const hours = parseInt(match[1], 10) + const minutes = parseInt(match[2], 10) + const seconds = match[3] + const totalMinutes = hours * 60 + minutes + const timestamp = `[${String(totalMinutes).padStart(2, '0')}:${seconds}]` + const text = textLines.join(' ') + + return `${timestamp} ${text}` }) - .filter(line => line !== null) + .filter((line): line is string => line !== null) .join('\n') } diff --git a/src/utils/runTranscription.ts b/src/utils/runTranscription.ts index 51e5837..ec5876e 100644 --- a/src/utils/runTranscription.ts +++ b/src/utils/runTranscription.ts @@ -11,7 +11,7 @@ import { callWhisper } from '../transcription/whisper.js' import { callDeepgram } from '../transcription/deepgram.js' import { callAssembly } from '../transcription/assembly.js' import { l, step } from '../globals.js' -import { TranscriptServices, ProcessingOptions, WhisperTranscriptServices } from '../types.js' +import type { TranscriptServices, ProcessingOptions, WhisperTranscriptServices } from '../types.js' /** * Orchestrates the transcription process using the specified service. @@ -81,7 +81,6 @@ import { TranscriptServices, ProcessingOptions, WhisperTranscriptServices } from export async function runTranscription( options: ProcessingOptions, finalPath: string, - frontMatter: string, transcriptServices?: TranscriptServices ): Promise { l(step(`\nStep 3 - Running transcription on audio file using ${transcriptServices}...`)) @@ -90,7 +89,7 @@ export async function runTranscription( switch (transcriptServices) { case 'deepgram': // Cloud-based service with advanced features - await callDeepgram(options, finalPath) + await callDeepgram(finalPath) break case 'assembly': diff --git a/tsconfig.json b/tsconfig.json index deb01f8..5fa6be6 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,19 +1,71 @@ { "compilerOptions": { - "target": "ESNext", // Target modern JS features - "module": "ESNext", // Use ES modules - "lib": ["ESNext"], // Include modern JS features - "moduleResolution": "bundler", - "esModuleInterop": true, // Allow default imports from CJS modules - "skipLibCheck": true, // Skip type checking for node_modules - "forceConsistentCasingInFileNames": true, // Enforce file name casing consistency - "outDir": "./dist", // Output directory for compiled files - "rootDir": "./src", // Root directory for source files - "resolveJsonModule": true, // Allow importing JSON files + // **Modern JavaScript Features** + "target": "ESNext", // Target the latest ECMAScript features + "module": "ESNext", // Use native ECMAScript module system + "lib": ["ESNext", "DOM", "DOM.Iterable"], // Include latest ECMAScript features and DOM types + + // **Module Resolution and Imports** + "moduleResolution": "bundler", // Use module resolution that's suitable for bundlers + "allowSyntheticDefaultImports": true, // Allow default imports from modules with no default export + "forceConsistentCasingInFileNames": true, // Disallow inconsistently-cased references to the same file + "resolveJsonModule": true, // Allow importing .json files + "verbatimModuleSyntax": true, // Preserve import/export syntax in the emitted code + + // **Strict Type Checking** "strict": true, // Enable all strict type-checking options - "declaration": true, // Generate .d.ts files - "noEmitOnError": true // Prevent emitting files if there are errors + "noImplicitAny": true, // Error on expressions and declarations with an implied 'any' type + "strictNullChecks": true, // Enable strict null checks + "strictFunctionTypes": true, // Enable strict checking of function types + "strictBindCallApply": true, // Enable strict 'bind', 'call', and 'apply' methods on functions + "strictPropertyInitialization": true, // Ensure class properties are correctly initialized + "noImplicitThis": true, // Error on 'this' expressions with an implied 'any' type + "useUnknownInCatchVariables": true, // Default 'catch' clause variables as 'unknown' instead of 'any' + "alwaysStrict": true, // Parse in strict mode and emit "use strict" for each source file + + // **Additional Type Checking** + "noUncheckedIndexedAccess": true, // Include 'undefined' in indexed access results + "noPropertyAccessFromIndexSignature": true, // Disallow property access via indexing signatures + "exactOptionalPropertyTypes": true, // Interpret optional property types as written + "noImplicitReturns": true, // Error when not all code paths in function return a value + "noFallthroughCasesInSwitch": true, // Error for fallthrough cases in switch statements + "noImplicitOverride": true, // Ensure overrides are explicitly marked with an 'override' modifier + "noUnusedLocals": true, // Error on unused local variables + "noUnusedParameters": true, // Error on unused parameters + "allowUnusedLabels": false, // Error when labels are unused + "allowUnreachableCode": false, // Error on unreachable code + + // **Build and Performance** + "skipLibCheck": false, // Do not skip type checking of declaration files + "incremental": true, // Enable incremental compilation for faster rebuilds + + // **Emit Configuration** + "outDir": "./dist", // Redirect output structure to the 'dist' directory + "rootDir": "./src", // Specify the root directory of input files + "declaration": true, // Generate corresponding '.d.ts' files + "declarationMap": true, // Create sourcemaps for '.d.ts' files + "sourceMap": true, // Generate source map files + "noEmitOnError": true, // Do not emit outputs if any errors were reported + "removeComments": false, // Do not remove comments from output + + // **Path Mapping** + "baseUrl": ".", // Base directory to resolve non-absolute module names + "paths": { // Path alias mapping + "@/*": ["src/*"] + }, + + // **Experimental Features** + "isolatedModules": true // Ensure each file can be safely transpiled without relying on other imports }, - "include": ["src/**/*"], // Include all TypeScript files in the `src` folder - "exclude": ["node_modules", "dist"] // Exclude output and dependencies + "include": [ + "src/**/*", // Include all files in 'src' directory + "tests/**/*" // Include all files in 'tests' directory + ], + "exclude": [ + "node_modules", // Exclude 'node_modules' directory + "dist", // Exclude 'dist' directory + "coverage", // Exclude 'coverage' directory + "**/*.spec.ts", // Exclude test specification files + "**/*.test.ts" // Exclude test files + ] } \ No newline at end of file From 50cc3219fc75e01a50b25f08e0d98b2358b9078b Mon Sep 17 00:00:00 2001 From: Anthony Campolo <12433465+ajcwebdev@users.noreply.github.com> Date: Sat, 2 Nov 2024 02:04:40 -0500 Subject: [PATCH 2/5] add config options for processChannel --- .gitignore | 3 +- docs/examples.md | 51 ++++++++-- src/commands/processChannel.ts | 171 ++++++++++++++++++++++++++++++--- 3 files changed, 202 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 941a324..ddb2063 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ dist NEW.md TODO.md nemo_msdd_configs -temp_outputs \ No newline at end of file +temp_outputs +tsconfig.tsbuildinfo \ No newline at end of file diff --git a/docs/examples.md b/docs/examples.md index e2be889..879aecf 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -5,6 +5,7 @@ - [Content and Feed Inputs](#content-and-feed-inputs) - [Process Single Video URLs](#process-single-video-urls) - [Process Multiple Videos in YouTube Playlist](#process-multiple-videos-in-youtube-playlist) + - [Process All Videos from a YouTube Channel](#process-all-videos-from-a-youtube-channel) - [Process Multiple Videos Specified in a URLs File](#process-multiple-videos-specified-in-a-urls-file) - [Process Single Audio or Video File](#process-single-audio-or-video-file) - [Process Podcast RSS Feed](#process-podcast-rss-feed) @@ -62,11 +63,45 @@ npm run as -- \ ### Process All Videos from a YouTube Channel +Process all videos from a YouTube channel (both live and non-live): + ```bash npm run as -- \ --channel "https://www.youtube.com/@ajcwebdev" ``` +Process videos starting from the oldest instead of newest: + +```bash +npm run as -- \ + --channel "https://www.youtube.com/@ajcwebdev" \ + --order oldest +``` + +Skip a certain number of videos before beginning processing (starts from newest by default and can be used with `--order oldest`): + +```bash +npm run as -- \ + --channel "https://www.youtube.com/@ajcwebdev" \ + --skip 1 +``` + +Process a certain number of the most recent videos, for example the last three videos released on the channel: + +```bash +npm run as -- \ + --channel "https://www.youtube.com/@ajcwebdev" \ + --last 3 +``` + +Run on a YouTube channel and generate JSON info file with markdown metadata of each video: + +```bash +npm run as -- \ + --channel "https://www.youtube.com/@ajcwebdev" \ + --info +``` + ### Process Multiple Videos Specified in a URLs File Run on an arbitrary list of URLs in `example-urls.md`. @@ -102,28 +137,28 @@ npm run as -- \ --rss "https://ajcwebdev.substack.com/feed" ``` -Process a certain number of the most recent items, for example the last three episodes released on the feed: +Process RSS feed from oldest to newest: ```bash npm run as -- \ --rss "https://feeds.transistor.fm/fsjam-podcast/" \ - --last 3 + --order oldest ``` -Process RSS feed from oldest to newest: +Start processing a different episode by selecting a number of episodes to skip: ```bash npm run as -- \ --rss "https://feeds.transistor.fm/fsjam-podcast/" \ - --order oldest + --skip 1 ``` -Start processing a different episode by selecting a number of episodes to skip: +Process a certain number of the most recent items, for example the last three episodes released on the feed: ```bash npm run as -- \ --rss "https://feeds.transistor.fm/fsjam-podcast/" \ - --skip 1 + --last 3 ``` Process a single specific episode from a podcast RSS feed by providing the episode's audio URL with the `--item` option: @@ -131,9 +166,7 @@ Process a single specific episode from a podcast RSS feed by providing the episo ```bash npm run as -- \ --rss "https://ajcwebdev.substack.com/feed" \ - --item "https://api.substack.com/feed/podcast/36236609/fd1f1532d9842fe1178de1c920442541.mp3" \ - --ollama \ - --prompt titles summary longChapters takeaways questions + --item "https://api.substack.com/feed/podcast/36236609/fd1f1532d9842fe1178de1c920442541.mp3" ``` Run on a podcast RSS feed and generate JSON info file with markdown metadata of each item: diff --git a/src/commands/processChannel.ts b/src/commands/processChannel.ts index e7dabf1..87c4f83 100644 --- a/src/commands/processChannel.ts +++ b/src/commands/processChannel.ts @@ -7,11 +7,133 @@ import { writeFile } from 'node:fs/promises' import { processVideo } from './processVideo.js' -import { l, err, opts, success, execFilePromise } from '../globals.js' +import { l, err, opts, success, execFilePromise, wait } from '../globals.js' import type { LLMServices, TranscriptServices, ProcessingOptions, VideoMetadata, } from '../types.js' +/** + * Validates channel processing options for consistency and correct values. + * + * @param options - Configuration options to validate + * @throws Will exit process if validation fails + */ +function validateChannelOptions(options: ProcessingOptions): void { + if (options.last !== undefined) { + if (!Number.isInteger(options.last) || options.last < 1) { + err('Error: The --last option must be a positive integer.') + process.exit(1) + } + if (options.skip !== undefined || options.order !== undefined) { + err('Error: The --last option cannot be used with --skip or --order.') + process.exit(1) + } + } + + if (options.skip !== undefined && (!Number.isInteger(options.skip) || options.skip < 0)) { + err('Error: The --skip option must be a non-negative integer.') + process.exit(1) + } + + if (options.order !== undefined && !['newest', 'oldest'].includes(options.order)) { + err("Error: The --order option must be either 'newest' or 'oldest'.") + process.exit(1) + } +} + +/** + * Logs the current processing action based on provided options. + * + * @param options - Configuration options determining what to process + */ +function logProcessingAction(options: ProcessingOptions): void { + if (options.last) { + l(wait(`\nProcessing the last ${options.last} videos`)) + } else if (options.skip) { + l(wait(`\nSkipping first ${options.skip || 0} videos`)) + } +} + +/** + * Video information including upload date, URL, and type. + */ +interface VideoInfo { + uploadDate: string + url: string + date: Date + timestamp: number // Unix timestamp for more precise sorting + isLive: boolean // Flag to identify live streams +} + +/** + * Gets detailed video information using yt-dlp. + * + * @param url - Video URL to get information for + * @returns Promise resolving to video information + */ +async function getVideoDetails(url: string): Promise { + try { + const { stdout } = await execFilePromise('yt-dlp', [ + '--print', '%(upload_date)s|%(timestamp)s|%(is_live)s|%(webpage_url)s', + '--no-warnings', + url, + ]) + + const [uploadDate, timestamp, isLive, videoUrl] = stdout.trim().split('|') + + if (!uploadDate || !timestamp || !videoUrl) { + throw new Error('Incomplete video information received from yt-dlp') + } + + // Convert upload date to Date object + const year = uploadDate.substring(0, 4) + const month = uploadDate.substring(4, 6) + const day = uploadDate.substring(6, 8) + const date = new Date(`${year}-${month}-${day}`) + + return { + uploadDate, + url: videoUrl, + date, + timestamp: parseInt(timestamp, 10) || date.getTime() / 1000, + isLive: isLive === 'True' + } + } catch (error) { + err(`Error getting details for video ${url}: ${error instanceof Error ? error.message : String(error)}`) + return null + } +} + +/** + * Selects which videos to process based on provided options. + * + * @param videos - All available videos with their dates and URLs + * @param options - Configuration options for filtering + * @returns Array of video info to process + */ +function selectVideosToProcess(videos: VideoInfo[], options: ProcessingOptions): VideoInfo[] { + if (options.last) { + return videos.slice(0, options.last) + } + return videos.slice(options.skip || 0) +} + +/** + * Logs the processing status and video counts. + */ +function logProcessingStatus(total: number, processing: number, options: ProcessingOptions): void { + if (options.last) { + l(wait(`\n - Found ${total} videos in the channel.`)) + l(wait(` - Processing the last ${processing} videos.`)) + } else if (options.skip) { + l(wait(`\n - Found ${total} videos in the channel.`)) + l(wait(` - Processing ${processing} videos after skipping ${options.skip || 0}.\n`)) + } else { + l(wait(`\n - Found ${total} videos in the channel.`)) + l(wait(` - Processing all ${processing} videos.\n`)) + } +} + /** * Processes an entire YouTube channel by: * 1. Fetching all video URLs from the channel using yt-dlp. @@ -38,10 +160,14 @@ export async function processChannel( l(opts(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}`)) try { - // Extract all video URLs from the channel using yt-dlp + // Validate options + validateChannelOptions(options) + logProcessingAction(options) + + // Get list of videos from channel const { stdout, stderr } = await execFilePromise('yt-dlp', [ '--flat-playlist', - '--print', 'url', + '--print', '%(url)s', '--no-warnings', channelUrl, ]) @@ -51,22 +177,40 @@ export async function processChannel( err(`yt-dlp warnings: ${stderr}`) } - // Convert stdout into array of video URLs, removing empty entries - const urls = stdout.trim().split('\n').filter(Boolean) + // Get detailed information for each video + const videoUrls = stdout.trim().split('\n').filter(Boolean) + l(opts(`\nFetching detailed information for ${videoUrls.length} videos...`)) + + const videoDetailsPromises = videoUrls.map(url => getVideoDetails(url)) + const videoDetailsResults = await Promise.all(videoDetailsPromises) + const videos = videoDetailsResults.filter((video): video is VideoInfo => video !== null) // Exit if no videos were found in the channel - if (urls.length === 0) { + if (videos.length === 0) { err('Error: No videos found in the channel.') process.exit(1) } - l(opts(`\nFound ${urls.length} videos in the channel...`)) + // Sort videos based on timestamp + videos.sort((a, b) => a.timestamp - b.timestamp) + + // If order is 'newest' (default), reverse the sorted array + if (options.order !== 'oldest') { + videos.reverse() + } + + l(opts(`\nFound ${videos.length} videos in the channel...`)) + + // Select videos to process based on options + const videosToProcess = selectVideosToProcess(videos, options) + logProcessingStatus(videos.length, videosToProcess.length, options) - // If the --info option is provided, extract metadata for all videos + // If the --info option is provided, extract metadata for selected videos if (options.info) { - // Collect metadata for all videos in parallel + // Collect metadata for selected videos in parallel const metadataList = await Promise.all( - urls.map(async (url) => { + videosToProcess.map(async (video) => { + const url = video.url try { // Execute yt-dlp command to extract metadata const { stdout } = await execFilePromise('yt-dlp', [ @@ -126,10 +270,11 @@ export async function processChannel( } // Process each video sequentially, with error handling for individual videos - for (const [index, url] of urls.entries()) { + for (const [index, video] of videosToProcess.entries()) { + const url = video.url // Visual separator for each video in the console l(opts(`\n================================================================================================`)) - l(opts(` Processing video ${index + 1}/${urls.length}: ${url}`)) + l(opts(` Processing video ${index + 1}/${videosToProcess.length}: ${url}`)) l(opts(`================================================================================================\n`)) try { // Process the video using the existing processVideo function @@ -144,4 +289,4 @@ export async function processChannel( err(`Error processing channel: ${(error as Error).message}`) process.exit(1) } -} +} \ No newline at end of file From c2653671ed3716f275b17510391f8579d2b854f9 Mon Sep 17 00:00:00 2001 From: Anthony Campolo <12433465+ajcwebdev@users.noreply.github.com> Date: Sun, 3 Nov 2024 02:04:19 -0700 Subject: [PATCH 3/5] add channel to interactive --- src/autoshow.ts | 2 +- src/interactive.ts | 14 ++++++++++++++ src/types.ts | 3 +++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/autoshow.ts b/src/autoshow.ts index 4dd30c2..a4e8065 100644 --- a/src/autoshow.ts +++ b/src/autoshow.ts @@ -66,7 +66,7 @@ program .option('--order ', 'Specify the order for RSS feed processing (newest or oldest)') .option('--skip ', 'Number of items to skip when processing RSS feed', parseInt) .option('--last ', 'Number of most recent items to process (overrides --order and --skip)', parseInt) - .option('--info', 'Generate JSON file with RSS feed or channel information instead of processing items') + .option('--info', 'Skip processing and write metadata to JSON objects (supports --urls, --rss, --playlist, --channel)') // Transcription service options .option('--whisper [model]', 'Use Whisper.cpp for transcription with optional model specification') .option('--whisperDocker [model]', 'Use Whisper.cpp in Docker for transcription with optional model specification') diff --git a/src/interactive.ts b/src/interactive.ts index b11a068..19b78cb 100644 --- a/src/interactive.ts +++ b/src/interactive.ts @@ -25,6 +25,7 @@ export async function handleInteractivePrompt( choices: [ { name: 'Single YouTube Video', value: 'video' }, { name: 'YouTube Playlist', value: 'playlist' }, + { name: 'YouTube Channel', value: 'channel' }, { name: 'List of URLs from File', value: 'urls' }, { name: 'Local Audio/Video File', value: 'file' }, { name: 'Podcast RSS Feed', value: 'rss' }, @@ -32,6 +33,7 @@ export async function handleInteractivePrompt( }, // Input prompts for different content sources { + // Input prompt for YouTube Video type: 'input', name: 'video', message: 'Enter the YouTube video URL:', @@ -39,6 +41,7 @@ export async function handleInteractivePrompt( validate: (input: string) => (input ? true : 'Please enter a valid URL.'), }, { + // Input prompt for YouTube Playlist type: 'input', name: 'playlist', message: 'Enter the YouTube playlist URL:', @@ -46,6 +49,15 @@ export async function handleInteractivePrompt( validate: (input: string) => (input ? true : 'Please enter a valid URL.'), }, { + // Input prompt for YouTube Channel + type: 'input', + name: 'channel', + message: 'Enter the YouTube channel URL:', + when: (answers: InquirerAnswers) => answers.action === 'channel', + validate: (input: string) => (input ? true : 'Please enter a valid URL.'), + }, + { + // Input prompt for file containing a list of URLs type: 'input', name: 'urls', message: 'Enter the file path containing URLs:', @@ -53,6 +65,7 @@ export async function handleInteractivePrompt( validate: (input: string) => (input ? true : 'Please enter a valid file path.'), }, { + // Input prompt for local audio and video files type: 'input', name: 'file', message: 'Enter the local audio/video file path:', @@ -61,6 +74,7 @@ export async function handleInteractivePrompt( }, // RSS feed specific prompts { + // Input prompt for RSS feed type: 'input', name: 'rss', message: 'Enter the podcast RSS feed URL:', diff --git a/src/types.ts b/src/types.ts index 2fa3dd7..68ac133 100644 --- a/src/types.ts +++ b/src/types.ts @@ -121,6 +121,9 @@ export type InquirerAnswers = { /** YouTube playlist URL provided by the user. */ playlist?: string + /** YouTube channel URL provided by the user. */ + channel?: string + /** File path containing URLs provided by the user. */ urls?: string From 25f3c1f804677556390b66d37d2d7901c1d5ab4e Mon Sep 17 00:00:00 2001 From: Anthony Campolo <12433465+ajcwebdev@users.noreply.github.com> Date: Sun, 3 Nov 2024 02:46:21 -0700 Subject: [PATCH 4/5] extract interactive variables --- src/autoshow.ts | 10 +-- src/globals.ts | 43 +++++++++- src/interactive.ts | 208 +++++++++++++++++++++++++-------------------- 3 files changed, 160 insertions(+), 101 deletions(-) diff --git a/src/autoshow.ts b/src/autoshow.ts index a4e8065..49c9482 100644 --- a/src/autoshow.ts +++ b/src/autoshow.ts @@ -119,11 +119,8 @@ program.action(async (options: ProcessingOptions) => { // Extract interactive mode flag const { interactive } = options - // Check if no action option was provided - const noActionProvided = ACTION_OPTIONS.every((opt) => !options[opt as keyof ProcessingOptions]) - // If in interactive mode or no action provided, prompt user for input - if (interactive || noActionProvided) { + if (interactive) { options = await handleInteractivePrompt(options) } @@ -131,9 +128,10 @@ program.action(async (options: ProcessingOptions) => { if (options.item && !Array.isArray(options.item)) { options.item = [options.item] } - + // Extract the action values from ACTION_OPTIONS for validation + const actionValues = ACTION_OPTIONS.map((opt) => opt.name) // Validate and get single options for action, LLM, and transcription - const action = validateOption(ACTION_OPTIONS, options, 'input option') + const action = validateOption(actionValues, options, 'input option') const llmKey = validateOption(LLM_OPTIONS, options, 'LLM option') const llmServices = llmKey as LLMServices | undefined const transcriptKey = validateOption(TRANSCRIPT_OPTIONS, options, 'transcription option') diff --git a/src/globals.ts b/src/globals.ts index dd90586..3e56959 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -74,10 +74,47 @@ export const l: typeof console.log = console.log export const err: typeof console.error = console.error /** - * Available action options for content processing - * @type {string[]} + * Available action options for content processing with additional metadata + * @type {{ name: string; value: string; message: string; validate: (input: string) => boolean | string }[]} */ -export const ACTION_OPTIONS = ['video', 'playlist', 'channel', 'urls', 'file', 'rss'] +export const ACTION_OPTIONS = [ + { + name: 'video', + description: 'Single YouTube Video', + message: 'Enter the YouTube video URL:', + validate: (input: string) => (input ? true : 'Please enter a valid URL.') + }, + { + name: 'playlist', + description: 'YouTube Playlist', + message: 'Enter the YouTube playlist URL:', + validate: (input: string) => (input ? true : 'Please enter a valid URL.') + }, + { + name: 'channel', + description: 'YouTube Channel', + message: 'Enter the YouTube channel URL:', + validate: (input: string) => (input ? true : 'Please enter a valid URL.') + }, + { + name: 'urls', + description: 'List of URLs from File', + message: 'Enter the file path containing URLs:', + validate: (input: string) => (input ? true : 'Please enter a valid file path.') + }, + { + name: 'file', + description: 'Local Audio/Video File', + message: 'Enter the local audio/video file path:', + validate: (input: string) => (input ? true : 'Please enter a valid file path.') + }, + { + name: 'rss', + description: 'Podcast RSS Feed', + message: 'Enter the podcast RSS feed URL:', + validate: (input: string) => (input ? true : 'Please enter a valid URL.') + } +] /** * Available LLM service options diff --git a/src/interactive.ts b/src/interactive.ts index 19b78cb..6d4e1ab 100644 --- a/src/interactive.ts +++ b/src/interactive.ts @@ -4,6 +4,109 @@ import inquirer from 'inquirer' import type { ProcessingOptions, InquirerAnswers, WhisperModelType } from './types.js' import { l } from './globals.js' +const TRANSCRIPT_CHOICES = [ + { name: 'Whisper.cpp', value: 'whisper' }, + { name: 'Whisper.cpp (Docker)', value: 'whisperDocker' }, + { name: 'Whisper Python', value: 'whisperPython' }, + { name: 'Whisper Diarization', value: 'whisperDiarization' }, + { name: 'Deepgram', value: 'deepgram' }, + { name: 'AssemblyAI', value: 'assembly' }, +] + +const WHISPER_MODEL_CHOICES = [ + 'tiny', + 'tiny.en', + 'base', + 'base.en', + 'small', + 'small.en', + 'medium', + 'medium.en', + 'large-v1', + 'large-v2', + 'turbo', +] + +const LLM_CHOICES = [ + 'ollama', + 'chatgpt', + 'claude', + 'cohere', + 'mistral', + 'fireworks', + 'together', + 'groq', + 'gemini', +] + +const OLLAMA_CHOICES = [ + { name: 'LLAMA 3 2 1B', value: 'LLAMA_3_2_1B' }, + { name: 'LLAMA 3 2 3B', value: 'LLAMA_3_2_3B' }, + { name: 'GEMMA 2 2B', value: 'GEMMA_2_2B' }, + { name: 'PHI 3 5', value: 'PHI_3_5' }, + { name: 'QWEN 2 5 1B', value: 'QWEN_2_5_1B' }, + { name: 'QWEN 2 5 3B', value: 'QWEN_2_5_3B' }, +] + +const CHATGPT_CHOICES = [ + { name: 'GPT 4 o MINI', value: 'GPT_4o_MINI' }, + { name: 'GPT 4 o', value: 'GPT_4o' }, + { name: 'GPT 4 TURBO', value: 'GPT_4_TURBO' }, + { name: 'GPT 4', value: 'GPT_4' }, +] + +const CLAUDE_CHOICES = [ + { name: 'Claude 3.5 Sonnet', value: 'CLAUDE_3_5_SONNET' }, + { name: 'Claude 3 Opus', value: 'CLAUDE_3_OPUS' }, + { name: 'Claude 3 Sonnet', value: 'CLAUDE_3_SONNET' }, + { name: 'Claude 3 Haiku', value: 'CLAUDE_3_HAIKU' }, +] + +const COHERE_CHOICES = [ + { name: 'Command R', value: 'COMMAND_R' }, + { name: 'Command R Plus', value: 'COMMAND_R_PLUS' }, +] + +const MISTRAL_CHOICES = [ + { name: 'Mixtral 8x7b', value: 'MIXTRAL_8x7b' }, + { name: 'Mixtral 8x22b', value: 'MIXTRAL_8x22b' }, + { name: 'Mistral Large', value: 'MISTRAL_LARGE' }, + { name: 'Mistral Nemo', value: 'MISTRAL_NEMO' }, +] + +const FIREWORKS_CHOICES = [ + { name: 'LLAMA 3 1 405B', value: 'LLAMA_3_1_405B' }, + { name: 'LLAMA 3 1 70B', value: 'LLAMA_3_1_70B' }, + { name: 'LLAMA 3 1 8B', value: 'LLAMA_3_1_8B' }, + { name: 'LLAMA 3 2 3B', value: 'LLAMA_3_2_3B' }, + { name: 'LLAMA 3 2 1B', value: 'LLAMA_3_2_1B' }, + { name: 'QWEN 2 5 72B', value: 'QWEN_2_5_72B' }, +] + +const TOGETHER_CHOICES = [ + { name: 'LLAMA 3 2 3B', value: 'LLAMA_3_2_3B' }, + { name: 'LLAMA 3 1 405B', value: 'LLAMA_3_1_405B' }, + { name: 'LLAMA 3 1 70B', value: 'LLAMA_3_1_70B' }, + { name: 'LLAMA 3 1 8B', value: 'LLAMA_3_1_8B' }, + { name: 'Gemma 2 27B', value: 'GEMMA_2_27B' }, + { name: 'Gemma 2 9B', value: 'GEMMA_2_9B' }, + { name: 'QWEN 2 5 72B', value: 'QWEN_2_5_72B' }, + { name: 'QWEN 2 5 7B', value: 'QWEN_2_5_7B' }, +] + +const GROQ_CHOICES = [ + { name: 'LLAMA 3 1 70B Versatile', value: 'LLAMA_3_1_70B_VERSATILE' }, + { name: 'LLAMA 3 1 8B Instant', value: 'LLAMA_3_1_8B_INSTANT' }, + { name: 'LLAMA 3 2 1B Preview', value: 'LLAMA_3_2_1B_PREVIEW' }, + { name: 'LLAMA 3 2 3B Preview', value: 'LLAMA_3_2_3B_PREVIEW' }, + { name: 'Mixtral 8x7b 32768', value: 'MIXTRAL_8X7B_32768' }, +] + +const GEMINI_CHOICES = [ + { name: 'Gemini 1.5 Flash', value: 'GEMINI_1_5_FLASH' }, + { name: 'Gemini 1.5 Pro', value: 'GEMINI_1_5_PRO' }, +] + /** * Prompts the user for input if interactive mode is selected. * Handles the collection and processing of user choices through a series of @@ -156,122 +259,43 @@ export async function handleInteractivePrompt( // Return appropriate model choices based on selected LLM service switch (answers.llmServices) { case 'ollama': - return [ - { name: 'LLAMA 3 2 1B', value: 'LLAMA_3_2_1B' }, - { name: 'LLAMA 3 2 3B', value: 'LLAMA_3_2_3B' }, - { name: 'GEMMA 2 2B', value: 'GEMMA_2_2B' }, - { name: 'PHI 3 5', value: 'PHI_3_5' }, - { name: 'QWEN 2 5 1B', value: 'QWEN_2_5_1B' }, - { name: 'QWEN 2 5 3B', value: 'QWEN_2_5_3B' }, - ] + return OLLAMA_CHOICES case 'chatgpt': - return [ - { name: 'GPT 4 o MINI', value: 'GPT_4o_MINI' }, - { name: 'GPT 4 o', value: 'GPT_4o' }, - { name: 'GPT 4 TURBO', value: 'GPT_4_TURBO' }, - { name: 'GPT 4', value: 'GPT_4' }, - ] + return CHATGPT_CHOICES case 'claude': - return [ - { name: 'Claude 3.5 Sonnet', value: 'CLAUDE_3_5_SONNET' }, - { name: 'Claude 3 Opus', value: 'CLAUDE_3_OPUS' }, - { name: 'Claude 3 Sonnet', value: 'CLAUDE_3_SONNET' }, - { name: 'Claude 3 Haiku', value: 'CLAUDE_3_HAIKU' }, - ] + return CLAUDE_CHOICES + case 'gemini': + return GEMINI_CHOICES case 'cohere': - return [ - { name: 'Command R', value: 'COMMAND_R' }, - { name: 'Command R Plus', value: 'COMMAND_R_PLUS' }, - ] + return COHERE_CHOICES case 'mistral': - return [ - { name: 'Mixtral 8x7b', value: 'MIXTRAL_8x7b' }, - { name: 'Mixtral 8x22b', value: 'MIXTRAL_8x22b' }, - { name: 'Mistral Large', value: 'MISTRAL_LARGE' }, - { name: 'Mistral Nemo', value: 'MISTRAL_NEMO' }, - ] + return MISTRAL_CHOICES case 'fireworks': - return [ - { name: 'LLAMA 3 1 405B', value: 'LLAMA_3_1_405B' }, - { name: 'LLAMA 3 1 70B', value: 'LLAMA_3_1_70B' }, - { name: 'LLAMA 3 1 8B', value: 'LLAMA_3_1_8B' }, - { name: 'LLAMA 3 2 3B', value: 'LLAMA_3_2_3B' }, - { name: 'LLAMA 3 2 1B', value: 'LLAMA_3_2_1B' }, - { name: 'QWEN 2 5 72B', value: 'QWEN_2_5_72B' }, - ] + return FIREWORKS_CHOICES case 'together': - return [ - { name: 'LLAMA 3 2 3B', value: 'LLAMA_3_2_3B' }, - { name: 'LLAMA 3 1 405B', value: 'LLAMA_3_1_405B' }, - { name: 'LLAMA 3 1 70B', value: 'LLAMA_3_1_70B' }, - { name: 'LLAMA 3 1 8B', value: 'LLAMA_3_1_8B' }, - { name: 'Gemma 2 27B', value: 'GEMMA_2_27B' }, - { name: 'Gemma 2 9B', value: 'GEMMA_2_9B' }, - { name: 'QWEN 2 5 72B', value: 'QWEN_2_5_72B' }, - { name: 'QWEN 2 5 7B', value: 'QWEN_2_5_7B' }, - ] + return TOGETHER_CHOICES case 'groq': - return [ - { name: 'LLAMA 3 1 70B Versatile', value: 'LLAMA_3_1_70B_VERSATILE' }, - { name: 'LLAMA 3 1 8B Instant', value: 'LLAMA_3_1_8B_INSTANT' }, - { name: 'LLAMA 3 2 1B Preview', value: 'LLAMA_3_2_1B_PREVIEW' }, - { name: 'LLAMA 3 2 3B Preview', value: 'LLAMA_3_2_3B_PREVIEW' }, - { name: 'Mixtral 8x7b 32768', value: 'MIXTRAL_8X7B_32768' }, - ] - case 'gemini': - return [ - { name: 'Gemini 1.5 Flash', value: 'GEMINI_1_5_FLASH' }, - { name: 'Gemini 1.5 Pro', value: 'GEMINI_1_5_PRO' }, - ] + return GROQ_CHOICES default: return [] } }, when: (answers: InquirerAnswers) => - [ - 'ollama', - 'chatgpt', - 'claude', - 'cohere', - 'mistral', - 'fireworks', - 'together', - 'groq', - 'gemini', - ].includes(answers.llmServices as string), + LLM_CHOICES.includes(answers.llmServices as string), }, // Transcription service configuration { type: 'list', name: 'transcriptServices', message: 'Select the transcription service you want to use:', - choices: [ - { name: 'Whisper.cpp', value: 'whisper' }, - { name: 'Whisper.cpp (Docker)', value: 'whisperDocker' }, - { name: 'Whisper Python', value: 'whisperPython' }, - { name: 'Whisper Diarization', value: 'whisperDiarization' }, - { name: 'Deepgram', value: 'deepgram' }, - { name: 'AssemblyAI', value: 'assembly' }, - ], + choices: TRANSCRIPT_CHOICES, }, // Whisper model configuration { type: 'list', name: 'whisperModel', message: 'Select the Whisper model type:', - choices: [ - 'tiny', - 'tiny.en', - 'base', - 'base.en', - 'small', - 'small.en', - 'medium', - 'medium.en', - 'large-v1', - 'large-v2', - 'turbo', - ], + choices: WHISPER_MODEL_CHOICES, when: (answers: InquirerAnswers) => ['whisper', 'whisperDocker', 'whisperPython', 'whisperDiarization'].includes( answers.transcriptServices as string From 0fa5b5dd8f65e4b8ae2873dc8244a4684e308786 Mon Sep 17 00:00:00 2001 From: Anthony Campolo <12433465+ajcwebdev@users.noreply.github.com> Date: Tue, 5 Nov 2024 10:51:00 -0800 Subject: [PATCH 5/5] refactor interactive cli file --- src/globals.ts | 142 +++++++++++++++++++++++++++++++++++++++++++++ src/interactive.ts | 141 +++----------------------------------------- 2 files changed, 149 insertions(+), 134 deletions(-) diff --git a/src/globals.ts b/src/globals.ts index 3e56959..9ad32cc 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -15,6 +15,148 @@ import type { WhisperModelType, ChatGPTModelType, ClaudeModelType, CohereModelTy export const execPromise = promisify(exec) export const execFilePromise = promisify(execFile) +export const PROCESS_CHOICES = [ + { name: 'Single YouTube Video', value: 'video' }, + { name: 'YouTube Playlist', value: 'playlist' }, + { name: 'YouTube Channel', value: 'channel' }, + { name: 'List of URLs from File', value: 'urls' }, + { name: 'Local Audio/Video File', value: 'file' }, + { name: 'Podcast RSS Feed', value: 'rss' }, +] + +export const TRANSCRIPT_CHOICES = [ + { name: 'Whisper.cpp', value: 'whisper' }, + { name: 'Whisper.cpp (Docker)', value: 'whisperDocker' }, + { name: 'Whisper Python', value: 'whisperPython' }, + { name: 'Whisper Diarization', value: 'whisperDiarization' }, + { name: 'Deepgram', value: 'deepgram' }, + { name: 'AssemblyAI', value: 'assembly' }, +] + +export const WHISPER_MODEL_CHOICES = [ + 'tiny', + 'tiny.en', + 'base', + 'base.en', + 'small', + 'small.en', + 'medium', + 'medium.en', + 'large-v1', + 'large-v2', + 'turbo', +] + +export const WHISPER_LIBRARY_CHOICES = [ + 'whisper', + 'whisperDocker', + 'whisperPython', + 'whisperDiarization' +] + +export const LLM_SERVICE_CHOICES = [ + { name: 'Skip LLM Processing', value: null }, + { name: 'Ollama (local inference)', value: 'ollama' }, + { name: 'OpenAI ChatGPT', value: 'chatgpt' }, + { name: 'Anthropic Claude', value: 'claude' }, + { name: 'Google Gemini', value: 'gemini' }, + { name: 'Cohere', value: 'cohere' }, + { name: 'Mistral', value: 'mistral' }, + { name: 'Fireworks AI', value: 'fireworks' }, + { name: 'Together AI', value: 'together' }, + { name: 'Groq', value: 'groq' }, +] + +export const LLM_CHOICES = [ + 'ollama', + 'chatgpt', + 'claude', + 'cohere', + 'mistral', + 'fireworks', + 'together', + 'groq', + 'gemini', +] + +export const OLLAMA_CHOICES = [ + { name: 'LLAMA 3 2 1B', value: 'LLAMA_3_2_1B' }, + { name: 'LLAMA 3 2 3B', value: 'LLAMA_3_2_3B' }, + { name: 'GEMMA 2 2B', value: 'GEMMA_2_2B' }, + { name: 'PHI 3 5', value: 'PHI_3_5' }, + { name: 'QWEN 2 5 1B', value: 'QWEN_2_5_1B' }, + { name: 'QWEN 2 5 3B', value: 'QWEN_2_5_3B' }, +] + +export const CHATGPT_CHOICES = [ + { name: 'GPT 4 o MINI', value: 'GPT_4o_MINI' }, + { name: 'GPT 4 o', value: 'GPT_4o' }, + { name: 'GPT 4 TURBO', value: 'GPT_4_TURBO' }, + { name: 'GPT 4', value: 'GPT_4' }, +] + +export const CLAUDE_CHOICES = [ + { name: 'Claude 3.5 Sonnet', value: 'CLAUDE_3_5_SONNET' }, + { name: 'Claude 3 Opus', value: 'CLAUDE_3_OPUS' }, + { name: 'Claude 3 Sonnet', value: 'CLAUDE_3_SONNET' }, + { name: 'Claude 3 Haiku', value: 'CLAUDE_3_HAIKU' }, +] + +export const COHERE_CHOICES = [ + { name: 'Command R', value: 'COMMAND_R' }, + { name: 'Command R Plus', value: 'COMMAND_R_PLUS' }, +] + +export const MISTRAL_CHOICES = [ + { name: 'Mixtral 8x7b', value: 'MIXTRAL_8x7b' }, + { name: 'Mixtral 8x22b', value: 'MIXTRAL_8x22b' }, + { name: 'Mistral Large', value: 'MISTRAL_LARGE' }, + { name: 'Mistral Nemo', value: 'MISTRAL_NEMO' }, +] + +export const FIREWORKS_CHOICES = [ + { name: 'LLAMA 3 1 405B', value: 'LLAMA_3_1_405B' }, + { name: 'LLAMA 3 1 70B', value: 'LLAMA_3_1_70B' }, + { name: 'LLAMA 3 1 8B', value: 'LLAMA_3_1_8B' }, + { name: 'LLAMA 3 2 3B', value: 'LLAMA_3_2_3B' }, + { name: 'LLAMA 3 2 1B', value: 'LLAMA_3_2_1B' }, + { name: 'QWEN 2 5 72B', value: 'QWEN_2_5_72B' }, +] + +export const TOGETHER_CHOICES = [ + { name: 'LLAMA 3 2 3B', value: 'LLAMA_3_2_3B' }, + { name: 'LLAMA 3 1 405B', value: 'LLAMA_3_1_405B' }, + { name: 'LLAMA 3 1 70B', value: 'LLAMA_3_1_70B' }, + { name: 'LLAMA 3 1 8B', value: 'LLAMA_3_1_8B' }, + { name: 'Gemma 2 27B', value: 'GEMMA_2_27B' }, + { name: 'Gemma 2 9B', value: 'GEMMA_2_9B' }, + { name: 'QWEN 2 5 72B', value: 'QWEN_2_5_72B' }, + { name: 'QWEN 2 5 7B', value: 'QWEN_2_5_7B' }, +] + +export const GROQ_CHOICES = [ + { name: 'LLAMA 3 1 70B Versatile', value: 'LLAMA_3_1_70B_VERSATILE' }, + { name: 'LLAMA 3 1 8B Instant', value: 'LLAMA_3_1_8B_INSTANT' }, + { name: 'LLAMA 3 2 1B Preview', value: 'LLAMA_3_2_1B_PREVIEW' }, + { name: 'LLAMA 3 2 3B Preview', value: 'LLAMA_3_2_3B_PREVIEW' }, + { name: 'Mixtral 8x7b 32768', value: 'MIXTRAL_8X7B_32768' }, +] + +export const GEMINI_CHOICES = [ + { name: 'Gemini 1.5 Flash', value: 'GEMINI_1_5_FLASH' }, + { name: 'Gemini 1.5 Pro', value: 'GEMINI_1_5_PRO' }, +] + +export const PROMPT_CHOICES = [ + { name: 'Titles', value: 'titles' }, + { name: 'Summary', value: 'summary' }, + { name: 'Short Chapters', value: 'shortChapters' }, + { name: 'Medium Chapters', value: 'mediumChapters' }, + { name: 'Long Chapters', value: 'longChapters' }, + { name: 'Key Takeaways', value: 'takeaways' }, + { name: 'Questions', value: 'questions' }, +] + /** * Configure XML parser for RSS feed processing * Handles attributes without prefixes and allows boolean values diff --git a/src/interactive.ts b/src/interactive.ts index 6d4e1ab..b0474f4 100644 --- a/src/interactive.ts +++ b/src/interactive.ts @@ -2,110 +2,9 @@ import inquirer from 'inquirer' import type { ProcessingOptions, InquirerAnswers, WhisperModelType } from './types.js' -import { l } from './globals.js' - -const TRANSCRIPT_CHOICES = [ - { name: 'Whisper.cpp', value: 'whisper' }, - { name: 'Whisper.cpp (Docker)', value: 'whisperDocker' }, - { name: 'Whisper Python', value: 'whisperPython' }, - { name: 'Whisper Diarization', value: 'whisperDiarization' }, - { name: 'Deepgram', value: 'deepgram' }, - { name: 'AssemblyAI', value: 'assembly' }, -] - -const WHISPER_MODEL_CHOICES = [ - 'tiny', - 'tiny.en', - 'base', - 'base.en', - 'small', - 'small.en', - 'medium', - 'medium.en', - 'large-v1', - 'large-v2', - 'turbo', -] - -const LLM_CHOICES = [ - 'ollama', - 'chatgpt', - 'claude', - 'cohere', - 'mistral', - 'fireworks', - 'together', - 'groq', - 'gemini', -] - -const OLLAMA_CHOICES = [ - { name: 'LLAMA 3 2 1B', value: 'LLAMA_3_2_1B' }, - { name: 'LLAMA 3 2 3B', value: 'LLAMA_3_2_3B' }, - { name: 'GEMMA 2 2B', value: 'GEMMA_2_2B' }, - { name: 'PHI 3 5', value: 'PHI_3_5' }, - { name: 'QWEN 2 5 1B', value: 'QWEN_2_5_1B' }, - { name: 'QWEN 2 5 3B', value: 'QWEN_2_5_3B' }, -] - -const CHATGPT_CHOICES = [ - { name: 'GPT 4 o MINI', value: 'GPT_4o_MINI' }, - { name: 'GPT 4 o', value: 'GPT_4o' }, - { name: 'GPT 4 TURBO', value: 'GPT_4_TURBO' }, - { name: 'GPT 4', value: 'GPT_4' }, -] - -const CLAUDE_CHOICES = [ - { name: 'Claude 3.5 Sonnet', value: 'CLAUDE_3_5_SONNET' }, - { name: 'Claude 3 Opus', value: 'CLAUDE_3_OPUS' }, - { name: 'Claude 3 Sonnet', value: 'CLAUDE_3_SONNET' }, - { name: 'Claude 3 Haiku', value: 'CLAUDE_3_HAIKU' }, -] - -const COHERE_CHOICES = [ - { name: 'Command R', value: 'COMMAND_R' }, - { name: 'Command R Plus', value: 'COMMAND_R_PLUS' }, -] - -const MISTRAL_CHOICES = [ - { name: 'Mixtral 8x7b', value: 'MIXTRAL_8x7b' }, - { name: 'Mixtral 8x22b', value: 'MIXTRAL_8x22b' }, - { name: 'Mistral Large', value: 'MISTRAL_LARGE' }, - { name: 'Mistral Nemo', value: 'MISTRAL_NEMO' }, -] - -const FIREWORKS_CHOICES = [ - { name: 'LLAMA 3 1 405B', value: 'LLAMA_3_1_405B' }, - { name: 'LLAMA 3 1 70B', value: 'LLAMA_3_1_70B' }, - { name: 'LLAMA 3 1 8B', value: 'LLAMA_3_1_8B' }, - { name: 'LLAMA 3 2 3B', value: 'LLAMA_3_2_3B' }, - { name: 'LLAMA 3 2 1B', value: 'LLAMA_3_2_1B' }, - { name: 'QWEN 2 5 72B', value: 'QWEN_2_5_72B' }, -] - -const TOGETHER_CHOICES = [ - { name: 'LLAMA 3 2 3B', value: 'LLAMA_3_2_3B' }, - { name: 'LLAMA 3 1 405B', value: 'LLAMA_3_1_405B' }, - { name: 'LLAMA 3 1 70B', value: 'LLAMA_3_1_70B' }, - { name: 'LLAMA 3 1 8B', value: 'LLAMA_3_1_8B' }, - { name: 'Gemma 2 27B', value: 'GEMMA_2_27B' }, - { name: 'Gemma 2 9B', value: 'GEMMA_2_9B' }, - { name: 'QWEN 2 5 72B', value: 'QWEN_2_5_72B' }, - { name: 'QWEN 2 5 7B', value: 'QWEN_2_5_7B' }, -] - -const GROQ_CHOICES = [ - { name: 'LLAMA 3 1 70B Versatile', value: 'LLAMA_3_1_70B_VERSATILE' }, - { name: 'LLAMA 3 1 8B Instant', value: 'LLAMA_3_1_8B_INSTANT' }, - { name: 'LLAMA 3 2 1B Preview', value: 'LLAMA_3_2_1B_PREVIEW' }, - { name: 'LLAMA 3 2 3B Preview', value: 'LLAMA_3_2_3B_PREVIEW' }, - { name: 'Mixtral 8x7b 32768', value: 'MIXTRAL_8X7B_32768' }, -] - -const GEMINI_CHOICES = [ - { name: 'Gemini 1.5 Flash', value: 'GEMINI_1_5_FLASH' }, - { name: 'Gemini 1.5 Pro', value: 'GEMINI_1_5_PRO' }, -] +import { + l, PROCESS_CHOICES, TRANSCRIPT_CHOICES, WHISPER_MODEL_CHOICES, WHISPER_LIBRARY_CHOICES, LLM_SERVICE_CHOICES, LLM_CHOICES, OLLAMA_CHOICES, CHATGPT_CHOICES, CLAUDE_CHOICES, COHERE_CHOICES, MISTRAL_CHOICES, FIREWORKS_CHOICES, TOGETHER_CHOICES, GROQ_CHOICES, GEMINI_CHOICES, PROMPT_CHOICES +} from './globals.js' /** * Prompts the user for input if interactive mode is selected. @@ -125,14 +24,7 @@ export async function handleInteractivePrompt( type: 'list', name: 'action', message: 'What would you like to process?', - choices: [ - { name: 'Single YouTube Video', value: 'video' }, - { name: 'YouTube Playlist', value: 'playlist' }, - { name: 'YouTube Channel', value: 'channel' }, - { name: 'List of URLs from File', value: 'urls' }, - { name: 'Local Audio/Video File', value: 'file' }, - { name: 'Podcast RSS Feed', value: 'rss' }, - ], + choices: PROCESS_CHOICES, }, // Input prompts for different content sources { @@ -237,18 +129,7 @@ export async function handleInteractivePrompt( type: 'list', name: 'llmServices', message: 'Select the Language Model (LLM) you want to use:', - choices: [ - { name: 'Skip LLM Processing', value: null }, - { name: 'Ollama (local inference)', value: 'ollama' }, - { name: 'OpenAI ChatGPT', value: 'chatgpt' }, - { name: 'Anthropic Claude', value: 'claude' }, - { name: 'Google Gemini', value: 'gemini' }, - { name: 'Cohere', value: 'cohere' }, - { name: 'Mistral', value: 'mistral' }, - { name: 'Fireworks AI', value: 'fireworks' }, - { name: 'Together AI', value: 'together' }, - { name: 'Groq', value: 'groq' }, - ], + choices: LLM_SERVICE_CHOICES, }, // Model selection based on chosen LLM service { @@ -314,15 +195,7 @@ export async function handleInteractivePrompt( type: 'checkbox', name: 'prompt', message: 'Select the prompt sections to include:', - choices: [ - { name: 'Titles', value: 'titles' }, - { name: 'Summary', value: 'summary' }, - { name: 'Short Chapters', value: 'shortChapters' }, - { name: 'Medium Chapters', value: 'mediumChapters' }, - { name: 'Long Chapters', value: 'longChapters' }, - { name: 'Key Takeaways', value: 'takeaways' }, - { name: 'Questions', value: 'questions' }, - ], + choices: PROMPT_CHOICES, default: ['summary', 'longChapters'], }, { @@ -351,7 +224,7 @@ export async function handleInteractivePrompt( // Configure transcription service options based on user selection if (answers.transcriptServices) { if ( - ['whisper', 'whisperDocker', 'whisperPython', 'whisperDiarization'].includes( + WHISPER_LIBRARY_CHOICES.includes( answers.transcriptServices ) ) {