From da9d493ed141d65abb573458b24c8fa662f87b8b Mon Sep 17 00:00:00 2001 From: Anthony Campolo <12433465+ajcwebdev@users.noreply.github.com> Date: Sun, 6 Oct 2024 21:23:31 -0500 Subject: [PATCH] refactor tests, logging, and global exports --- .gitignore | 4 +- package.json | 3 +- src/autoshow.ts | 78 ++++++++------- src/commands/processFile.ts | 30 ++---- src/commands/processPlaylist.ts | 19 ++-- src/commands/processRSS.ts | 62 +++++------- src/commands/processURLs.ts | 25 ++--- src/commands/processVideo.ts | 39 +++----- src/llms/llama.ts | 19 ++-- src/llms/ollama.ts | 9 +- src/models.ts | 6 +- src/transcription/whisper.ts | 11 +-- src/transcription/whisperDocker.ts | 4 + src/types.ts | 18 ++++ src/utils/cleanUpFiles.ts | 2 +- src/utils/generateMarkdown.ts | 24 ++--- src/utils/runLLM.ts | 2 +- test/all.test.js | 148 +++++++++++++---------------- test/local.test.js | 48 +++++----- 19 files changed, 249 insertions(+), 302 deletions(-) diff --git a/.gitignore b/.gitignore index 8e434e86..ccb17886 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,6 @@ build deno.lock out types -dist \ No newline at end of file +dist +NEW.md +TODO.md \ No newline at end of file diff --git a/package.json b/package.json index 9d02fd01..b3a05cf0 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "serve": "npm run build && node --env-file=.env --no-warnings --watch packages/server/index.js", "fetch-local": "npm run build && node --env-file=.env --no-warnings packages/server/tests/fetch-local.js", "fetch-all": "npm run build && node --env-file=.env --no-warnings packages/server/tests/fetch-all.js", + "t": "npm run build && node --test test/local.test.js", "test-local": "npm run build && node --test test/local.test.js", "test-all": "npm run build && node --test test/all.test.js" }, @@ -53,7 +54,7 @@ "ffmpeg-static": "^5.2.0", "file-type": "^19.4.1", "inquirer": "^10.2.2", - "node-llama-cpp": "^3.0.0-beta.44", + "node-llama-cpp": "^3.1.0", "ollama": "^0.5.9", "openai": "^4.55.7", "typescript": "^5.6.2" diff --git a/src/autoshow.ts b/src/autoshow.ts index 8750b00e..c4ae8477 100644 --- a/src/autoshow.ts +++ b/src/autoshow.ts @@ -20,7 +20,7 @@ import { processURLs } from './commands/processURLs.js' import { processFile } from './commands/processFile.js' import { processRSS } from './commands/processRSS.js' import { argv, exit } from 'node:process' -import { log, opts } from './models.js' +import { log, opts, final, ACTION_OPTIONS, LLM_OPTIONS, TRANSCRIPT_OPTIONS } from './models.js' import type { ProcessingOptions, HandlerFunction, LLMServices, TranscriptServices } from './types.js' // Initialize the command-line interface @@ -72,6 +72,22 @@ Report Issues: https://github.com/ajcwebdev/autoshow/issues ` ) +/** + * Helper function to validate that only one option from a list is provided. + * @param {string[]} optionKeys - The list of option keys to check. + * @param {ProcessingOptions} options - The options object. + * @param {string} errorMessage - The prefix of the error message. + * @returns {string | undefined} - The selected option or undefined. + */ +function getSingleOption(optionKeys: string[], options: ProcessingOptions, errorMessage: string): string | undefined { + const selectedOptions = optionKeys.filter((opt) => options[opt as keyof ProcessingOptions]) + if (selectedOptions.length > 1) { + console.error(`Error: Multiple ${errorMessage} provided (${selectedOptions.join(', ')}). Please specify only one.`) + exit(1) + } + return selectedOptions[0] as string | undefined +} + /** * Main action for the program. * @param {ProcessingOptions} options - The command-line options provided by the user. @@ -90,12 +106,8 @@ program.action(async (options: ProcessingOptions) => { rss: processRSS, } - const ACTION_OPTIONS = ['video', 'playlist', 'urls', 'file', 'rss'] - const LLM_OPTIONS = ['chatgpt', 'claude', 'cohere', 'mistral', 'octo', 'llama', 'ollama', 'gemini'] - const TRANSCRIPT_OPTIONS = ['whisper', 'whisperDocker', 'deepgram', 'assembly'] - - const { video, playlist, urls, file, rss, interactive } = options - const noActionProvided = [video, playlist, urls, file, rss].every((opt) => !opt) + const { interactive } = options + const noActionProvided = ACTION_OPTIONS.every((opt) => !options[opt as keyof ProcessingOptions]) if (interactive || noActionProvided) { options = await handleInteractivePrompt(options) @@ -106,25 +118,16 @@ program.action(async (options: ProcessingOptions) => { options.item = [options.item] } - const actionsProvided = ACTION_OPTIONS.filter((opt) => options[opt as keyof ProcessingOptions]) - if (actionsProvided.length > 1) { - console.error(`Error: Multiple input options provided (${actionsProvided.join(', ')}). Please specify only one input option.`) - exit(1) - } - - const selectedLLMs = LLM_OPTIONS.filter((opt) => options[opt as keyof ProcessingOptions]) - if (selectedLLMs.length > 1) { - console.error(`Error: Multiple LLM options provided (${selectedLLMs.join(', ')}). Please specify only one LLM option.`) - exit(1) - } - const llmServices = selectedLLMs[0] as LLMServices | undefined + // Validate and retrieve single action option + const action = getSingleOption(ACTION_OPTIONS, options, 'input option') + + // Validate and retrieve single LLM option + const llmKey = getSingleOption(LLM_OPTIONS, options, 'LLM option') + const llmServices = llmKey as LLMServices | undefined - const selectedTranscripts = TRANSCRIPT_OPTIONS.filter((opt) => options[opt as keyof ProcessingOptions]) - if (selectedTranscripts.length > 1) { - console.error(`Error: Multiple transcription options provided (${selectedTranscripts.join(', ')}). Please specify only one transcription option.`) - exit(1) - } - const transcriptServices = selectedTranscripts[0] as TranscriptServices | undefined + // Validate and retrieve single transcription option + const transcriptKey = getSingleOption(TRANSCRIPT_OPTIONS, options, 'transcription option') + const transcriptServices: TranscriptServices | undefined = transcriptKey as TranscriptServices | undefined // Set default transcription service if not provided const finalTranscriptServices: TranscriptServices = transcriptServices || 'whisper' @@ -134,16 +137,21 @@ program.action(async (options: ProcessingOptions) => { options.whisper = 'base' } - // Execute the appropriate handler based on the action - for (const [key, handler] of Object.entries(PROCESS_HANDLERS)) { - if (options[key as keyof ProcessingOptions]) { - try { - await handler(options, options[key as keyof ProcessingOptions] as string, llmServices, finalTranscriptServices) - exit(0) - } catch (error) { - console.error(`Error processing ${key}:`, (error as Error).message) - exit(1) - } + if (action) { + try { + await PROCESS_HANDLERS[action]( + options, + options[action as keyof ProcessingOptions] as string, + llmServices, + finalTranscriptServices + ) + log(final(`\n==================================================`)) + log(final(` ${action} Processing Completed Successfully.`)) + log(final(`==================================================\n`)) + exit(0) + } catch (error) { + console.error(`Error processing ${action}:`, (error as Error).message) + exit(1) } } }) diff --git a/src/commands/processFile.ts b/src/commands/processFile.ts index 3827ad0e..6dc4c77b 100644 --- a/src/commands/processFile.ts +++ b/src/commands/processFile.ts @@ -5,7 +5,7 @@ import { downloadAudio } from '../utils/downloadAudio.js' import { runTranscription } from '../utils/runTranscription.js' import { runLLM } from '../utils/runLLM.js' import { cleanUpFiles } from '../utils/cleanUpFiles.js' -import { log, final } from '../models.js' +import { log, opts, wait } from '../models.js' import type { LLMServices, TranscriptServices, ProcessingOptions } from '../types.js' /** @@ -22,30 +22,16 @@ export async function processFile( llmServices?: LLMServices, transcriptServices?: TranscriptServices ): Promise { - // log(`Options received in processFile:\n`) - // log(options) - // log(`filePath:`, filePath) - // log(`llmServices:`, llmServices) - // log(`transcriptServices:`, transcriptServices) + log(opts('Parameters passed to processFile:\n')) + log(wait(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}\n`)) try { - // Generate markdown for the file - const { frontMatter, finalPath, filename } = await generateMarkdown(options, filePath) - - // Convert the audio or video file to the required format - await downloadAudio(options, filePath, filename) - - // Run transcription on the file - await runTranscription(options, finalPath, frontMatter, transcriptServices) - - // Process the transcript with the selected Language Model - await runLLM(options, finalPath, frontMatter, llmServices) - - // Clean up temporary files if the noCleanUp option is not set - if (!options.noCleanUp) { + const { frontMatter, finalPath, filename } = await generateMarkdown(options, filePath) // Generate markdown for the file + await downloadAudio(options, filePath, filename) // Convert the audio or video file to the required format + await runTranscription(options, finalPath, frontMatter, transcriptServices) // Run transcription on the file + await runLLM(options, finalPath, frontMatter, llmServices) // Process the transcript with the selected Language Model + if (!options.noCleanUp) { // Clean up temporary files if the noCleanUp option is not set await cleanUpFiles(finalPath) } - - log(final('\nLocal file processing completed successfully.\n')) } catch (error) { console.error(`Error processing file: ${(error as Error).message}`) process.exit(1) // Exit with an error code diff --git a/src/commands/processPlaylist.ts b/src/commands/processPlaylist.ts index 7448e99b..807c6f60 100644 --- a/src/commands/processPlaylist.ts +++ b/src/commands/processPlaylist.ts @@ -6,7 +6,7 @@ import { promisify } from 'node:util' import { processVideo } from './processVideo.js' import { extractVideoMetadata } from '../utils/extractVideoMetadata.js' import { checkDependencies } from '../utils/checkDependencies.js' -import { log, final, wait } from '../models.js' +import { log, opts, success, wait } from '../models.js' import type { LLMServices, TranscriptServices, ProcessingOptions } from '../types.js' const execFilePromise = promisify(execFile) @@ -24,11 +24,8 @@ export async function processPlaylist( llmServices?: LLMServices, transcriptServices?: TranscriptServices ): Promise { - // log(`Options received in processPlaylist:\n`) - // log(options) - // log(`playlistUrl:`, playlistUrl) - // log(`llmServices:`, llmServices) - // log(`transcriptServices:`, transcriptServices) + log(opts('Parameters passed to processPlaylist:\n')) + log(wait(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}`)) try { // Check for required dependencies await checkDependencies(['yt-dlp']) @@ -52,7 +49,7 @@ export async function processPlaylist( process.exit(1) // Exit with an error code } - log(wait(` Found ${urls.length} videos in the playlist...`)) + log(opts(`\nFound ${urls.length} videos in the playlist...`)) // Extract metadata for all videos const metadataPromises = urls.map(extractVideoMetadata) @@ -64,13 +61,15 @@ export async function processPlaylist( const jsonContent = JSON.stringify(validMetadata, null, 2) const jsonFilePath = 'content/playlist_info.json' await writeFile(jsonFilePath, jsonContent) - log(wait(`Playlist information saved to: ${jsonFilePath}`)) + log(success(`Playlist information saved to: ${jsonFilePath}`)) return } // Process each video in the playlist for (const [index, url] of urls.entries()) { - log(wait(`\n Processing video ${index + 1}/${urls.length}:\n - ${url}\n`)) + log(opts(`\n==============================================================`)) + log(opts(` Processing video ${index + 1}/${urls.length}: ${url}`)) + log(opts(`==============================================================\n`)) try { await processVideo(options, url, llmServices, transcriptServices) } catch (error) { @@ -78,8 +77,6 @@ export async function processPlaylist( // Continue processing the next video } } - - log(final('\nPlaylist processing completed successfully.\n')) } catch (error) { console.error(`Error processing playlist: ${(error as Error).message}`) process.exit(1) // Exit with an error code diff --git a/src/commands/processRSS.ts b/src/commands/processRSS.ts index 82c307ba..d1a49dc6 100644 --- a/src/commands/processRSS.ts +++ b/src/commands/processRSS.ts @@ -7,7 +7,7 @@ import { downloadAudio } from '../utils/downloadAudio.js' import { runTranscription } from '../utils/runTranscription.js' import { runLLM } from '../utils/runLLM.js' import { cleanUpFiles } from '../utils/cleanUpFiles.js' -import { log, final, wait } from '../models.js' +import { log, final, wait, opts } from '../models.js' import type { LLMServices, TranscriptServices, ProcessingOptions, RSSItem } from '../types.js' @@ -32,30 +32,16 @@ async function processItem( llmServices?: LLMServices, transcriptServices?: TranscriptServices ): Promise { - // log(`Options received in processItem:\n`) - // log(options) - // log(`item\n\n`, item) - // log(`llmServices:`, llmServices) - // log(`transcriptServices:`, transcriptServices) + log(opts('Parameters passed to processItem:\n')) + log(wait(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}\n`)) try { - // Generate markdown for the item - const { frontMatter, finalPath, filename } = await generateMarkdown(options, item) - - // Download audio - await downloadAudio(options, item.showLink, filename) - - // Run transcription - await runTranscription(options, finalPath, frontMatter, transcriptServices) - - // Process with Language Model - await runLLM(options, finalPath, frontMatter, llmServices) - - // Clean up temporary files if necessary - if (!options.noCleanUp) { + const { frontMatter, finalPath, filename } = await generateMarkdown(options, item) // Generate markdown for the item + await downloadAudio(options, item.showLink, filename) // Download audio + await runTranscription(options, finalPath, frontMatter, transcriptServices) // Run transcription + await runLLM(options, finalPath, frontMatter, llmServices) // Process with Language Model + if (!options.noCleanUp) { // Clean up temporary files if necessary await cleanUpFiles(finalPath) } - - log(final(`\nItem processing completed successfully: ${item.title}`)) } catch (error) { console.error(`Error processing item ${item.title}: ${(error as Error).message}`) // Continue processing the next item @@ -76,11 +62,8 @@ export async function processRSS( llmServices?: LLMServices, transcriptServices?: TranscriptServices ): Promise { - log(`Options received in processRSS:\n`) - log(options) - log(`rssUrl:`, rssUrl) - log(`llmServices:`, llmServices) - log(`transcriptServices:`, transcriptServices) + log(opts('Parameters passed to processRSS:\n')) + log(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}`) try { // Validate that --last is a positive integer if provided if (options.last !== undefined) { @@ -115,11 +98,11 @@ export async function processRSS( if (options.item && options.item.length > 0) { // If specific items are provided, list them log(wait('\nProcessing specific items:')) - options.item.forEach((url) => log(` - ${url}`)) + options.item.forEach((url) => log(wait(` - ${url}`))) } else if (options.last) { - console.log(`\nProcessing the last ${options.last} items`) - } else { - console.log(` - Skipping first ${options.skip || 0} items`) + log(wait(`\nProcessing the last ${options.last} items`)) + } else if (options.skip) { + log(wait(` - Skipping first ${options.skip || 0} items`)) } // Fetch the RSS feed with a timeout @@ -213,28 +196,31 @@ export async function processRSS( process.exit(1) // Exit with an error code } itemsToProcess = matchedItems - log(wait(` Found ${items.length} items in the RSS feed.`)) + log(wait(`\n - Found ${items.length} items in the RSS feed.`)) log(wait(` - Processing ${itemsToProcess.length} specified items.`)) } else if (options.last) { // Process the most recent N items itemsToProcess = items.slice(0, options.last) - log(wait(` Found ${items.length} items in the RSS feed.`)) + log(wait(`\n - Found ${items.length} items in the RSS feed.`)) log(wait(` - Processing the last ${options.last} items.`)) } else { // Sort items based on the specified order and apply skip const sortedItems = options.order === 'oldest' ? items.slice().reverse() : items itemsToProcess = sortedItems.slice(options.skip || 0) - log(wait(` Found ${items.length} items in the RSS feed.`)) - log(wait(` - Processing ${itemsToProcess.length} items after skipping ${options.skip || 0}.\n`)) + log(wait(`\n - Found ${items.length} item(s) in the RSS feed.`)) + log(wait(` - Processing ${itemsToProcess.length} item(s) after skipping ${options.skip || 0}.\n`)) } // Process each item in the feed for (const [index, item] of itemsToProcess.entries()) { - log(wait(` Processing item ${index + 1}/${itemsToProcess.length}:\n - ${item.title}\n`)) + log(opts(`\n==============================================================`)) + log(opts(` Item ${index + 1}/${itemsToProcess.length} processing: ${item.title}`)) + log(opts(`==============================================================\n`)) await processItem(options, item, llmServices, transcriptServices) + log(final(`\n==============================================================`)) + log(final(` ${index + 1}/${itemsToProcess.length} item processing completed successfully`)) + log(final(`==============================================================\n`)) } - - log(final('\nRSS feed processing completed successfully.\n')) } catch (error) { console.error(`Error processing RSS feed: ${(error as Error).message}`) process.exit(1) // Exit with an error code diff --git a/src/commands/processURLs.ts b/src/commands/processURLs.ts index 68c86a7a..33a80e3b 100644 --- a/src/commands/processURLs.ts +++ b/src/commands/processURLs.ts @@ -1,11 +1,10 @@ // src/commands/processURLs.ts import { readFile, writeFile } from 'node:fs/promises' -import { resolve } from 'node:path' import { processVideo } from './processVideo.js' import { extractVideoMetadata } from '../utils/extractVideoMetadata.js' import { checkDependencies } from '../utils/checkDependencies.js' -import { log, final, wait } from '../models.js' +import { log, wait, opts } from '../models.js' import type { LLMServices, TranscriptServices, ProcessingOptions } from '../types.js' /** @@ -21,29 +20,23 @@ export async function processURLs( llmServices?: LLMServices, transcriptServices?: TranscriptServices ): Promise { - // log(`Options received in processURLs:\n`) - // log(options) - // log(`filePath:`, filePath) - // log(`llmServices:`, llmServices) - // log(`transcriptServices:`, transcriptServices) + log(opts('Parameters passed to processURLs:\n')) + log(wait(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}\n`)) try { // Check for required dependencies await checkDependencies(['yt-dlp']) - const absolutePath = resolve(filePath) - // Read and parse the content of the file into an array of URLs - const content = await readFile(absolutePath, 'utf8') + const content = await readFile(filePath, 'utf8') const urls = content.split('\n') .map(line => line.trim()) .filter(line => line && !line.startsWith('#')) if (urls.length === 0) { console.error('Error: No URLs found in the file.') - process.exit(1) // Exit with an error code + process.exit(1) } - - log(wait(`\n Found ${urls.length} URLs in the file...`)) + log(opts(`\n=== Found ${urls.length} URLs in the file... ===`)) // Extract metadata for all videos const metadataPromises = urls.map(extractVideoMetadata) @@ -61,7 +54,9 @@ export async function processURLs( // Process each URL for (const [index, url] of urls.entries()) { - log(wait(`\n Processing URL ${index + 1}/${urls.length}:\n - ${url}\n`)) + log(opts(`\n============================================================`)) + log(opts(` Processing URL ${index + 1}/${urls.length}: ${url}`)) + log(opts(`============================================================\n`)) try { await processVideo(options, url, llmServices, transcriptServices) } catch (error) { @@ -69,8 +64,6 @@ export async function processURLs( // Continue processing the next URL } } - - log(final('\nURL file processing completed successfully.\n')) } catch (error) { console.error(`Error reading or processing file ${filePath}: ${(error as Error).message}`) process.exit(1) // Exit with an error code diff --git a/src/commands/processVideo.ts b/src/commands/processVideo.ts index 8c873b6d..f087e311 100644 --- a/src/commands/processVideo.ts +++ b/src/commands/processVideo.ts @@ -6,7 +6,7 @@ import { downloadAudio } from '../utils/downloadAudio.js' import { runTranscription } from '../utils/runTranscription.js' import { runLLM } from '../utils/runLLM.js' import { cleanUpFiles } from '../utils/cleanUpFiles.js' -import { log, final } from '../models.js' +import { log, opts, wait } from '../models.js' import type { LLMServices, TranscriptServices, ProcessingOptions } from '../types.js' /** @@ -23,36 +23,19 @@ export async function processVideo( llmServices?: LLMServices, transcriptServices?: TranscriptServices ): Promise { - // log(`Options received in processVideo:\n`) - // log(options) - // log(`url:`, url) - // log(`llmServices:`, llmServices) - // log(`transcriptServices:`, transcriptServices) + log(opts('Parameters passed to processVideo:\n')) + log(wait(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}\n`)) try { - // Check for required dependencies - await checkDependencies(['yt-dlp']) - - // Generate markdown with video metadata - const { frontMatter, finalPath, filename } = await generateMarkdown(options, url) - - // Download audio from the video - await downloadAudio(options, url, filename) - - // Run transcription on the audio - await runTranscription(options, finalPath, frontMatter, transcriptServices) - - // Process transcript with an LLM if llmServices is defined, concatenate prompt and transcript if undefined - await runLLM(options, finalPath, frontMatter, llmServices) - - // Clean up temporary files if the noCleanUp option is not set - if (!options.noCleanUp) { + await checkDependencies(['yt-dlp']) // Check for required dependencies. + const { frontMatter, finalPath, filename } = await generateMarkdown(options, url) // Generate markdown with video metadata. + await downloadAudio(options, url, filename) // Download audio from the video. + await runTranscription(options, finalPath, frontMatter, transcriptServices) // Run transcription on the audio. + await runLLM(options, finalPath, frontMatter, llmServices) // If llmServices is set, process with LLM. If llmServices is undefined, bypass LLM processing. + if (!options.noCleanUp) { // Clean up temporary files if the noCleanUp option is not set. await cleanUpFiles(finalPath) } - - log(final('\nVideo processing completed successfully.\n')) } catch (error) { - // Log any errors that occur during video processing - console.error('Error processing video:', (error as Error).message) - throw error // Re-throw to be handled by caller + console.error('Error processing video:', (error as Error).message) // Log any errors that occur during video processing + throw error // Re-throw to be handled by caller } } \ No newline at end of file diff --git a/src/llms/llama.ts b/src/llms/llama.ts index 9a48d453..56cee8cf 100644 --- a/src/llms/llama.ts +++ b/src/llms/llama.ts @@ -6,7 +6,7 @@ import { existsSync } from 'node:fs' import { exec } from 'node:child_process' import { promisify } from 'node:util' import { LLAMA_MODELS } from '../models.js' -import { log, wait } from '../models.js' +import { log, success, wait } from '../models.js' import type { LlamaModelType, LLMFunction } from '../types.js' @@ -20,11 +20,15 @@ const execAsync = promisify(exec) * @returns A Promise that resolves when the processing is complete. * @throws {Error} - If an error occurs during processing. */ -export const callLlama: LLMFunction = async (promptAndTranscript: string, tempPath: string, model?: string) => { +export const callLlama: LLMFunction = async ( + promptAndTranscript: string, + tempPath: string, + model?: string +) => { try { // Get the model object from LLAMA_MODELS using the provided model name or default to GEMMA_2_2B const selectedModel = LLAMA_MODELS[model as LlamaModelType] || LLAMA_MODELS.GEMMA_2_2B - log(wait(` - Model selected: ${selectedModel.filename}`)) + log(wait(` - filename: ${selectedModel.filename}\n - url: ${selectedModel.url}\n`)) // If no valid model is found, throw an error if (!selectedModel) { @@ -36,7 +40,7 @@ export const callLlama: LLMFunction = async (promptAndTranscript: string, tempPa // Check if the model file already exists, if not, download it if (!existsSync(modelPath)) { - log(wait(`\nDownloading ${selectedModel.filename}...`)) + log(success(`\nDownloading ${selectedModel.filename}...`)) try { // Create the directory for storing models if it doesn't exist @@ -45,16 +49,15 @@ export const callLlama: LLMFunction = async (promptAndTranscript: string, tempPa // Download the model using curl const { stderr } = await execAsync(`curl -L ${selectedModel.url} -o ${modelPath}`) - // If there's any stderr output, log it - if (stderr) log(stderr) - log('Download completed') + // If there's any stderr output, log completed + if (stderr) log(success('Download completed')) } catch (err) { // If an error occurs during download, log it and throw a new error console.error(`Download failed: ${err instanceof Error ? err.message : String(err)}`) throw new Error('Failed to download the model') } } else { - log(wait(` - Model path: ${modelPath}`)) + log(wait(` modelPath found:\n - ${modelPath}`)) } // Initialize Llama and load the local model diff --git a/src/llms/ollama.ts b/src/llms/ollama.ts index 3a02c594..74955e6e 100644 --- a/src/llms/ollama.ts +++ b/src/llms/ollama.ts @@ -5,14 +5,7 @@ import { env } from 'node:process' import { OLLAMA_MODELS } from '../models.js' import { log, wait } from '../models.js' -import type { LLMFunction, OllamaModelType } from '../types.js' - -// Define the expected structure of the response from Ollama API -interface OllamaResponse { - message: { - content: string - } -} +import type { LLMFunction, OllamaModelType, OllamaResponse } from '../types.js' /** * Main function to call the Llama model using the Ollama REST API. diff --git a/src/models.ts b/src/models.ts index 1bd7b7c1..5f637fb2 100644 --- a/src/models.ts +++ b/src/models.ts @@ -8,11 +8,15 @@ export const step: ChalkInstance = chalk.bold.underline export const dim: ChalkInstance = chalk.dim export const success: ChalkInstance = chalk.bold.blue export const opts: ChalkInstance = chalk.magentaBright.bold -export const wait: ChalkInstance = chalk.cyan.dim +export const wait: ChalkInstance = chalk.bold.cyan export const final: ChalkInstance = chalk.bold.italic export const log: typeof console.log = console.log +export const ACTION_OPTIONS = ['video', 'playlist', 'urls', 'file', 'rss'] +export const LLM_OPTIONS = ['chatgpt', 'claude', 'cohere', 'mistral', 'octo', 'llama', 'ollama', 'gemini'] +export const TRANSCRIPT_OPTIONS = ['whisper', 'whisperDocker', 'deepgram', 'assembly'] + /** * Define available Whisper models * @type {Record} diff --git a/src/transcription/whisper.ts b/src/transcription/whisper.ts index 5aa56634..2e25dcfe 100644 --- a/src/transcription/whisper.ts +++ b/src/transcription/whisper.ts @@ -5,7 +5,7 @@ import { exec } from 'node:child_process' import { promisify } from 'node:util' import { existsSync } from 'node:fs' import { WHISPER_MODELS } from '../models.js' -import { log, success, wait } from '../models.js' +import { log, wait } from '../models.js' import type { ProcessingOptions } from '../types.js' const execPromise = promisify(exec) @@ -30,8 +30,7 @@ export async function callWhisper(options: ProcessingOptions, finalPath: string) // Get the model ggml file name const modelGGMLName = WHISPER_MODELS[whisperModel] - log(wait(` - whisperModel: ${whisperModel}`)) - log(wait(` - modelGGMLName: ${modelGGMLName}`)) + log(wait(`\n - whisperModel: ${whisperModel}\n - modelGGMLName: ${modelGGMLName}`)) // Setup Whisper if (!existsSync('./whisper.cpp')) { @@ -42,14 +41,14 @@ export async function callWhisper(options: ProcessingOptions, finalPath: string) // Ensure model is downloaded if (!existsSync(`./whisper.cpp/models/ggml-${whisperModel}.bin`)) { - log(wait(` - Model not found, downloading: ${whisperModel}...\n`)) + log(wait(` Model not found, downloading...\n - ${whisperModel}\n`)) await execPromise(`bash ./whisper.cpp/models/download-ggml-model.sh ${whisperModel}`) - log(success(' Model download completed.\n')) + log(wait(' Model download completed, running transcription...\n')) } // Run transcription await execPromise(`./whisper.cpp/main -m "whisper.cpp/models/${modelGGMLName}" -f "${finalPath}.wav" -of "${finalPath}" --output-lrc`) - log(wait(`\n Transcript LRC file successfully completed...\n - ${finalPath}.lrc\n`)) + log(wait(`\n Transcript LRC file successfully completed...\n - ${finalPath}.lrc`)) // Read the generated LRC file const lrcContent = await readFile(`${finalPath}.lrc`, 'utf8') diff --git a/src/transcription/whisperDocker.ts b/src/transcription/whisperDocker.ts index 295ea05d..22b25669 100644 --- a/src/transcription/whisperDocker.ts +++ b/src/transcription/whisperDocker.ts @@ -31,6 +31,7 @@ export async function callWhisperDocker(options: ProcessingOptions, finalPath: s const modelGGMLName = WHISPER_MODELS[whisperModel] const CONTAINER_NAME = 'autoshow-whisper-1' const modelPathContainer = `/app/models/${modelGGMLName}` + log(wait(` - whisperModel: ${whisperModel}`)) log(wait(` - modelGGMLName: ${modelGGMLName}`)) log(wait(` - CONTAINER_NAME: ${CONTAINER_NAME}`)) @@ -52,14 +53,17 @@ export async function callWhisperDocker(options: ProcessingOptions, finalPath: s // Process transcript const lrcContent = await readFile(`${finalPath}.lrc`, 'utf8') + // Process and format the LRC content const txtContent = lrcContent.split('\n') .filter(line => !line.startsWith('[by:whisper.cpp]')) .map(line => line.replace(/\[(\d{2,3}):(\d{2})\.(\d{2})\]/g, (_, p1, p2) => `[${p1}:${p2}]`)) .join('\n') + // Write the formatted content to a text file await writeFile(`${finalPath}.txt`, txtContent) log(wait(` Transcript transformation successfully completed...\n - ${finalPath}.txt\n`)) + // Return the processed content return txtContent } catch (error) { console.error('Error in callWhisperDocker:', error) diff --git a/src/types.ts b/src/types.ts index c6ca1fa9..d1019830 100644 --- a/src/types.ts +++ b/src/types.ts @@ -325,6 +325,24 @@ export type LlamaModelType = 'QWEN_2_5_3B' | 'PHI_3_5' | 'LLAMA_3_2_1B' | 'GEMMA /** Define local model with Ollama. */ export type OllamaModelType = 'LLAMA_3_2_1B' | 'LLAMA_3_2_3B' | 'GEMMA_2_2B' | 'PHI_3_5' | 'QWEN_2_5_1B' | 'QWEN_2_5_3B' +// Define the expected structure of the response from Ollama API +export type OllamaResponse = { + model: string + created_at: string + message: { + role: string + content: string + } + done_reason: string + done: boolean + total_duration: number + load_duration: number + prompt_eval_count: number + prompt_eval_duration: number + eval_count: number + eval_duration: number +} + /** * Represents the function signature for cleaning up temporary files. */ diff --git a/src/utils/cleanUpFiles.ts b/src/utils/cleanUpFiles.ts index 07afea4e..35effe16 100644 --- a/src/utils/cleanUpFiles.ts +++ b/src/utils/cleanUpFiles.ts @@ -14,7 +14,7 @@ export async function cleanUpFiles(id: string): Promise { // Array of file extensions to delete const extensions = ['.wav', '.txt', '.md', '.lrc'] - log(success(` Deleted:`)) + log(success(` Temporary files deleted:`)) for (const ext of extensions) { try { await unlink(`${id}${ext}`) diff --git a/src/utils/generateMarkdown.ts b/src/utils/generateMarkdown.ts index af0267c7..ed53d673 100644 --- a/src/utils/generateMarkdown.ts +++ b/src/utils/generateMarkdown.ts @@ -23,9 +23,7 @@ export async function generateMarkdown( options: ProcessingOptions, input: string | RSSItem ): Promise { - // log(`Options received in generateMarkdown:\n`) - // log(options) - // log(`input:`, input) + // log(` - input: ${input}\n`) /** * Sanitizes a title string for use in filenames. * @@ -35,11 +33,11 @@ export async function generateMarkdown( function sanitizeTitle(title: string): string { return title .replace(/[^\w\s-]/g, '') // Remove all non-word chars except spaces and hyphens - .trim() // Remove leading and trailing whitespace - .replace(/[\s_]+/g, '-') // Replace spaces and underscores with a single hyphen - .replace(/-+/g, '-') // Replace multiple hyphens with a single hyphen - .toLowerCase() // Convert to lowercase - .slice(0, 200) // Limit to 200 characters + .trim() // Remove leading and trailing whitespace + .replace(/[\s_]+/g, '-') // Replace spaces and underscores with a single hyphen + .replace(/-+/g, '-') // Replace multiple hyphens with a single hyphen + .toLowerCase() // Convert to lowercase + .slice(0, 200) // Limit to 200 characters } // Declare variables to store generated content @@ -88,9 +86,6 @@ export async function generateMarkdown( `coverImage: "${thumbnail}"`, '---\n', ] - - // Log progress - log(step('\nStep 1 - Generating video markdown...\n')) break case !!options.file: @@ -114,9 +109,6 @@ export async function generateMarkdown( `coverImage: ""`, '---\n', ] - - // Log progress - log(step('\nStep 1 - Generating file markdown...\n')) break case !!options.rss: @@ -140,9 +132,6 @@ export async function generateMarkdown( `coverImage: "${coverImage}"`, '---\n', ] - - // Log progress - log(step('\nStep 1 - Generating RSS markdown...\n')) break default: @@ -158,6 +147,7 @@ export async function generateMarkdown( // Log the generated front matter and success message log(dim(frontMatterContent)) + log(step('\nStep 1 - Generating markdown...\n')) log(success(` Front matter successfully created and saved:\n - ${finalPath}.md`)) // Return the generated markdown data diff --git a/src/utils/runLLM.ts b/src/utils/runLLM.ts index be5fe722..419b5a02 100644 --- a/src/utils/runLLM.ts +++ b/src/utils/runLLM.ts @@ -50,7 +50,7 @@ export async function runLLM( const promptAndTranscript = `${prompt}${transcript}` if (llmServices) { - log(wait(` Processing with ${llmServices} Language Model...`)) + log(wait(` Processing with ${llmServices} Language Model...\n`)) const llmFunction: LLMFunction = LLM_FUNCTIONS[llmServices] if (!llmFunction) { throw new Error(`Invalid LLM option: ${llmServices}`) diff --git a/test/all.test.js b/test/all.test.js index fea4a8d6..a86828f3 100644 --- a/test/all.test.js +++ b/test/all.test.js @@ -10,20 +10,34 @@ const commands = [ { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk"', expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '01---2024-09-24-ep0-fsjam-podcast-prompt.md' + newName: '01---ep0-fsjam-podcast-prompt.md' }, { cmd: 'npm run as -- --playlist "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr"', expectedFiles: [ - { file: '2024-09-24-ep1-fsjam-podcast-prompt.md', newName: '02---2024-09-24-ep1-fsjam-podcast-prompt.md' }, - { file: '2024-09-24-ep0-fsjam-podcast-prompt.md', newName: '03---2024-09-24-ep0-fsjam-podcast-prompt.md' } + { file: '2024-09-24-ep1-fsjam-podcast-prompt.md', newName: '02A---ep1-fsjam-podcast-prompt.md' }, + { file: '2024-09-24-ep0-fsjam-podcast-prompt.md', newName: '02B---ep0-fsjam-podcast-prompt.md' } + ] + }, + { + cmd: 'npm run as -- --playlist "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr" --prompt titles --whisper tiny --llama', + expectedFiles: [ + { file: '2024-09-24-ep1-fsjam-podcast-llama-shownotes.md', newName: '03A---ep1-fsjam-podcast-llama-shownotes.md' }, + { file: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', newName: '03B---ep0-fsjam-podcast-llama-shownotes.md' } ] }, { cmd: 'npm run as -- --urls "content/example-urls.md"', expectedFiles: [ - { file: '2024-09-24-ep1-fsjam-podcast-prompt.md', newName: '04---2024-09-24-ep1-fsjam-podcast-prompt.md' }, - { file: '2024-09-24-ep0-fsjam-podcast-prompt.md', newName: '05---2024-09-24-ep0-fsjam-podcast-prompt.md' } + { file: '2024-09-24-ep1-fsjam-podcast-prompt.md', newName: '04A---ep1-fsjam-podcast-prompt.md' }, + { file: '2024-09-24-ep0-fsjam-podcast-prompt.md', newName: '04B---ep0-fsjam-podcast-prompt.md' } + ] + }, + { + cmd: 'npm run as -- --urls "content/example-urls.md" --prompt titles --whisper tiny --llama', + expectedFiles: [ + { file: '2024-09-24-ep1-fsjam-podcast-llama-shownotes.md', newName: '05A---ep1-fsjam-podcast-llama-shownotes.md' }, + { file: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', newName: '05B---ep0-fsjam-podcast-llama-shownotes.md' } ] }, { @@ -31,159 +45,125 @@ const commands = [ expectedFile: 'audio-prompt.md', newName: '06---audio-prompt.md' }, + { + cmd: 'npm run as -- --file "content/audio.mp3" --prompt titles --whisper tiny --llama', + expectedFile: 'audio-llama-shownotes.md', + newName: '07---audio-llama-shownotes.md' + }, + { + cmd: 'npm run as -- --rss "https://ajcwebdev.substack.com/feed"', + expectedFile: '2021-05-10-thoughts-on-lambda-school-layoffs-prompt.md', + newName: '08---thoughts-on-lambda-school-layoffs-prompt.md' + }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --chatgpt', expectedFile: '2024-09-24-ep0-fsjam-podcast-chatgpt-shownotes.md', - newName: '07---2024-09-24-ep0-fsjam-podcast-chatgpt-shownotes.md' + newName: '09---ep0-fsjam-podcast-chatgpt-shownotes.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --chatgpt GPT_4o_MINI', expectedFile: '2024-09-24-ep0-fsjam-podcast-chatgpt-shownotes.md', - newName: '08---2024-09-24-ep0-fsjam-podcast-chatgpt-shownotes.md' + newName: '10---ep0-fsjam-podcast-chatgpt-shownotes.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --claude', expectedFile: '2024-09-24-ep0-fsjam-podcast-claude-shownotes.md', - newName: '09---2024-09-24-ep0-fsjam-podcast-claude-shownotes.md' + newName: '11---ep0-fsjam-podcast-claude-shownotes.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --claude CLAUDE_3_SONNET', expectedFile: '2024-09-24-ep0-fsjam-podcast-claude-shownotes.md', - newName: '10---2024-09-24-ep0-fsjam-podcast-claude-shownotes.md' - }, - { - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --gemini', - expectedFile: '2024-09-24-ep0-fsjam-podcast-gemini-shownotes.md', - newName: '11---2024-09-24-ep0-fsjam-podcast-gemini-shownotes.md' - }, - { - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --gemini GEMINI_1_5_FLASH', - expectedFile: '2024-09-24-ep0-fsjam-podcast-gemini-shownotes.md', - newName: '12---2024-09-24-ep0-fsjam-podcast-gemini-shownotes.md' - }, + newName: '12---ep0-fsjam-podcast-claude-shownotes.md' + }, + // { + // cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --gemini', + // expectedFile: '2024-09-24-ep0-fsjam-podcast-gemini-shownotes.md', + // newName: '13---ep0-fsjam-podcast-gemini-shownotes.md' + // }, + // { + // cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --gemini GEMINI_1_5_FLASH', + // expectedFile: '2024-09-24-ep0-fsjam-podcast-gemini-shownotes.md', + // newName: '14---ep0-fsjam-podcast-gemini-shownotes.md' + // }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --cohere', expectedFile: '2024-09-24-ep0-fsjam-podcast-cohere-shownotes.md', - newName: '13---2024-09-24-ep0-fsjam-podcast-cohere-shownotes.md' + newName: '15---ep0-fsjam-podcast-cohere-shownotes.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --cohere COMMAND_R_PLUS', expectedFile: '2024-09-24-ep0-fsjam-podcast-cohere-shownotes.md', - newName: '14---2024-09-24-ep0-fsjam-podcast-cohere-shownotes.md' + newName: '16---ep0-fsjam-podcast-cohere-shownotes.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --mistral', expectedFile: '2024-09-24-ep0-fsjam-podcast-mistral-shownotes.md', - newName: '15---2024-09-24-ep0-fsjam-podcast-mistral-shownotes.md' + newName: '17---ep0-fsjam-podcast-mistral-shownotes.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --mistral MIXTRAL_8x7b', expectedFile: '2024-09-24-ep0-fsjam-podcast-mistral-shownotes.md', - newName: '16---2024-09-24-ep0-fsjam-podcast-mistral-shownotes.md' + newName: '18---ep0-fsjam-podcast-mistral-shownotes.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --octo', expectedFile: '2024-09-24-ep0-fsjam-podcast-octo-shownotes.md', - newName: '17---2024-09-24-ep0-fsjam-podcast-octo-shownotes.md' + newName: '19---ep0-fsjam-podcast-octo-shownotes.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --octo LLAMA_3_1_8B', expectedFile: '2024-09-24-ep0-fsjam-podcast-octo-shownotes.md', - newName: '18---2024-09-24-ep0-fsjam-podcast-octo-shownotes.md' + newName: '20---ep0-fsjam-podcast-octo-shownotes.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --llama', expectedFile: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', - newName: '19---2024-09-24-ep0-fsjam-podcast-llama-shownotes.md' + newName: '21---ep0-fsjam-podcast-llama-shownotes.md' + }, + { + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --ollama', + expectedFile: '2024-09-24-ep0-fsjam-podcast-ollama-shownotes.md', + newName: '22---ep0-fsjam-podcast-ollama-shownotes.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --deepgram', expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '20---2024-09-24-ep0-fsjam-podcast-prompt.md' + newName: '23---ep0-fsjam-podcast-prompt.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --deepgram --llama', expectedFile: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', - newName: '21---2024-09-24-ep0-fsjam-podcast-llama-shownotes.md' + newName: '24---ep0-fsjam-podcast-llama-shownotes.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --assembly', expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '22---2024-09-24-ep0-fsjam-podcast-prompt.md' + newName: '25---ep0-fsjam-podcast-prompt.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --assembly --llama', expectedFile: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', - newName: '23---2024-09-24-ep0-fsjam-podcast-llama-shownotes.md' + newName: '26---ep0-fsjam-podcast-llama-shownotes.md' }, { cmd: 'npm run as -- --video "https://ajc.pics/audio/fsjam-short.mp3" --assembly --speakerLabels', expectedFile: '2024-05-08-fsjam-short-prompt.md', - newName: '24---2024-05-08-fsjam-short-prompt.md' - }, - { - cmd: 'npm run as -- --video "https://ajc.pics/audio/fsjam-short.mp3" --assembly --speakerLabels --llama', - expectedFile: '2024-05-08-fsjam-short-llama-shownotes.md', - newName: '25---2024-05-08-fsjam-short-llama-shownotes.md' - }, - { - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisper tiny', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '26---2024-09-24-ep0-fsjam-podcast-prompt.md' + newName: '27---fsjam-short-prompt.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperDocker tiny', expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '26B---2024-09-24-ep0-fsjam-podcast-prompt.md' - }, - { - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --prompt titles', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '27---2024-09-24-ep0-fsjam-podcast-prompt.md' + newName: '28---ep0-fsjam-podcast-prompt.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --prompt titles summary shortChapters mediumChapters longChapters takeaways questions', expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '28---2024-09-24-ep0-fsjam-podcast-prompt.md' + newName: '29---ep0-fsjam-podcast-prompt.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --prompt titles summary shortChapters takeaways questions --whisper tiny --llama', expectedFile: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', - newName: '29---2024-09-24-ep0-fsjam-podcast-llama-shownotes.md' - }, - { - cmd: 'npm run as -- --playlist "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr" --prompt titles --whisper tiny --llama', - expectedFiles: [ - { file: '2024-09-24-ep1-fsjam-podcast-llama-shownotes.md', newName: '30---2024-09-24-ep1-fsjam-podcast-llama-shownotes.md' }, - { file: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', newName: '31---2024-09-24-ep0-fsjam-podcast-llama-shownotes.md' } - ] - }, - { - cmd: 'npm run as -- --urls "content/example-urls.md" --prompt titles --whisper tiny --llama', - expectedFiles: [ - { file: '2024-09-24-ep1-fsjam-podcast-llama-shownotes.md', newName: '32---2024-09-24-ep1-fsjam-podcast-llama-shownotes.md' }, - { file: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', newName: '33---2024-09-24-ep0-fsjam-podcast-llama-shownotes.md' } - ] - }, - { - cmd: 'npm run as -- --file "content/audio.mp3" --prompt titles --whisper tiny --llama', - expectedFile: 'audio-llama-shownotes.md', - newName: '34---audio-llama-shownotes.md' - }, - { - cmd: 'npm run as -- --rss "https://ajcwebdev.substack.com/feed"', - expectedFile: '2021-05-10-thoughts-on-lambda-school-layoffs-prompt.md', - newName: '35---2021-05-10-thoughts-on-lambda-school-layoffs-prompt.md' - }, - { - cmd: 'npm run as -- --rss "https://feeds.transistor.fm/fsjam-podcast/" --order newest --skip 94 --whisper tiny', - expectedFile: '2020-10-27-episode-0-the-fullstack-jamstack-podcast-with-anthony-campolo-and-christopher-burns-prompt.md', - newName: '36---2020-10-27-episode-0-the-fullstack-jamstack-podcast-with-anthony-campolo-and-christopher-burns-prompt.md' - }, - { - cmd: 'npm run as -- --rss "https://feeds.transistor.fm/fsjam-podcast/" --order oldest --skip 94 --whisper tiny', - expectedFile: '2023-06-28-episode-94-clerk-with-james-perkins-prompt.md', - newName: '37---2023-06-28-episode-94-clerk-with-james-perkins-prompt.md' + newName: '30---ep0-fsjam-podcast-llama-shownotes.md' } ] diff --git a/test/local.test.js b/test/local.test.js index 90328f46..2d7a520b 100644 --- a/test/local.test.js +++ b/test/local.test.js @@ -31,74 +31,74 @@ const commands = [ expectedFile: 'audio-prompt.md', newName: 'FILE_04.md' }, + { + cmd: 'npm run as -- --file "content/audio.mp3" --prompt titles --whisper tiny --llama', + expectedFile: 'audio-llama-shownotes.md', + newName: 'FILE_05.md' + }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --llama', expectedFile: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', - newName: 'FILE_05.md' + newName: 'FILE_06.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --ollama', expectedFile: '2024-09-24-ep0-fsjam-podcast-ollama-shownotes.md', - newName: 'FILE_06.md' + newName: 'FILE_07.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisper tiny', expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: 'FILE_07A.md' + newName: 'FILE_08.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperDocker tiny', expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: 'FILE_07B.md' + newName: 'FILE_09.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --prompt titles', expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: 'FILE_08.md' + newName: 'FILE_10.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --prompt titles summary mediumChapters takeaways questions', expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: 'FILE_09.md' + newName: 'FILE_11.md' }, { cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --prompt titles summary shortChapters takeaways questions --whisper tiny --llama', expectedFile: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', - newName: 'FILE_10.md' + newName: 'FILE_12.md' }, { cmd: 'npm run as -- --playlist "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr" --prompt titles --whisper tiny --llama', expectedFiles: [ - { file: '2024-09-24-ep1-fsjam-podcast-llama-shownotes.md', newName: 'FILE_11A.md' }, - { file: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', newName: 'FILE_11B.md' } + { file: '2024-09-24-ep1-fsjam-podcast-llama-shownotes.md', newName: 'FILE_13A.md' }, + { file: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', newName: 'FILE_13B.md' } ] }, { - cmd: 'npm run as -- --urls "content/example-urls.md" --prompt titles --whisper tiny --llama', + cmd: 'npm run as -- --urls "content/example-urls.md" --prompt titles --whisper tiny --ollama', expectedFiles: [ - { file: '2024-09-24-ep1-fsjam-podcast-llama-shownotes.md', newName: 'FILE_12A.md' }, - { file: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', newName: 'FILE_12B.md' } + { file: '2024-09-24-ep1-fsjam-podcast-ollama-shownotes.md', newName: 'FILE_14A.md' }, + { file: '2024-09-24-ep0-fsjam-podcast-ollama-shownotes.md', newName: 'FILE_14B.md' } ] }, - { - cmd: 'npm run as -- --file "content/audio.mp3" --prompt titles --whisper tiny --llama', - expectedFile: 'audio-llama-shownotes.md', - newName: 'FILE_13.md' - }, { cmd: 'npm run as -- --rss "https://ajcwebdev.substack.com/feed"', expectedFile: '2021-05-10-thoughts-on-lambda-school-layoffs-prompt.md', - newName: 'FILE_14.md' + newName: 'FILE_15.md' }, { - cmd: 'npm run as -- --rss "https://feeds.transistor.fm/fsjam-podcast/" --order newest --skip 94 --whisper tiny', - expectedFile: '2020-10-27-episode-0-the-fullstack-jamstack-podcast-with-anthony-campolo-and-christopher-burns-prompt.md', - newName: 'FILE_15.md' + cmd: 'npm run as -- --rss "https://ajcwebdev.substack.com/feed" --item "https://api.substack.com/feed/podcast/36236609/fd1f1532d9842fe1178de1c920442541.mp3" --whisper tiny --llama --prompt titles summary longChapters takeaways questions', + expectedFile: '2021-05-10-thoughts-on-lambda-school-layoffs-llama-shownotes.md', + newName: 'FILE_16.md', }, { - cmd: 'npm run as -- --rss "https://feeds.transistor.fm/fsjam-podcast/" --order oldest --skip 94 --whisper tiny', - expectedFile: '2023-06-28-episode-94-clerk-with-james-perkins-prompt.md', - newName: 'FILE_16.md' + cmd: 'npm run as -- --rss "https://ajcwebdev.substack.com/feed" --info', + expectedFile: 'rss_info.json', + newName: 'FILE_17_rss_info.json', } ]