diff --git a/.env.example b/.env.example index 3e7a254..aabbf68 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,8 @@ GEMINI_API_KEY="" COHERE_API_KEY="" MISTRAL_API_KEY="" OCTOAI_API_KEY="" +TOGETHER_API_KEY="" +FIREWORKS_API_KEY="" DEEPGRAM_API_KEY="" ASSEMBLY_API_KEY="" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 34b70f1..a37105b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,10 +11,9 @@ services: - /var/run/docker.sock:/var/run/docker.sock depends_on: - whisper - # - ollama - # environment: - # - OLLAMA_HOST=ollama - # - OLLAMA_PORT=11434 + - ollama + environment: + - OLLAMA_HOST=ollama networks: - autoshownet whisper: @@ -29,20 +28,16 @@ services: stdin_open: true networks: - autoshownet - # ollama: - # image: ollama/ollama - # command: ["ollama", "serve", "--address", "0.0.0.0"] # Listen on all interfaces - # ports: - # - "11434:11434" - # volumes: - # - ./ollama:/root/.ollama - # healthcheck: - # test: ["CMD", "curl", "-f", "http://localhost:11434/healthz"] - # interval: 10s - # timeout: 5s - # retries: 5 - # networks: - # - autoshownet + ollama: + image: ollama/ollama + volumes: + - ollama:/root/.ollama + environment: + - OLLAMA_HOST=0.0.0.0 + networks: + - autoshownet networks: autoshownet: - driver: bridge \ No newline at end of file + driver: bridge +volumes: + ollama: \ No newline at end of file diff --git a/docs/examples.md b/docs/examples.md index 64fec6d..8f39e15 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -281,6 +281,22 @@ npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --together Q npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --together QWEN_2_5_7B ``` +### Groq + +```bash +npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --groq +``` + +Select Groq model: + +```bash +npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --groq LLAMA_3_1_70B_VERSATILE +npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --groq LLAMA_3_1_8B_INSTANT +npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --groq LLAMA_3_2_1B_PREVIEW +npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --groq LLAMA_3_2_3B_PREVIEW +npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --groq MIXTRAL_8X7B_32768 +``` + ### Llama.cpp ```bash @@ -334,6 +350,9 @@ npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisper me # large-v2 model npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisper large-v2 + +# large-v3-turbo model +npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisper large-v3-turbo ``` Run `whisper.cpp` in a Docker container with `--whisperDocker`: @@ -342,6 +361,22 @@ Run `whisper.cpp` in a Docker container with `--whisperDocker`: npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperDocker base ``` +### Whisper Python + +Use the original [`openai/whisper`](https://github.com/openai/whisper) Python library with the newly released [`turbo`](https://github.com/openai/whisper/discussions/2363) model: + +```bash +npm run as -- --file "content/audio.mp3" --whisperPython turbo +``` + +### Whisper Diarization + +Use [`whisper-diarization`](https://github.com/MahmoudAshraf97/whisper-diarization) to provide speaker labels: + +```bash +npm run as -- --file "content/audio.mp3" --whisperDiarization tiny +``` + ### Deepgram Create a `.env` file and set API key as demonstrated in `.env.example` for `DEEPGRAM_API_KEY`. @@ -424,25 +459,29 @@ npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --prompt tit ### Docker Compose -This will start both `whisper.cpp` and the AutoShow Commander CLI in their own Docker containers. +This will start `whisper.cpp`, Ollama, and the AutoShow Commander CLI in their own Docker containers. ```bash npm run docker-up ``` -Replace `as` with `docker` to run most other commands explained in this document. +Replace `as` with `docker` to run most other commands explained in this document. Does not support all options at this time, notably `--llama`, `--whisperPython`, and `--whisperDiarization`. + +```bash +npm run docker -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" +npm run docker -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperDocker tiny +``` -- Does not support all commands at this time, notably `--llama` and `--ollama`. -- Currently working on the `llama.cpp` Docker integration so the entire project can be encapsulated in one local Docker Compose file. +Currently supports Ollama's official Docker image so the entire project can be encapsulated in one local Docker Compose file: ```bash -npm run docker -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperDocker base +npm run docker -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperDocker tiny --ollama ``` To reset your Docker images and containers, run: ```bash -docker system prune -af --volumes +npm run prune ``` ### Bun diff --git a/package.json b/package.json index c1be6ee..a5f1c47 100644 --- a/package.json +++ b/package.json @@ -19,11 +19,13 @@ }, "scripts": { "setup": "bash ./scripts/setup.sh", + "setup-python": "bash ./scripts/setup-python.sh", "autoshow": "tsx --env-file=.env --no-warnings src/autoshow.ts", "as": "tsx --env-file=.env --no-warnings src/autoshow.ts", - "docker": "docker compose run --remove-orphans autoshow", + "docker": "docker compose run --remove-orphans --rm autoshow --whisperDocker", "docker-up": "docker compose up --build -d --remove-orphans --no-start", "ds": "docker compose images && docker compose ls", + "prune": "docker system prune -af --volumes && docker image prune -af && docker container prune -f && docker volume prune -af", "v": "tsx --env-file=.env --no-warnings src/autoshow.ts --whisper large-v2 --video", "u": "tsx --env-file=.env --no-warnings src/autoshow.ts --whisper large-v2 --urls", "p": "tsx --env-file=.env --no-warnings src/autoshow.ts --whisper large-v2 --playlist", @@ -34,8 +36,9 @@ "fetch-local": "tsx --env-file=.env --no-warnings packages/server/tests/fetch-local.ts", "fetch-all": "tsx --env-file=.env --no-warnings packages/server/tests/fetch-all.ts", "t": "npm run test-local", - "test-local": "tsx --test test/local.test.js", - "test-all": "tsx --test test/all.test.js", + "test-local": "tsx --test test/local.test.ts", + "test-docker": "tsx --test test/docker.test.ts", + "test-integrations": "tsx --test test/integrations.test.ts", "clean": "tsx scripts/cleanContent.ts", "bun-as": "bun --env-file=.env --no-warnings src/autoshow.ts", "deno-as": "deno run --allow-sys --allow-read --allow-run --allow-write --allow-env src/autoshow.ts" @@ -53,7 +56,6 @@ "commander": "12.1.0", "fast-xml-parser": "4.5.0", "fastify": "5.0.0", - "ffmpeg-static": "5.2.0", "file-type": "19.5.0", "inquirer": "12.0.0", "node-llama-cpp": "3.1.1", @@ -64,6 +66,7 @@ "@types/inquirer": "9.0.7", "@types/node": "22.7.5", "tsx": "4.19.1", + "typedoc": "^0.26.10", "typescript": "5.6.3" } } diff --git a/scripts/setup-python.sh b/scripts/setup-python.sh index 634719b..4f91c7d 100755 --- a/scripts/setup-python.sh +++ b/scripts/setup-python.sh @@ -1,14 +1,33 @@ #!/bin/bash -# Clone the repository -git clone https://github.com/MahmoudAshraf97/whisper-diarization.git +# scripts/setup-python.sh -# Create and activate virtual environment -python3.12 -m venv whisper-diarization/venv -source whisper-diarization/venv/bin/activate +# Clone the repository if it doesn't exist +if [ ! -d "whisper-diarization" ]; then + echo "Cloning the whisper-diarization repository..." + git clone https://github.com/MahmoudAshraf97/whisper-diarization.git +else + echo "whisper-diarization repository already exists. Skipping clone." +fi -# Install the requirements -pip install -c whisper-diarization/constraints.txt -r whisper-diarization/requirements.txt +# Create the virtual environment if it doesn't exist +if [ ! -f "whisper-diarization/venv/bin/activate" ]; then + echo "Creating virtual environment..." + python3.12 -m venv whisper-diarization/venv +else + echo "Virtual environment already exists. Skipping creation." +fi + +# Install requirements if not already installed +if [ ! -f "whisper-diarization/.requirements_installed" ]; then + echo "Installing requirements..." + source whisper-diarization/venv/bin/activate + pip install -c whisper-diarization/constraints.txt -r whisper-diarization/requirements.txt + deactivate + touch whisper-diarization/.requirements_installed +else + echo "Requirements already installed. Skipping installation." +fi echo "Setup complete. To activate this environment in the future, run:" echo "source whisper-diarization/venv/bin/activate" diff --git a/src/autoshow.ts b/src/autoshow.ts index 625f68c..986862d 100644 --- a/src/autoshow.ts +++ b/src/autoshow.ts @@ -8,8 +8,7 @@ * Automate processing of audio and video content from various sources. * Supports processing YouTube videos, playlists, local files, and podcast RSS feeds. * - * Documentation: https://github.com/ajcwebdev/autoshow#readme - * Report Issues: https://github.com/ajcwebdev/autoshow/issues + * @packageDocumentation */ import { Command } from 'commander' @@ -26,7 +25,9 @@ import type { ProcessingOptions, HandlerFunction, LLMServices, TranscriptService // Initialize the command-line interface const program = new Command() -// Define command-line options and their descriptions +/** + * Defines the command-line interface options and descriptions. + */ program .name('autoshow') .version('0.0.1') @@ -57,6 +58,7 @@ program .option('--octo [model]', 'Use Octo for processing') .option('--fireworks [model]', 'Use Fireworks AI for processing with optional model specification') .option('--together [model]', 'Use Together AI for processing with optional model specification') + .option('--groq [model]', 'Use Groq for processing with optional model specification') .option('--llama [model]', 'Use Node Llama for processing with optional model specification') .option('--ollama [model]', 'Use Ollama for processing with optional model specification') .option('--gemini [model]', 'Use Gemini for processing with optional model specification') @@ -78,12 +80,16 @@ Report Issues: https://github.com/ajcwebdev/autoshow/issues /** * Helper function to validate that only one option from a list is provided. - * @param {string[]} optionKeys - The list of option keys to check. - * @param {ProcessingOptions} options - The options object. - * @param {string} errorMessage - The prefix of the error message. - * @returns {string | undefined} - The selected option or undefined. + * @param optionKeys - The list of option keys to check. + * @param options - The options object. + * @param errorMessage - The prefix of the error message. + * @returns The selected option or undefined. */ -function getSingleOption(optionKeys: string[], options: ProcessingOptions, errorMessage: string): string | undefined { +function getSingleOption( + optionKeys: string[], + options: ProcessingOptions, + errorMessage: string +): string | undefined { const selectedOptions = optionKeys.filter((opt) => options[opt as keyof ProcessingOptions]) if (selectedOptions.length > 1) { console.error(`Error: Multiple ${errorMessage} provided (${selectedOptions.join(', ')}). Please specify only one.`) @@ -94,8 +100,7 @@ function getSingleOption(optionKeys: string[], options: ProcessingOptions, error /** * Main action for the program. - * @param {ProcessingOptions} options - The command-line options provided by the user. - * @returns {Promise} + * @param options - The command-line options provided by the user. */ program.action(async (options: ProcessingOptions) => { log(opts(`Options received at beginning of command:\n`)) diff --git a/src/interactive.ts b/src/interactive.ts index 012d86a..bdecec5 100644 --- a/src/interactive.ts +++ b/src/interactive.ts @@ -1,4 +1,4 @@ -// src/inquirer.ts +// src/interactive.ts import inquirer from 'inquirer' import type { ProcessingOptions, InquirerAnswers, WhisperModelType } from './types.js' @@ -6,10 +6,12 @@ import { log } from './models.js' /** * Prompts the user for input if interactive mode is selected. - * @param {ProcessingOptions} options - The initial command-line options. - * @returns {Promise} - The updated options after user input. + * @param options - The initial command-line options. + * @returns The updated options after user input. */ -export async function handleInteractivePrompt(options: ProcessingOptions): Promise { +export async function handleInteractivePrompt( + options: ProcessingOptions +): Promise { const answers: InquirerAnswers = await inquirer.prompt([ { type: 'list', @@ -62,7 +64,7 @@ export async function handleInteractivePrompt(options: ProcessingOptions): Promi type: 'confirm', name: 'specifyItem', message: 'Do you want to process specific episodes by providing their audio URLs?', - when: (answers: InquirerAnswers) => answers.action === 'rss', + when: (answers: InquirerAnswers) => answers.action === 'rss' && !answers.info, default: false, }, { @@ -72,6 +74,43 @@ export async function handleInteractivePrompt(options: ProcessingOptions): Promi when: (answers: InquirerAnswers) => answers.action === 'rss' && answers.specifyItem, validate: (input: string) => (input ? true : 'Please enter at least one valid audio URL.'), }, + { + type: 'confirm', + name: 'info', + message: + 'Do you want to generate JSON file with RSS feed information instead of processing items?', + when: (answers: InquirerAnswers) => answers.action === 'rss', + default: false, + }, + { + type: 'list', + name: 'order', + message: 'Specify the order for RSS feed processing:', + choices: [ + { name: 'Newest first', value: 'newest' }, + { name: 'Oldest first', value: 'oldest' }, + ], + when: (answers: InquirerAnswers) => answers.action === 'rss' && !answers.info, + default: 'newest', + }, + { + type: 'input', + name: 'skip', + message: 'Number of items to skip when processing RSS feed:', + when: (answers: InquirerAnswers) => answers.action === 'rss' && !answers.info, + validate: (input: string) => + !isNaN(Number(input)) ? true : 'Please enter a valid number.', + filter: (input: string) => Number(input), + }, + { + type: 'input', + name: 'last', + message: 'Number of most recent items to process (overrides order and skip):', + when: (answers: InquirerAnswers) => answers.action === 'rss' && !answers.info, + validate: (input: string) => + !isNaN(Number(input)) ? true : 'Please enter a valid number.', + filter: (input: string) => Number(input), + }, { type: 'list', name: 'llmServices', @@ -86,19 +125,120 @@ export async function handleInteractivePrompt(options: ProcessingOptions): Promi { name: 'Cohere', value: 'cohere' }, { name: 'Mistral', value: 'mistral' }, { name: 'OctoAI', value: 'octo' }, + { name: 'Fireworks AI', value: 'fireworks' }, + { name: 'Together AI', value: 'together' }, + { name: 'Groq', value: 'groq' }, ], }, { type: 'list', - name: 'llama', - message: 'Select the LLAMA model you want to use:', - choices: [ - { name: 'LLAMA 3 8B Q4 Model', value: 'LLAMA_3_1_8B_Q4_MODEL' }, - { name: 'LLAMA 3 8B Q6 Model', value: 'LLAMA_3_1_8B_Q6_MODEL' }, - { name: 'GEMMA 2 2B Q4 Model', value: 'GEMMA_2_2B_Q4_MODEL' }, - { name: 'GEMMA 2 2B Q6 Model', value: 'GEMMA_2_2B_Q6_MODEL' }, - ], - when: (answers: InquirerAnswers) => answers.llmServices === 'llama', + name: 'llmModel', + message: 'Select the model you want to use:', + choices: (answers: InquirerAnswers) => { + switch (answers.llmServices) { + case 'llama': + return [ + { name: 'LLAMA 3 8B Q4 Model', value: 'LLAMA_3_1_8B_Q4_MODEL' }, + { name: 'LLAMA 3 8B Q6 Model', value: 'LLAMA_3_1_8B_Q6_MODEL' }, + { name: 'GEMMA 2 2B Q4 Model', value: 'GEMMA_2_2B_Q4_MODEL' }, + { name: 'GEMMA 2 2B Q6 Model', value: 'GEMMA_2_2B_Q6_MODEL' }, + ] + case 'ollama': + return [ + { name: 'LLAMA 3 2 1B', value: 'LLAMA_3_2_1B' }, + { name: 'LLAMA 3 2 3B', value: 'LLAMA_3_2_3B' }, + { name: 'GEMMA 2 2B', value: 'GEMMA_2_2B' }, + { name: 'PHI 3 5', value: 'PHI_3_5' }, + { name: 'QWEN 2 5 1B', value: 'QWEN_2_5_1B' }, + { name: 'QWEN 2 5 3B', value: 'QWEN_2_5_3B' }, + ] + case 'chatgpt': + return [ + { name: 'GPT 4 o MINI', value: 'GPT_4o_MINI' }, + { name: 'GPT 4 o', value: 'GPT_4o' }, + { name: 'GPT 4 TURBO', value: 'GPT_4_TURBO' }, + { name: 'GPT 4', value: 'GPT_4' }, + ] + case 'claude': + return [ + { name: 'Claude 3.5 Sonnet', value: 'CLAUDE_3_5_SONNET' }, + { name: 'Claude 3 Opus', value: 'CLAUDE_3_OPUS' }, + { name: 'Claude 3 Sonnet', value: 'CLAUDE_3_SONNET' }, + { name: 'Claude 3 Haiku', value: 'CLAUDE_3_HAIKU' }, + ] + case 'cohere': + return [ + { name: 'Command R', value: 'COMMAND_R' }, + { name: 'Command R Plus', value: 'COMMAND_R_PLUS' }, + ] + case 'mistral': + return [ + { name: 'Mixtral 8x7b', value: 'MIXTRAL_8x7b' }, + { name: 'Mixtral 8x22b', value: 'MIXTRAL_8x22b' }, + { name: 'Mistral Large', value: 'MISTRAL_LARGE' }, + { name: 'Mistral Nemo', value: 'MISTRAL_NEMO' }, + ] + case 'octo': + return [ + { name: 'LLAMA 3 1 8B', value: 'LLAMA_3_1_8B' }, + { name: 'LLAMA 3 1 70B', value: 'LLAMA_3_1_70B' }, + { name: 'LLAMA 3 1 405B', value: 'LLAMA_3_1_405B' }, + { name: 'Mistral 7B', value: 'MISTRAL_7B' }, + { name: 'Mixtral 8x7b', value: 'MIXTRAL_8X_7B' }, + { name: 'Nous Hermes Mixtral 8x7b', value: 'NOUS_HERMES_MIXTRAL_8X_7B' }, + { name: 'Wizard 2 8x22b', value: 'WIZARD_2_8X_22B' }, + ] + case 'fireworks': + return [ + { name: 'LLAMA 3 1 405B', value: 'LLAMA_3_1_405B' }, + { name: 'LLAMA 3 1 70B', value: 'LLAMA_3_1_70B' }, + { name: 'LLAMA 3 1 8B', value: 'LLAMA_3_1_8B' }, + { name: 'LLAMA 3 2 3B', value: 'LLAMA_3_2_3B' }, + { name: 'LLAMA 3 2 1B', value: 'LLAMA_3_2_1B' }, + { name: 'QWEN 2 5 72B', value: 'QWEN_2_5_72B' }, + ] + case 'together': + return [ + { name: 'LLAMA 3 2 3B', value: 'LLAMA_3_2_3B' }, + { name: 'LLAMA 3 1 405B', value: 'LLAMA_3_1_405B' }, + { name: 'LLAMA 3 1 70B', value: 'LLAMA_3_1_70B' }, + { name: 'LLAMA 3 1 8B', value: 'LLAMA_3_1_8B' }, + { name: 'Gemma 2 27B', value: 'GEMMA_2_27B' }, + { name: 'Gemma 2 9B', value: 'GEMMA_2_9B' }, + { name: 'QWEN 2 5 72B', value: 'QWEN_2_5_72B' }, + { name: 'QWEN 2 5 7B', value: 'QWEN_2_5_7B' }, + ] + case 'groq': + return [ + { name: 'LLAMA 3 1 70B Versatile', value: 'LLAMA_3_1_70B_VERSATILE' }, + { name: 'LLAMA 3 1 8B Instant', value: 'LLAMA_3_1_8B_INSTANT' }, + { name: 'LLAMA 3 2 1B Preview', value: 'LLAMA_3_2_1B_PREVIEW' }, + { name: 'LLAMA 3 2 3B Preview', value: 'LLAMA_3_2_3B_PREVIEW' }, + { name: 'Mixtral 8x7b 32768', value: 'MIXTRAL_8X7B_32768' }, + ] + case 'gemini': + return [ + { name: 'Gemini 1.5 Flash', value: 'GEMINI_1_5_FLASH' }, + { name: 'Gemini 1.5 Pro', value: 'GEMINI_1_5_PRO' }, + ] + default: + return [] + } + }, + when: (answers: InquirerAnswers) => + [ + 'llama', + 'ollama', + 'chatgpt', + 'claude', + 'cohere', + 'mistral', + 'octo', + 'fireworks', + 'together', + 'groq', + 'gemini', + ].includes(answers.llmServices as string), }, { type: 'list', @@ -107,6 +247,8 @@ export async function handleInteractivePrompt(options: ProcessingOptions): Promi choices: [ { name: 'Whisper.cpp', value: 'whisper' }, { name: 'Whisper.cpp (Docker)', value: 'whisperDocker' }, + { name: 'Whisper Python', value: 'whisperPython' }, + { name: 'Whisper Diarization', value: 'whisperDiarization' }, { name: 'Deepgram', value: 'deepgram' }, { name: 'AssemblyAI', value: 'assembly' }, ], @@ -115,8 +257,23 @@ export async function handleInteractivePrompt(options: ProcessingOptions): Promi type: 'list', name: 'whisperModel', message: 'Select the Whisper model type:', - choices: ['tiny', 'tiny.en', 'base', 'base.en', 'small', 'small.en', 'medium', 'medium.en', 'large-v1', 'large-v2'], - when: (answers: InquirerAnswers) => answers.transcriptServices === 'whisper' || answers.transcriptServices === 'whisperDocker', + choices: [ + 'tiny', + 'tiny.en', + 'base', + 'base.en', + 'small', + 'small.en', + 'medium', + 'medium.en', + 'large-v1', + 'large-v2', + 'turbo', + ], + when: (answers: InquirerAnswers) => + ['whisper', 'whisperDocker', 'whisperPython', 'whisperDiarization'].includes( + answers.transcriptServices as string + ), default: 'large-v2', }, { @@ -161,6 +318,7 @@ export async function handleInteractivePrompt(options: ProcessingOptions): Promi process.exit(0) } + // Merge answers into options options = { ...options, ...answers, @@ -168,21 +326,36 @@ export async function handleInteractivePrompt(options: ProcessingOptions): Promi // Handle transcription options if (answers.transcriptServices) { - if (answers.transcriptServices === 'whisper' || answers.transcriptServices === 'whisperDocker') { - options[answers.transcriptServices] = answers.whisperModel as WhisperModelType + if ( + ['whisper', 'whisperDocker', 'whisperPython', 'whisperDiarization'].includes( + answers.transcriptServices + ) + ) { + // Assign the Whisper model + (options as any)[answers.transcriptServices] = answers.whisperModel as WhisperModelType + } else if (answers.transcriptServices === 'deepgram' || answers.transcriptServices === 'assembly') { + // Assign boolean true for these services + (options as any)[answers.transcriptServices] = true + } + } + + // Handle LLM options + if (answers.llmServices) { + if (answers.llmModel) { + (options as any)[answers.llmServices] = answers.llmModel } else { - options[answers.transcriptServices] = true + (options as any)[answers.llmServices] = true } } // Handle 'item' for RSS feed if (typeof answers.item === 'string') { - options.item = answers.item.split(',').map(item => item.trim()) + options.item = answers.item.split(',').map((item) => item.trim()) } // Remove unnecessary properties - const keysToRemove = ['action', 'specifyItem', 'confirmAction'] - keysToRemove.forEach(key => delete options[key as keyof typeof options]) + const keysToRemove = ['action', 'specifyItem', 'confirmAction', 'llmModel', 'whisperModel'] + keysToRemove.forEach((key) => delete options[key as keyof typeof options]) return options } \ No newline at end of file diff --git a/src/llms/groq.ts b/src/llms/groq.ts new file mode 100644 index 0000000..2731b6c --- /dev/null +++ b/src/llms/groq.ts @@ -0,0 +1,86 @@ +// src/llms/groq.ts + +import { writeFile } from 'node:fs/promises' +import { env } from 'node:process' +import { log, wait, GROQ_MODELS } from '../models.js' +import type { GroqChatCompletionResponse, GroqModelType } from '../types.js' + +// Define the Groq API URL +const GROQ_API_URL = 'https://api.groq.com/openai/v1/chat/completions' + +/** + * Function to call the Groq chat completion API. + * @param {string} promptAndTranscript - The combined prompt and transcript text to process. + * @param {string} tempPath - The temporary file path to write the LLM output. + * @param {string} model - The model to use, e.g., 'MIXTRAL_8X7B_32768'. + */ +export const callGroq = async (promptAndTranscript: string, tempPath: string, model: string = 'MIXTRAL_8X7B_32768'): Promise => { + // Ensure that the API key is set + if (!env.GROQ_API_KEY) { + throw new Error('GROQ_API_KEY environment variable is not set. Please set it to your Groq API key.') + } + + try { + const actualModel = GROQ_MODELS[model as GroqModelType] || GROQ_MODELS.MIXTRAL_8X7B_32768 // Retrieve the actual model string from GROQ_MODELS + + // Prepare the request body + const requestBody = { + model: actualModel, + messages: [ + { + role: 'user', + content: promptAndTranscript, + }, + ], + // max_tokens: 4000, + } + + // Send the POST request + const response = await fetch(GROQ_API_URL, { + method: 'POST', + headers: { + Authorization: `Bearer ${env.GROQ_API_KEY}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + }) + + // Check if the response is OK + if (!response.ok) { + const errorText = await response.text() + throw new Error(`Groq API error: ${response.status} ${response.statusText} - ${errorText}`) + } + + // Parse the JSON response + const data = (await response.json()) as GroqChatCompletionResponse + + // Extract the generated content + const content = data.choices[0]?.message?.content + const finishReason = data.choices[0]?.finish_reason + const usedModel = data.model + const usage = data.usage + + if (!content) { + throw new Error('No content generated from the Groq API') + } + + // Write the generated content to the specified output file + await writeFile(tempPath, content) + log(wait(`\n Groq response saved to ${tempPath}`)) + + // Log finish reason, used model, and token usage + log(wait(`\n Finish Reason: ${finishReason}\n Model Used: ${usedModel}`)) + if (usage) { + const { prompt_tokens, completion_tokens, total_tokens } = usage + log( + wait( + ` Token Usage:\n - ${prompt_tokens} prompt tokens\n - ${completion_tokens} completion tokens\n - ${total_tokens} total tokens` + ) + ) + } + } catch (error) { + // Log any errors that occur during the process + console.error(`Error in callGroq: ${(error as Error).message}`) + throw error // Re-throw the error for handling by the caller + } +} \ No newline at end of file diff --git a/src/llms/ollama.ts b/src/llms/ollama.ts index 6722764..f735c1b 100644 --- a/src/llms/ollama.ts +++ b/src/llms/ollama.ts @@ -41,25 +41,31 @@ export const callOllama: LLMFunction = async (promptAndTranscript: string, tempP if (await checkServer()) { log(wait(' - Ollama server is already running.')) } else { - log(wait(' - Ollama server is not running. Attempting to start...')) - const ollamaProcess = spawn('ollama', ['serve'], { - detached: true, - stdio: 'ignore' - }) - ollamaProcess.unref() + if (ollamaHost === 'ollama') { + // Running inside Docker, do not attempt to start the server + throw new Error('Ollama server is not running. Please ensure the Ollama server is running and accessible.') + } else { + // Not running in Docker, attempt to start the server + log(wait(' - Ollama server is not running. Attempting to start...')) + const ollamaProcess = spawn('ollama', ['serve'], { + detached: true, + stdio: 'ignore' + }) + ollamaProcess.unref() - // Wait for the server to be ready - let attempts = 0 - while (attempts < 30) { // Increased to 30 attempts, 30 seconds total - if (await checkServer()) { - log(wait(' - Ollama server is now ready.')) - break + // Wait for the server to be ready + let attempts = 0 + while (attempts < 30) { // Wait up to 30 seconds + if (await checkServer()) { + log(wait(' - Ollama server is now ready.')) + break + } + await new Promise(resolve => setTimeout(resolve, 1000)) + attempts++ + } + if (attempts === 30) { + throw new Error('Ollama server failed to become ready in time.') } - await new Promise(resolve => setTimeout(resolve, 1000)) - attempts++ - } - if (attempts === 30) { - throw new Error('Ollama server failed to become ready in time.') } } @@ -174,7 +180,7 @@ export const callOllama: LLMFunction = async (promptAndTranscript: string, tempP // Write the full content to the output file await writeFile(tempPath, fullContent) } catch (error) { - console.error(`Error in callOllama: ${error instanceof Error ? (error as Error).message : String(error)}`) + console.error(`Error in callOllama: ${error instanceof Error ? error.message : String(error)}`) console.error(`Stack Trace: ${error instanceof Error ? error.stack : 'No stack trace available'}`) throw error } diff --git a/src/models.ts b/src/models.ts index 97942dc..943fc45 100644 --- a/src/models.ts +++ b/src/models.ts @@ -2,7 +2,7 @@ import chalk from 'chalk' import type { ChalkInstance } from 'chalk' -import type { WhisperModelType, ChatGPTModelType, ClaudeModelType, CohereModelType, GeminiModelType, MistralModelType, OctoModelType, LlamaModelType, OllamaModelType, TogetherModelType, FireworksModelType } from './types.js' +import type { WhisperModelType, ChatGPTModelType, ClaudeModelType, CohereModelType, GeminiModelType, MistralModelType, OctoModelType, LlamaModelType, OllamaModelType, TogetherModelType, FireworksModelType, GroqModelType } from './types.js' export const step: ChalkInstance = chalk.bold.underline export const dim: ChalkInstance = chalk.dim @@ -14,7 +14,7 @@ export const final: ChalkInstance = chalk.bold.italic export const log: typeof console.log = console.log export const ACTION_OPTIONS = ['video', 'playlist', 'urls', 'file', 'rss'] -export const LLM_OPTIONS = ['chatgpt', 'claude', 'cohere', 'mistral', 'octo', 'llama', 'ollama', 'gemini', 'fireworks', 'together'] +export const LLM_OPTIONS = ['chatgpt', 'claude', 'cohere', 'mistral', 'octo', 'llama', 'ollama', 'gemini', 'fireworks', 'together', 'groq'] export const TRANSCRIPT_OPTIONS = ['whisper', 'whisperDocker', 'whisperPython', 'whisperDiarization', 'deepgram', 'assembly'] /** @@ -32,6 +32,7 @@ export const WHISPER_MODELS: Record = { 'medium.en': 'ggml-medium.en.bin', 'large-v1': 'ggml-large-v1.bin', 'large-v2': 'ggml-large-v2.bin', + 'large-v3-turbo': 'ggml-large-v3-turbo.bin', } /** @@ -50,6 +51,7 @@ export const WHISPER_PYTHON_MODELS: Record = { 'large-v1': 'large-v1', 'large-v2': 'large-v2', turbo: 'turbo', + 'large-v3-turbo': 'large-v3-turbo' } /** @@ -146,6 +148,18 @@ export const TOGETHER_MODELS: Record = { QWEN_2_5_7B: "Qwen/Qwen2.5-7B-Instruct-Turbo", } +/** + * Map of Groq model identifiers to their API names. + * @type {Record} + */ +export const GROQ_MODELS: Record = { + LLAMA_3_1_70B_VERSATILE: 'llama-3.1-70b-versatile', + LLAMA_3_1_8B_INSTANT: 'llama-3.1-8b-instant', + LLAMA_3_2_1B_PREVIEW: 'llama-3.2-1b-preview', + LLAMA_3_2_3B_PREVIEW: 'llama-3.2-3b-preview', + MIXTRAL_8X7B_32768: 'mixtral-8x7b-32768', +} + /** * Map of local model identifiers to their filenames and URLs * @type {Record} diff --git a/src/transcription/whisper.ts b/src/transcription/whisper.ts index 63d2b8f..759c66c 100644 --- a/src/transcription/whisper.ts +++ b/src/transcription/whisper.ts @@ -4,8 +4,7 @@ import { readFile, writeFile } from 'node:fs/promises' import { exec } from 'node:child_process' import { promisify } from 'node:util' import { existsSync } from 'node:fs' -import { WHISPER_MODELS } from '../models.js' -import { log, wait } from '../models.js' +import { log, wait, WHISPER_MODELS } from '../models.js' import type { ProcessingOptions, WhisperModelType } from '../types.js' const execPromise = promisify(exec) diff --git a/src/transcription/whisperDiarization.ts b/src/transcription/whisperDiarization.ts index 0b42c4b..418c937 100644 --- a/src/transcription/whisperDiarization.ts +++ b/src/transcription/whisperDiarization.ts @@ -3,10 +3,9 @@ import { readFile, writeFile, unlink } from 'node:fs/promises' import { exec } from 'node:child_process' import { promisify } from 'node:util' -// import { existsSync } from 'node:fs' -import { log, wait } from '../models.js' +import { existsSync } from 'node:fs' +import { log, wait, WHISPER_PYTHON_MODELS } from '../models.js' import type { ProcessingOptions } from '../types.js' -import { WHISPER_PYTHON_MODELS } from '../models.js' const execPromise = promisify(exec) @@ -18,7 +17,7 @@ const execPromise = promisify(exec) * @throws {Error} - If an error occurs during transcription. */ export async function callWhisperDiarization(options: ProcessingOptions, finalPath: string): Promise { - log(wait('\n Using openai-whisper Python library for transcription...')) + log(wait('\n Using whisper-diarization for transcription...')) try { // Get the whisper model from options or use 'base' as default @@ -26,7 +25,7 @@ export async function callWhisperDiarization(options: ProcessingOptions, finalPa if (typeof options.whisperDiarization === 'string') { whisperModel = options.whisperDiarization } else if (options.whisperDiarization !== true) { - throw new Error('Invalid whisperPython option') + throw new Error('Invalid whisperDiarization option') } if (!(whisperModel in WHISPER_PYTHON_MODELS)) { @@ -35,34 +34,22 @@ export async function callWhisperDiarization(options: ProcessingOptions, finalPa log(wait(`\n - whisperModel: ${whisperModel}`)) - // // Check if ffmpeg is installed - // try { - // await execPromise('ffmpeg -version') - // } catch (error) { - // throw new Error('ffmpeg is not installed or not available in PATH') - // } - - // // Check if Python is installed - // try { - // await execPromise('python3 --version') - // } catch (error) { - // throw new Error('Python is not installed or not available in PATH') - // } - - // // Check if the whisper-diarization repo is cloned - // if (!existsSync('./whisper-diarization')) { - // log(`\n No whisper-diarization repo found, running git clone...\n`) - // await execPromise('git clone https://github.com/MahmoudAshraf97/whisper-diarization.git') - // log(`\n - whisper-diarization clone complete.\n`) - // } - - // Prepare the command to run the transcription - const command = `python whisper-diarization/diarize.py -a ${finalPath}.wav --whisper-model ${whisperModel}` + // Check if the virtual environment exists + const venvPythonPath = 'whisper-diarization/venv/bin/python' + if (!existsSync(venvPythonPath)) { + log(wait(`\n Virtual environment not found, running setup script...\n`)) + await execPromise('bash scripts/setup-python.sh') + log(wait(` - whisper-diarization setup complete.\n`)) + } - log(wait(`\n Running transcription with command:\n ${command}\n`)) + // Prepare the command to run the transcription, example command after interpolation: + // whisper-diarization/venv/bin/python whisper-diarization/diarize.py -a content/audio.wav --whisper-model tiny + const command = `${venvPythonPath} whisper-diarization/diarize.py -a ${finalPath}.wav --whisper-model ${whisperModel}` + log(wait(`\n Running transcription with command:\n\n ${command}\n`)) // Execute the command await execPromise(command) + await unlink(`${finalPath}.txt`) // Read the generated transcript file const srtContent = await readFile(`${finalPath}.srt`, 'utf8') @@ -97,13 +84,13 @@ export async function callWhisperDiarization(options: ProcessingOptions, finalPa // Write the formatted content to a text file await writeFile(`${finalPath}.txt`, txtContent) - log(wait(`\n Transcript transformation successfully completed...\n - ${finalPath}.txt\n`)) + log(wait(`\n Transcript transformation successfully completed:\n - ${finalPath}.txt\n`)) // Create an empty LRC file to prevent cleanup errors and unlink SRT file await writeFile(`${finalPath}.lrc`, '') - log(wait(`\n Empty LRC file created:\n - ${finalPath}.lrc\n`)) + log(wait(` Empty LRC file created:\n - ${finalPath}.lrc`)) await unlink(`${finalPath}.srt`) - log(wait(`\n SRT file deleted:\n - ${finalPath}.srt\n`)) + log(wait(` SRT file deleted:\n - ${finalPath}.srt`)) // Return the processed content return txtContent @@ -112,4 +99,4 @@ export async function callWhisperDiarization(options: ProcessingOptions, finalPa console.error('Error in callWhisperDiarization:', (error as Error).message) process.exit(1) } -} +} \ No newline at end of file diff --git a/src/transcription/whisperDocker.ts b/src/transcription/whisperDocker.ts index ac11458..09659dd 100644 --- a/src/transcription/whisperDocker.ts +++ b/src/transcription/whisperDocker.ts @@ -4,8 +4,7 @@ import { readFile, writeFile } from 'node:fs/promises' import { exec } from 'node:child_process' import { promisify } from 'node:util' import { join } from 'node:path' -import { WHISPER_MODELS } from '../models.js' -import { log, wait } from '../models.js' +import { log, wait, WHISPER_MODELS } from '../models.js' import type { ProcessingOptions, WhisperModelType } from '../types.js' const execPromise = promisify(exec) diff --git a/src/transcription/whisperPython.ts b/src/transcription/whisperPython.ts index 6472f3d..52374d7 100644 --- a/src/transcription/whisperPython.ts +++ b/src/transcription/whisperPython.ts @@ -3,9 +3,8 @@ import { readFile, writeFile, unlink } from 'node:fs/promises' import { exec } from 'node:child_process' import { promisify } from 'node:util' -import { log, wait } from '../models.js' +import { log, wait, WHISPER_PYTHON_MODELS } from '../models.js' import type { ProcessingOptions } from '../types.js' -import { WHISPER_PYTHON_MODELS } from '../models.js' const execPromise = promisify(exec) diff --git a/src/types.ts b/src/types.ts index fc2ab85..f2b258a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -2,6 +2,7 @@ /** * @file This file contains all the custom type definitions used across the Autoshow project. + * @packageDocumentation */ /** @@ -48,10 +49,12 @@ export type ProcessingOptions = { mistral?: string /** OctoAI model to use (e.g., 'LLAMA_3_1_8B'). */ octo?: string - /** Fireworks model to use (e.g., ''). */ + /** Fireworks model to use (e.g., ''). */ fireworks?: string - /** Together model to use (e.g., ''). */ + /** Together model to use (e.g., ''). */ together?: string + /** Groq model to use (e.g., ''). */ + groq?: string /** Ollama model to use for local inference (e.g., 'LLAMA_3_2_1B'). */ ollama?: string /** Llama model to use for local inference (e.g., 'LLAMA_3_1_8B'). */ @@ -61,15 +64,15 @@ export type ProcessingOptions = { /** Array of prompt sections to include (e.g., ['titles', 'summary']). */ prompt?: string[] /** The selected LLM option. */ - llmServices?: LLMServices | undefined + llmServices?: LLMServices /** The selected transcription option. */ - transcriptServices?: TranscriptServices | undefined + transcriptServices?: TranscriptServices /** Number of items to skip in RSS feed processing. */ skip?: number /** Order in which to process RSS feed items ('newest' or 'oldest'). */ - last?: number - /** Number of most recent items to process (overrides --order and --skip). */ order?: string + /** Number of most recent items to process (overrides --order and --skip). */ + last?: number /** Whether to run in interactive mode. */ interactive?: boolean } @@ -79,7 +82,7 @@ export type ProcessingOptions = { */ export type InquirerAnswers = { /** The action selected by the user (e.g., 'video', 'playlist'). */ - action?: string // Make this optional + action?: string /** YouTube video URL provided by the user. */ video?: string /** YouTube playlist URL provided by the user. */ @@ -94,24 +97,28 @@ export type InquirerAnswers = { specifyItem?: boolean /** Comma-separated audio URLs of specific RSS items. */ item?: string | string[] + /** Whether to generate JSON file with RSS feed information instead of processing items. */ + info?: boolean + /** Number of items to skip in RSS feed processing. */ + skip?: number + /** Number of most recent items to process (overrides order and skip). */ + last?: number + /** Order in which to process RSS feed items ('newest' or 'oldest'). */ + order?: string /** LLM option selected by the user. */ - llmServices?: LLMServices | undefined - /** Specific Llama model selected by the user. */ - llamaModel?: string + llmServices?: LLMServices + /** Specific LLM model selected by the user. */ + llmModel?: string /** Transcription option selected by the user. */ - transcriptServices?: TranscriptServices | undefined + transcriptServices?: TranscriptServices /** Whisper model type selected by the user. */ - whisperModel?: WhisperModelType // Add whisperModel to the InquirerAnswers + whisperModel?: WhisperModelType /** Whether to use speaker labels in transcription. */ speakerLabels?: boolean /** Prompt sections selected by the user. */ prompt?: string[] /** Whether to keep temporary files after processing. */ noCleanUp?: boolean - /** Order in which to process RSS feed items ('newest' or 'oldest'). */ - order?: string - /** Number of items to skip in RSS feed processing. */ - skip?: number /** Whether to proceed with the action. */ confirmAction?: boolean } @@ -127,17 +134,21 @@ export type InquirerQuestions = Array<{ /** The message to display to the user. */ message: string /** The choices available for selection (for 'list' and 'checkbox' types). */ - choices?: Array | Function + choices?: Array | (() => Array) /** A function to determine when to display the prompt. */ - when?: Function + when?: () => boolean /** A function to validate the user's input. */ - validate?: Function + validate?: (input: any) => boolean | string /** The default value for the prompt. */ default?: any }> /** * Represents a handler function for processing different actions (e.g., video, playlist). + * @param options - The options containing various inputs. + * @param input - The specific input (URL or file path). + * @param llmServices - The selected LLM service (optional). + * @param transcriptServices - The selected transcription service (optional). */ export type HandlerFunction = ( // The options containing various inputs @@ -145,9 +156,9 @@ export type HandlerFunction = ( // The specific input (URL or file path) input: string, // Allow llmServices to be optional or undefined - llmServices?: LLMServices | undefined, + llmServices?: LLMServices, // Allow transcriptServices to be optional or undefined - transcriptServices?: TranscriptServices | undefined + transcriptServices?: TranscriptServices ) => Promise /** @@ -254,8 +265,9 @@ export type TranscriptServices = 'whisper' | 'whisperDocker' | 'whisperPython' | * - medium.en: Medium English-only model. * - large-v1: Large multilingual model version 1. * - large-v2: Large multilingual model version 2. + * - large-v3-turbo: Large multilingual model version 3 with new turbo model. */ -export type WhisperModelType = 'tiny' | 'tiny.en' | 'base' | 'base.en' | 'small' | 'small.en' | 'medium' | 'medium.en' | 'large-v1' | 'large-v2' | 'turbo' +export type WhisperModelType = 'tiny' | 'tiny.en' | 'base' | 'base.en' | 'small' | 'small.en' | 'medium' | 'medium.en' | 'large-v1' | 'large-v2' | 'large-v3-turbo' | 'turbo' /** * Represents the object containing the different prompts, their instructions to the LLM, and their expected example output. @@ -279,7 +291,7 @@ export type PromptSection = { * - ollama: Use Ollama for processing. * - gemini: Use Google's Gemini models. */ -export type LLMServices = 'chatgpt' | 'claude' | 'cohere' | 'mistral' | 'octo' | 'llama' | 'ollama' | 'gemini' | 'fireworks' | 'together' +export type LLMServices = 'chatgpt' | 'claude' | 'cohere' | 'mistral' | 'octo' | 'llama' | 'ollama' | 'gemini' | 'fireworks' | 'together' | 'groq' /** * Represents the options for LLM processing. @@ -297,6 +309,9 @@ export type LLMOptions = { /** * Represents a function that calls an LLM for processing. + * @param promptAndTranscript - The combined prompt and transcript. + * @param tempPath - The temporary file path. + * @param llmModel - The specific LLM model to use (optional). */ export type LLMFunction = ( promptAndTranscript: string, @@ -332,6 +347,8 @@ export type OctoModelType = 'LLAMA_3_1_8B' | 'LLAMA_3_1_70B' | 'LLAMA_3_1_405B' export type FireworksModelType = 'LLAMA_3_1_405B' | 'LLAMA_3_1_70B' | 'LLAMA_3_1_8B' | 'LLAMA_3_2_3B' | 'LLAMA_3_2_1B' | 'QWEN_2_5_72B' /** Define available Together models. */ export type TogetherModelType = 'LLAMA_3_2_3B' | 'LLAMA_3_1_405B' | 'LLAMA_3_1_70B' | 'LLAMA_3_1_8B' | 'GEMMA_2_27B' | 'GEMMA_2_9B' | 'QWEN_2_5_72B' | 'QWEN_2_5_7B' +/** Define available Groq models. */ +export type GroqModelType = 'LLAMA_3_1_70B_VERSATILE' | 'LLAMA_3_1_8B_INSTANT' | 'LLAMA_3_2_1B_PREVIEW' | 'LLAMA_3_2_3B_PREVIEW' | 'MIXTRAL_8X7B_32768' /** Define local model configurations. */ export type LlamaModelType = 'QWEN_2_5_1B' | 'QWEN_2_5_3B' | 'PHI_3_5' | 'LLAMA_3_2_1B' | 'GEMMA_2_2B' /** Define local model with Ollama. */ @@ -403,6 +420,36 @@ export type TogetherResponse = { } } +export type GroqChatCompletionResponse = { + id: string + object: string + created: number // UNIX timestamp + model: string // e.g., "mixtral-8x7b-32768" + system_fingerprint: string | null // Nullable field + choices: { + index: number + message: { + role: 'assistant' | 'user' | 'system' // Role of the message author + content: string // The actual text of the message + } + finish_reason: string // Reason why the completion stopped, e.g., "stop" + logprobs?: { + tokens: string[] // Tokens generated by the model + token_logprobs: number[] // Log probabilities for each token + top_logprobs: Record[] // Top logprobs for the tokens + text_offset: number[] // Text offsets for the tokens + } | null // Optional logprobs object + }[] + usage?: { + prompt_tokens: number // Tokens used in the prompt + completion_tokens: number // Tokens used in the generated completion + total_tokens: number // Total tokens used + prompt_time?: number // Optional timing for the prompt + completion_time?: number // Optional timing for the completion + total_time?: number // Optional total time for both prompt and completion + } +} + // Define the expected structure of the response from Ollama API export type OllamaResponse = { model: string @@ -475,5 +522,6 @@ export type DeepgramResponse = { /** * Represents the function signature for cleaning up temporary files. + * @param id - The unique identifier for the temporary files. */ export type CleanUpFunction = (id: string) => Promise \ No newline at end of file diff --git a/src/utils/downloadAudio.ts b/src/utils/downloadAudio.ts index 80fca63..ac2f2df 100644 --- a/src/utils/downloadAudio.ts +++ b/src/utils/downloadAudio.ts @@ -4,7 +4,6 @@ import { exec, execFile } from 'node:child_process' import { promisify } from 'node:util' import { readFile, access } from 'node:fs/promises' import { fileTypeFromBuffer } from 'file-type' -import ffmpeg from 'ffmpeg-static' import { checkDependencies } from './checkDependencies.js' import { log, step, success, wait } from '../models.js' import type { SupportedFileType, ProcessingOptions } from '../types.js' @@ -76,7 +75,7 @@ export async function downloadAudio(options: ProcessingOptions, input: string, f // Convert the file to WAV format await execPromise( - `${ffmpeg} -i "${input}" -ar 16000 -ac 1 -vn "${outputPath}"` + `ffmpeg -i "${input}" -ar 16000 -ac 1 -c:a pcm_s16le "${outputPath}"` ) log(success(` File converted to WAV format successfully:\n - ${outputPath}`)) } catch (error) { diff --git a/src/utils/runLLM.ts b/src/utils/runLLM.ts index 1514142..d7d05ef 100644 --- a/src/utils/runLLM.ts +++ b/src/utils/runLLM.ts @@ -11,6 +11,7 @@ import { callMistral } from '../llms/mistral.js' import { callOcto } from '../llms/octo.js' import { callFireworks } from '../llms/fireworks.js' import { callTogether } from '../llms/together.js' +import { callGroq } from '../llms/groq.js' import { generatePrompt } from '../llms/prompt.js' import { log, step, success, wait } from '../models.js' import type { LLMServices, ProcessingOptions, LLMFunction, LLMFunctions } from '../types.js' @@ -42,6 +43,7 @@ export async function runLLM( octo: callOcto, fireworks: callFireworks, together: callTogether, + groq: callGroq, } try { diff --git a/test/all.test.js b/test/all.test.js deleted file mode 100644 index d3b7247..0000000 --- a/test/all.test.js +++ /dev/null @@ -1,222 +0,0 @@ -// test/all.test.js - -import test from 'node:test' -import { strictEqual } from 'node:assert/strict' -import { execSync } from 'node:child_process' -import { existsSync, renameSync } from 'node:fs' -import { join } from 'node:path' - -const commands = [ - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk"', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '01---ep0-fsjam-podcast-prompt.md' - }, - { - // Process all videos in a specified YouTube playlist. - cmd: 'npm run as -- --playlist "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr"', - expectedFiles: [ - { file: '2024-09-24-ep1-fsjam-podcast-prompt.md', newName: '02A---ep1-fsjam-podcast-prompt.md' }, - { file: '2024-09-24-ep0-fsjam-podcast-prompt.md', newName: '02B---ep0-fsjam-podcast-prompt.md' } - ] - }, - { - // Process playlist videos with custom title prompt, tiny Whisper model, and Llama for LLM processing. - cmd: 'npm run as -- --playlist "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr" --prompt titles --whisper tiny --llama', - expectedFiles: [ - { file: '2024-09-24-ep1-fsjam-podcast-llama-shownotes.md', newName: '03A---ep1-fsjam-podcast-llama-shownotes.md' }, - { file: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', newName: '03B---ep0-fsjam-podcast-llama-shownotes.md' } - ] - }, - { - // Process multiple YouTube videos from URLs listed in a file. - cmd: 'npm run as -- --urls "content/example-urls.md"', - expectedFiles: [ - { file: '2024-09-24-ep1-fsjam-podcast-prompt.md', newName: '04A---ep1-fsjam-podcast-prompt.md' }, - { file: '2024-09-24-ep0-fsjam-podcast-prompt.md', newName: '04B---ep0-fsjam-podcast-prompt.md' } - ] - }, - { - // Process multiple YouTube videos from URLs with title prompts, Whisper 'tiny' model, and Llama. - cmd: 'npm run as -- --urls "content/example-urls.md" --prompt titles --whisper tiny --llama', - expectedFiles: [ - { file: '2024-09-24-ep1-fsjam-podcast-llama-shownotes.md', newName: '05A---ep1-fsjam-podcast-llama-shownotes.md' }, - { file: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', newName: '05B---ep0-fsjam-podcast-llama-shownotes.md' } - ] - }, - { - // Process a single local audio file. - cmd: 'npm run as -- --file "content/audio.mp3"', - expectedFile: 'audio-prompt.md', - newName: '06---audio-prompt.md' - }, - { - // Process local audio file with title prompts, Whisper 'tiny' model, and Llama. - cmd: 'npm run as -- --file "content/audio.mp3" --prompt titles --whisper tiny --llama', - expectedFile: 'audio-llama-shownotes.md', - newName: '07---audio-llama-shownotes.md' - }, - { - // Process podcast RSS feed from default order. - cmd: 'npm run as -- --rss "https://ajcwebdev.substack.com/feed"', - expectedFile: '2021-05-10-thoughts-on-lambda-school-layoffs-prompt.md', - newName: '08---thoughts-on-lambda-school-layoffs-prompt.md' - }, - { - // Process a video using ChatGPT for LLM operations. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --chatgpt', - expectedFile: '2024-09-24-ep0-fsjam-podcast-chatgpt-shownotes.md', - newName: '09---ep0-fsjam-podcast-chatgpt-shownotes.md' - }, - { - // Process video with ChatGPT using GPT_4o_MINI model. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --chatgpt GPT_4o_MINI', - expectedFile: '2024-09-24-ep0-fsjam-podcast-chatgpt-shownotes.md', - newName: '10---ep0-fsjam-podcast-chatgpt-shownotes.md' - }, - { - // Process a video using Claude for LLM operations. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --claude', - expectedFile: '2024-09-24-ep0-fsjam-podcast-claude-shownotes.md', - newName: '11---ep0-fsjam-podcast-claude-shownotes.md' - }, - { - // Process video with Claude using CLAUDE_3_SONNET model. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --claude CLAUDE_3_SONNET', - expectedFile: '2024-09-24-ep0-fsjam-podcast-claude-shownotes.md', - newName: '12---ep0-fsjam-podcast-claude-shownotes.md' - }, - // { - // // Process a video using Gemini for LLM operations. - // cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --gemini', - // expectedFile: '2024-09-24-ep0-fsjam-podcast-gemini-shownotes.md', - // newName: '13---ep0-fsjam-podcast-gemini-shownotes.md' - // }, - // { - // // Process video with Gemini using GEMINI_1_5_FLASH model. - // cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --gemini GEMINI_1_5_FLASH', - // expectedFile: '2024-09-24-ep0-fsjam-podcast-gemini-shownotes.md', - // newName: '14---ep0-fsjam-podcast-gemini-shownotes.md' - // }, - { - // Process a video using Cohere for LLM operations - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --cohere', - expectedFile: '2024-09-24-ep0-fsjam-podcast-cohere-shownotes.md', - newName: '15---ep0-fsjam-podcast-cohere-shownotes.md' - }, - { - // Process video with Cohere using COMMAND_R_PLUS model. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --cohere COMMAND_R_PLUS', - expectedFile: '2024-09-24-ep0-fsjam-podcast-cohere-shownotes.md', - newName: '16---ep0-fsjam-podcast-cohere-shownotes.md' - }, - { - // Process a video using Mistral for LLM operations - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --mistral', - expectedFile: '2024-09-24-ep0-fsjam-podcast-mistral-shownotes.md', - newName: '17---ep0-fsjam-podcast-mistral-shownotes.md' - }, - { - // Process video with Mistral using MIXTRAL_8x7b model. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --mistral MIXTRAL_8x7b', - expectedFile: '2024-09-24-ep0-fsjam-podcast-mistral-shownotes.md', - newName: '18---ep0-fsjam-podcast-mistral-shownotes.md' - }, - { - // Process a video using OctoAI for LLM operations - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --octo', - expectedFile: '2024-09-24-ep0-fsjam-podcast-octo-shownotes.md', - newName: '19---ep0-fsjam-podcast-octo-shownotes.md' - }, - { - // Process video with Octo using LLAMA_3_1_8B model. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --octo LLAMA_3_1_8B', - expectedFile: '2024-09-24-ep0-fsjam-podcast-octo-shownotes.md', - newName: '20---ep0-fsjam-podcast-octo-shownotes.md' - }, - { - // Process a video using Llama for local LLM operations - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --llama', - expectedFile: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', - newName: '21---ep0-fsjam-podcast-llama-shownotes.md' - }, - { - // Process a video using Ollama for LLM operations - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --ollama', - expectedFile: '2024-09-24-ep0-fsjam-podcast-ollama-shownotes.md', - newName: '22---ep0-fsjam-podcast-ollama-shownotes.md' - }, - { - // Process a video using Deepgram for transcription - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --deepgram', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '23---ep0-fsjam-podcast-prompt.md' - }, - { - // Process video using Deepgram and Llama. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --deepgram --llama', - expectedFile: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', - newName: '24---ep0-fsjam-podcast-llama-shownotes.md' - }, - { - // Process a video using AssemblyAI for transcription - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --assembly', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '25---ep0-fsjam-podcast-prompt.md' - }, - { - // Process video using AssemblyAI and Llama. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --assembly --llama', - expectedFile: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', - newName: '26---ep0-fsjam-podcast-llama-shownotes.md' - }, - { - // Process an audio file using AssemblyAI with speaker labels - cmd: 'npm run as -- --video "https://ajc.pics/audio/fsjam-short.mp3" --assembly --speakerLabels', - expectedFile: '2024-05-08-fsjam-short-prompt.md', - newName: '27---fsjam-short-prompt.md' - }, - { - // Process video using Whisper.cpp in Docker with 'tiny' model. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperDocker tiny', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '28---ep0-fsjam-podcast-prompt.md' - }, - { - // Process a video with all available prompt options (except smallChapters and longChapters) - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --prompt titles summary shortChapters mediumChapters longChapters takeaways questions', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '29---ep0-fsjam-podcast-prompt.md' - }, - { - // Process video with multiple prompt sections, Whisper 'tiny' model, and Llama. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --prompt titles summary shortChapters takeaways questions --whisper tiny --llama', - expectedFile: '2024-09-24-ep0-fsjam-podcast-llama-shownotes.md', - newName: '30---ep0-fsjam-podcast-llama-shownotes.md' - } -] - -test('Autoshow Command Tests', async (t) => { - for (const [index, command] of commands.entries()) { - await t.test(`should run command ${index + 1} successfully`, async () => { - // Run the command - execSync(command.cmd, { stdio: 'inherit' }) - if (Array.isArray(command.expectedFiles)) { - for (const { file, newName } of command.expectedFiles) { - const filePath = join('content', file) - strictEqual(existsSync(filePath), true, `Expected file ${file} was not created`) - const newPath = join('content', newName) - renameSync(filePath, newPath) - strictEqual(existsSync(newPath), true, `File was not renamed to ${newName}`) - } - } else { - const filePath = join('content', command.expectedFile) - strictEqual(existsSync(filePath), true, `Expected file ${command.expectedFile} was not created`) - const newPath = join('content', command.newName) - renameSync(filePath, newPath) - strictEqual(existsSync(newPath), true, `File was not renamed to ${command.newName}`) - } - }) - } -}) \ No newline at end of file diff --git a/test/docker.test.ts b/test/docker.test.ts new file mode 100644 index 0000000..3882624 --- /dev/null +++ b/test/docker.test.ts @@ -0,0 +1,108 @@ +// test/docker.test.js + +import test from 'node:test' +import { strictEqual } from 'node:assert/strict' +import { execSync } from 'node:child_process' +import { existsSync, renameSync } from 'node:fs' +import { join } from 'node:path' + +const commands = [ + { + cmd: 'npm run docker -- --video "https://www.youtube.com/watch?v=MORMZXEaONk"', + expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', + newName: 'FILE_01.md' + }, + { + cmd: 'npm run docker -- --playlist "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr"', + expectedFiles: [ + { file: '2024-09-24-ep1-fsjam-podcast-prompt.md', newName: 'FILE_02A.md' }, + { file: '2024-09-24-ep0-fsjam-podcast-prompt.md', newName: 'FILE_02B.md' } + ] + }, + { + cmd: 'npm run docker -- --urls "content/example-urls.md"', + expectedFiles: [ + { file: '2024-09-24-ep1-fsjam-podcast-prompt.md', newName: 'FILE_03A.md' }, + { file: '2024-09-24-ep0-fsjam-podcast-prompt.md', newName: 'FILE_03B.md' } + ] + }, + { + cmd: 'npm run docker -- --file "content/audio.mp3"', + expectedFile: 'audio-prompt.md', + newName: 'FILE_04.md' + }, + { + cmd: 'npm run docker -- --file "content/audio.mp3" --prompt titles --whisperDocker tiny --ollama', + expectedFile: 'audio-ollama-shownotes.md', + newName: 'FILE_05.md' + }, + { + cmd: 'npm run docker -- --file "content/audio.mp3" --ollama LLAMA_3_2_3B', + expectedFile: 'audio-ollama-shownotes.md', + newName: 'FILE_06.md' + }, + { + cmd: 'npm run docker -- --file "content/audio.mp3" --whisperDocker tiny', + expectedFile: 'audio-prompt.md', + newName: 'FILE_07.md' + }, + { + cmd: 'npm run docker -- --file "content/audio.mp3" --prompt titles summary mediumChapters takeaways questions', + expectedFile: 'audio-prompt.md', + newName: 'FILE_08.md' + }, + { + cmd: 'npm run docker -- --file "content/audio.mp3" --prompt titles summary shortChapters takeaways questions --whisperDocker tiny --ollama', + expectedFile: 'audio-ollama-shownotes.md', + newName: 'FILE_09.md' + }, + { + cmd: 'npm run docker -- --playlist "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr" --prompt titles --whisperDocker tiny --ollama', + expectedFiles: [ + { file: '2024-09-24-ep1-fsjam-podcast-ollama-shownotes.md', newName: 'FILE_10A.md' }, + { file: '2024-09-24-ep0-fsjam-podcast-ollama-shownotes.md', newName: 'FILE_10B.md' } + ] + }, + { + cmd: 'npm run docker -- --urls "content/example-urls.md" --prompt titles --whisperDocker tiny --ollama', + expectedFiles: [ + { file: '2024-09-24-ep1-fsjam-podcast-ollama-shownotes.md', newName: 'FILE_11A.md' }, + { file: '2024-09-24-ep0-fsjam-podcast-ollama-shownotes.md', newName: 'FILE_11B.md' } + ] + }, + { + cmd: 'npm run docker -- --rss "https://ajcwebdev.substack.com/feed"', + expectedFile: '2021-05-10-thoughts-on-lambda-school-layoffs-prompt.md', + newName: 'FILE_12.md' + }, + { + cmd: 'npm run docker -- --rss "https://ajcwebdev.substack.com/feed" --info', + expectedFile: 'rss_info.json', + newName: 'FILE_13_rss_info.json', + } +] + +test('Autoshow Command Tests', async (t) => { + for (const [index, command] of commands.entries()) { + await t.test(`should run command ${index + 1} successfully`, async () => { + // Run the command + execSync(command.cmd, { stdio: 'inherit' }) + + if (Array.isArray(command.expectedFiles)) { + for (const { file, newName } of command.expectedFiles) { + const filePath = join('content', file) + strictEqual(existsSync(filePath), true, `Expected file ${file} was not created`) + const newPath = join('content', newName) + renameSync(filePath, newPath) + strictEqual(existsSync(newPath), true, `File was not renamed to ${newName}`) + } + } else { + const filePath = join('content', command.expectedFile as string) + strictEqual(existsSync(filePath), true, `Expected file ${command.expectedFile} was not created`) + const newPath = join('content', command.newName as string) + renameSync(filePath, newPath) + strictEqual(existsSync(newPath), true, `File was not renamed to ${command.newName}`) + } + }) + } +}) \ No newline at end of file diff --git a/test/integrations.test.ts b/test/integrations.test.ts new file mode 100644 index 0000000..0d45ad9 --- /dev/null +++ b/test/integrations.test.ts @@ -0,0 +1,144 @@ +// test/all.test.js + +import test from 'node:test' +import { strictEqual } from 'node:assert/strict' +import { execSync } from 'node:child_process' +import { existsSync, renameSync } from 'node:fs' +import { join } from 'node:path' + +const commands = [ + { + // Process multiple YouTube videos from URLs with title prompts, Whisper 'tiny' model, and ChatGPT GPT_4o_MINI model. + cmd: 'npm run as -- --urls "content/example-urls.md" --prompt titles --whisper tiny --chatgpt GPT_4o_MINI', + expectedFiles: [ + { file: '2024-09-24-ep1-fsjam-podcast-chatgpt-shownotes.md', newName: '01A---ep1-fsjam-podcast-chatgpt-shownotes.md' }, + { file: '2024-09-24-ep0-fsjam-podcast-chatgpt-shownotes.md', newName: '01B---ep0-fsjam-podcast-chatgpt-shownotes.md' } + ] + }, + { + // Process a video using ChatGPT for LLM operations. + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --chatgpt', + expectedFile: '2024-09-24-ep0-fsjam-podcast-chatgpt-shownotes.md', + newName: '02---ep0-fsjam-podcast-chatgpt-shownotes.md' + }, + { + // Process video with ChatGPT using GPT_4o_MINI model. + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --chatgpt GPT_4o_MINI', + expectedFile: '2024-09-24-ep0-fsjam-podcast-chatgpt-shownotes.md', + newName: '03---ep0-fsjam-podcast-chatgpt-shownotes.md' + }, + { + // Process a video using Claude for LLM operations. + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --claude', + expectedFile: '2024-09-24-ep0-fsjam-podcast-claude-shownotes.md', + newName: '04---ep0-fsjam-podcast-claude-shownotes.md' + }, + { + // Process video with Claude using CLAUDE_3_SONNET model. + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --claude CLAUDE_3_SONNET', + expectedFile: '2024-09-24-ep0-fsjam-podcast-claude-shownotes.md', + newName: '05---ep0-fsjam-podcast-claude-shownotes.md' + }, + { + // Process a video using Gemini for LLM operations. + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --gemini', + expectedFile: '2024-09-24-ep0-fsjam-podcast-gemini-shownotes.md', + newName: '06---ep0-fsjam-podcast-gemini-shownotes.md' + }, + { + // Process video with Gemini using GEMINI_1_5_FLASH model. + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --gemini GEMINI_1_5_FLASH', + expectedFile: '2024-09-24-ep0-fsjam-podcast-gemini-shownotes.md', + newName: '07---ep0-fsjam-podcast-gemini-shownotes.md' + }, + { + // Process a video using Cohere for LLM operations + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --cohere', + expectedFile: '2024-09-24-ep0-fsjam-podcast-cohere-shownotes.md', + newName: '08---ep0-fsjam-podcast-cohere-shownotes.md' + }, + { + // Process video with Cohere using COMMAND_R_PLUS model. + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --cohere COMMAND_R_PLUS', + expectedFile: '2024-09-24-ep0-fsjam-podcast-cohere-shownotes.md', + newName: '09---ep0-fsjam-podcast-cohere-shownotes.md' + }, + { + // Process a video using Mistral for LLM operations + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --mistral', + expectedFile: '2024-09-24-ep0-fsjam-podcast-mistral-shownotes.md', + newName: '10---ep0-fsjam-podcast-mistral-shownotes.md' + }, + { + // Process video with Mistral using MIXTRAL_8x7b model. + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --mistral MIXTRAL_8x7b', + expectedFile: '2024-09-24-ep0-fsjam-podcast-mistral-shownotes.md', + newName: '11---ep0-fsjam-podcast-mistral-shownotes.md' + }, + { + // Process a video using OctoAI for LLM operations + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --octo', + expectedFile: '2024-09-24-ep0-fsjam-podcast-octo-shownotes.md', + newName: '12---ep0-fsjam-podcast-octo-shownotes.md' + }, + { + // Process video with Octo using LLAMA_3_1_8B model. + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --octo LLAMA_3_1_8B', + expectedFile: '2024-09-24-ep0-fsjam-podcast-octo-shownotes.md', + newName: '13---ep0-fsjam-podcast-octo-shownotes.md' + }, + { + // Process a video using Deepgram for transcription + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --deepgram', + expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', + newName: '14---ep0-fsjam-podcast-prompt.md' + }, + { + // Process video using Deepgram and Llama. + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --deepgram --ollama', + expectedFile: '2024-09-24-ep0-fsjam-podcast-ollama-shownotes.md', + newName: '15---ep0-fsjam-podcast-ollama-shownotes.md' + }, + { + // Process a video using AssemblyAI for transcription + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --assembly', + expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', + newName: '16---ep0-fsjam-podcast-prompt.md' + }, + { + // Process video using AssemblyAI and Llama. + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --assembly --ollama', + expectedFile: '2024-09-24-ep0-fsjam-podcast-ollama-shownotes.md', + newName: '17---ep0-fsjam-podcast-ollama-shownotes.md' + }, + { + // Process an audio file using AssemblyAI with speaker labels + cmd: 'npm run as -- --video "https://ajc.pics/audio/fsjam-short.mp3" --assembly --speakerLabels', + expectedFile: '2024-05-08-fsjam-short-prompt.md', + newName: '18---fsjam-short-prompt.md' + } +] + +test('Autoshow Command Tests', async (t) => { + for (const [index, command] of commands.entries()) { + await t.test(`should run command ${index + 1} successfully`, async () => { + // Run the command + execSync(command.cmd, { stdio: 'inherit' }) + if (Array.isArray(command.expectedFiles)) { + for (const { file, newName } of command.expectedFiles) { + const filePath = join('content', file) + strictEqual(existsSync(filePath), true, `Expected file ${file} was not created`) + const newPath = join('content', newName) + renameSync(filePath, newPath) + strictEqual(existsSync(newPath), true, `File was not renamed to ${newName}`) + } + } else { + const filePath = join('content', command.expectedFile as string) + strictEqual(existsSync(filePath), true, `Expected file ${command.expectedFile} was not created`) + const newPath = join('content', command.newName as string) + renameSync(filePath, newPath) + strictEqual(existsSync(newPath), true, `File was not renamed to ${command.newName}`) + } + }) + } +}) \ No newline at end of file diff --git a/test/local.test.js b/test/local.test.ts similarity index 62% rename from test/local.test.js rename to test/local.test.ts index bc99858..eb32b95 100644 --- a/test/local.test.js +++ b/test/local.test.ts @@ -8,11 +8,13 @@ import { join } from 'node:path' const commands = [ { + // Process a single YouTube video using Autoshow's default settings. cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk"', expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', newName: 'FILE_01.md' }, { + // Process all videos in a specified YouTube playlist. cmd: 'npm run as -- --playlist "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr"', expectedFiles: [ { file: '2024-09-24-ep1-fsjam-podcast-prompt.md', newName: 'FILE_02A.md' }, @@ -20,6 +22,7 @@ const commands = [ ] }, { + // Process multiple YouTube videos from URLs listed in a file. cmd: 'npm run as -- --urls "content/example-urls.md"', expectedFiles: [ { file: '2024-09-24-ep1-fsjam-podcast-prompt.md', newName: 'FILE_03A.md' }, @@ -27,63 +30,68 @@ const commands = [ ] }, { + // Process a single local audio file. cmd: 'npm run as -- --file "content/audio.mp3"', expectedFile: 'audio-prompt.md', newName: 'FILE_04.md' }, { + // Process local audio file with title prompts, Whisper 'tiny' model, and Llama. cmd: 'npm run as -- --file "content/audio.mp3" --prompt titles --whisper tiny --llama', expectedFile: 'audio-llama-shownotes.md', newName: 'FILE_05.md' }, { - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --ollama LLAMA_3_2_3B', - expectedFile: '2024-09-24-ep0-fsjam-podcast-ollama-shownotes.md', + // Process a local audio file with Ollama using LLAMA_3_2_3B model. + cmd: 'npm run as -- --file "content/audio.mp3" --ollama LLAMA_3_2_3B', + expectedFile: 'audio-ollama-shownotes.md', newName: 'FILE_06.md' }, { - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisper tiny', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', + // Process local audio file with Whisper 'tiny' model. + cmd: 'npm run as -- --file "content/audio.mp3" --whisper tiny', + expectedFile: 'audio-prompt.md', newName: 'FILE_07.md' }, - // { - // cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperDocker tiny', - // expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - // newName: 'FILE_08.md' - // }, { - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --prompt titles summary mediumChapters takeaways questions', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: 'FILE_09.md' + // Process a local audio file with all available prompt options (except smallChapters and longChapters) + cmd: 'npm run as -- --file "content/audio.mp3" --prompt titles summary mediumChapters takeaways questions', + expectedFile: 'audio-prompt.md', + newName: 'FILE_08.md' }, { - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --prompt titles summary shortChapters takeaways questions --whisper tiny --ollama', - expectedFile: '2024-09-24-ep0-fsjam-podcast-ollama-shownotes.md', - newName: 'FILE_10.md' + // Process a local audio file with multiple prompt sections, Whisper 'tiny' model, and Ollama. + cmd: 'npm run as -- --file "content/audio.mp3" --prompt titles summary shortChapters takeaways questions --whisper tiny --ollama', + expectedFile: 'audio-ollama-shownotes.md', + newName: 'FILE_09.md' }, { - cmd: 'npm run as -- --playlist "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr" --prompt titles --whisper tiny --ollama', + // Process playlist videos with titles and longChapters prompts, tiny Whisper model, and Ollama for LLM processing. + cmd: 'npm run as -- --playlist "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr" --prompt titles longChapters --whisper tiny --ollama', expectedFiles: [ - { file: '2024-09-24-ep1-fsjam-podcast-ollama-shownotes.md', newName: 'FILE_11A.md' }, - { file: '2024-09-24-ep0-fsjam-podcast-ollama-shownotes.md', newName: 'FILE_11B.md' } + { file: '2024-09-24-ep1-fsjam-podcast-ollama-shownotes.md', newName: 'FILE_10A.md' }, + { file: '2024-09-24-ep0-fsjam-podcast-ollama-shownotes.md', newName: 'FILE_10B.md' } ] }, { + // Process multiple YouTube videos from URLs with title prompts, Whisper 'tiny' model, and Ollama. cmd: 'npm run as -- --urls "content/example-urls.md" --prompt titles --whisper tiny --ollama', expectedFiles: [ - { file: '2024-09-24-ep1-fsjam-podcast-ollama-shownotes.md', newName: 'FILE_12A.md' }, - { file: '2024-09-24-ep0-fsjam-podcast-ollama-shownotes.md', newName: 'FILE_12B.md' } + { file: '2024-09-24-ep1-fsjam-podcast-ollama-shownotes.md', newName: 'FILE_11A.md' }, + { file: '2024-09-24-ep0-fsjam-podcast-ollama-shownotes.md', newName: 'FILE_11B.md' } ] }, { + // Process podcast RSS feed from default order. cmd: 'npm run as -- --rss "https://ajcwebdev.substack.com/feed"', expectedFile: '2021-05-10-thoughts-on-lambda-school-layoffs-prompt.md', - newName: 'FILE_13.md' + newName: 'FILE_12.md' }, { + // Download JSON file with metadata for each item in the RSS feed. cmd: 'npm run as -- --rss "https://ajcwebdev.substack.com/feed" --info', expectedFile: 'rss_info.json', - newName: 'FILE_14_rss_info.json', + newName: 'FILE_13_rss_info.json', } ] @@ -102,9 +110,9 @@ test('Autoshow Command Tests', async (t) => { strictEqual(existsSync(newPath), true, `File was not renamed to ${newName}`) } } else { - const filePath = join('content', command.expectedFile) + const filePath = join('content', command.expectedFile as string) strictEqual(existsSync(filePath), true, `Expected file ${command.expectedFile} was not created`) - const newPath = join('content', command.newName) + const newPath = join('content', command.newName as string) renameSync(filePath, newPath) strictEqual(existsSync(newPath), true, `File was not renamed to ${command.newName}`) } diff --git a/typedoc.json b/typedoc.json new file mode 100644 index 0000000..796849d --- /dev/null +++ b/typedoc.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://typedoc.org/schema.json", + "entryPoints": ["src/types.ts", "src/autoshow.ts"], + "out": "out", + "tsconfig": "tsconfig.json", + "excludePrivate": true, + "excludeProtected": true, + "includeVersion": true, + "categorizeByGroup": false, + "theme": "default", + "disableSources": false +} \ No newline at end of file