diff --git a/.env.example b/.env.example index aabbf68..31439ac 100644 --- a/.env.example +++ b/.env.example @@ -3,9 +3,9 @@ ANTHROPIC_API_KEY="" GEMINI_API_KEY="" COHERE_API_KEY="" MISTRAL_API_KEY="" -OCTOAI_API_KEY="" TOGETHER_API_KEY="" FIREWORKS_API_KEY="" +GROQ_API_KEY="" DEEPGRAM_API_KEY="" ASSEMBLY_API_KEY="" \ No newline at end of file diff --git a/.github/llama.Dockerfile b/.github/llama.Dockerfile deleted file mode 100644 index d070f71..0000000 --- a/.github/llama.Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# .github/llama.cpp/Dockerfile - -FROM --platform=linux/arm64 ubuntu:22.04 AS build - -WORKDIR /app - -# Install build dependencies -RUN apt-get update && \ - apt-get install -y build-essential libopenblas-dev pkg-config git curl && \ - rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* - -# Copy the local llama.cpp directory into the Docker image -COPY . . - -# Clean any previous builds -RUN make clean - -# Build llama.cpp with OpenBLAS support -RUN make LLAMA_OPENBLAS=1 - -FROM --platform=linux/arm64 ubuntu:22.04 AS runtime - -WORKDIR /app - -# Install runtime dependencies -RUN apt-get update && \ - apt-get install -y curl libopenblas-dev && \ - rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* - -# Copy the built llama.cpp binaries from the build stage -COPY --from=build /app /app - -ENTRYPOINT [ "bash", "-c" ] \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8720a32..941a324 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ types dist NEW.md TODO.md -nemo_msdd_configs \ No newline at end of file +nemo_msdd_configs +temp_outputs \ No newline at end of file diff --git a/README.md b/README.md index 144f9a1..c107ff5 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,6 @@ - [Project Overview](#project-overview) - [Key Features](#key-features) - [Setup](#setup) - - [Copy Environment Variable File](#copy-environment-variable-file) - - [Install Local Dependencies](#install-local-dependencies) - - [Clone Whisper Repo](#clone-whisper-repo) - [Run Autoshow Node Scripts](#run-autoshow-node-scripts) - [Project Structure](#project-structure) @@ -29,8 +26,10 @@ The Autoshow workflow includes the following steps: ### Key Features - Support for multiple input types (YouTube links, RSS feeds, local video and audio files) -- Integration with various LLMs (ChatGPT, Claude, Cohere, Mistral) and transcription services (Whisper.cpp, Deepgram, Assembly) -- Local LLM support (Llama 3.1, Phi 3, Qwen 2, Mistral) +- Integration with various: + - LLMs (ChatGPT, Claude, Gemini, Cohere, Mistral, Fireworks, Together, Groq) + - Transcription services (Whisper.cpp, Deepgram, Assembly) +- Local LLM support with Ollama - Customizable prompts for generating titles, summaries, chapter titles/descriptions, key takeaways, and questions to test comprehension - Markdown output with metadata and formatted content - Command-line interface for easy usage @@ -40,36 +39,12 @@ See [`docs/roadmap.md`](/docs/roadmap.md) for details about current development ## Setup -### Copy Environment Variable File - -`npm run autoshow` expects a `.env` file even for commands that don't require API keys. You can create a blank `.env` file or use the default provided: - -```bash -cp .env.example .env -``` - -### Install Local Dependencies - -Install `yt-dlp`, `ffmpeg`, and run `npm i`. +`scripts/setup.sh` checks to ensure a `.env` file exists, Node dependencies are installed, and the `whisper.cpp` repository is cloned and built. Run the script with the `setup` script in `package.json`. ```bash -brew install yt-dlp ffmpeg -npm i +npm run setup ``` -### Clone Whisper Repo - -Run the following commands to clone `whisper.cpp` and build the `base` model: - -```bash -git clone https://github.com/ggerganov/whisper.cpp.git && \ - bash ./whisper.cpp/models/download-ggml-model.sh base && \ - make -C whisper.cpp && \ - cp .github/whisper.Dockerfile whisper.cpp/Dockerfile -``` - -> Replace `base` with `large-v2` for the largest model, `medium` for a middle sized model, or `tiny` for the smallest model. - ## Run Autoshow Node Scripts Run on a single YouTube video. @@ -105,7 +80,7 @@ npm run as -- --rss "https://ajcwebdev.substack.com/feed" Use local LLM. ```bash -npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --llama +npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --ollama ``` Use 3rd party LLM providers. @@ -116,45 +91,49 @@ npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --claude CLA npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --gemini GEMINI_1_5_PRO npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --cohere COMMAND_R_PLUS npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --mistral MISTRAL_LARGE -npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --octo LLAMA_3_1_405B +npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --fireworks +npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --together +npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --groq ``` Example commands for all available CLI options can be found in [`docs/examples.md`](/docs/examples.md). ## Project Structure -- Main Entry Point (`src/autoshow.js`) +- Main Entry Point (`src/autoshow.ts`) - Defines the command-line interface using Commander.js - Handles various input options (video, playlist, URLs, file, RSS) - Manages LLM and transcription options - Command Processors (`src/commands`) - - `processVideo.js`: Handles single YouTube video processing - - `processPlaylist.js`: Processes all videos in a YouTube playlist - - `processURLs.js`: Processes videos from a list of URLs in a file - - `processFile.js`: Handles local audio/video file processing - - `processRSS.js`: Processes podcast RSS feeds + - `processVideo.ts`: Handles single YouTube video processing + - `processPlaylist.ts`: Processes all videos in a YouTube playlist + - `processURLs.ts`: Processes videos from a list of URLs in a file + - `processFile.ts`: Handles local audio/video file processing + - `processRSS.ts`: Processes podcast RSS feeds - Utility Functions (`src/utils`) - - `downloadAudio.js`: Downloads audio from YouTube videos - - `runTranscription.js`: Manages the transcription process - - `runLLM.js`: Handles LLM processing for summarization and chapter generation - - `generateMarkdown.js`: Creates initial markdown files with metadata - - `cleanUpFiles.js`: Removes temporary files after processing + - `downloadAudio.ts`: Downloads audio from YouTube videos + - `runTranscription.ts`: Manages the transcription process + - `runLLM.ts`: Handles LLM processing for summarization and chapter generation + - `generateMarkdown.ts`: Creates initial markdown files with metadata + - `cleanUpFiles.ts`: Removes temporary files after processing - Transcription Services (`src/transcription`) - - `whisper.js`: Uses Whisper.cpp for transcription - - `deepgram.js`: Integrates Deepgram transcription service - - `assembly.js`: Integrates AssemblyAI transcription service + - `whisper.ts`: Uses Whisper.cpp, openai-whisper, or whisper-diarization for transcription + - `deepgram.ts`: Integrates Deepgram transcription service + - `assembly.ts`: Integrates AssemblyAI transcription service - Language Models (`src/llms`) - - `chatgpt.js`: Integrates OpenAI's GPT models - - `claude.js`: Integrates Anthropic's Claude models - - `cohere.js`: Integrates Cohere's language models - - `mistral.js`: Integrates Mistral AI's language models - - `octo.js`: Integrates OctoAI's language models - - `llama.js`: Integrates Llama models (local inference) - - `prompt.js`: Defines the prompt structure for summarization and chapter generation + - `chatgpt.ts`: Integrates OpenAI's GPT models + - `claude.ts`: Integrates Anthropic's Claude models + - `gemini.ts`: Integrates Google's Gemini models + - `cohere.ts`: Integrates Cohere's language models + - `mistral.ts`: Integrates Mistral AI's language models + - `fireworks.ts`: Integrates Fireworks's open source models + - `together.ts`: Integrates Together's open source models + - `groq.ts`: Integrates Groq's open source models + - `prompt.ts`: Defines the prompt structure for summarization and chapter generation - Web Interface (`web`) and Server (`server`) - Web interface built with React and Vite diff --git a/docs/examples.md b/docs/examples.md index fc4b3ba..10f8d9c 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -143,9 +143,12 @@ Create a `.env` file and set API key as demonstrated in `.env.example` for eithe - `OPENAI_API_KEY` - `ANTHROPIC_API_KEY` +- `GEMINI_API_KEY` - `COHERE_API_KEY` - `MISTRAL_API_KEY` -- `OCTOAI_API_KEY` +- `TOGETHER_API_KEY` +- `FIREWORKS_API_KEY` +- `GROQ_API_KEY` For each model available for each provider, I have collected the following details: @@ -401,34 +404,6 @@ npm run as -- \ --groq MIXTRAL_8X7B_32768 ``` -### Llama.cpp - -```bash -npm run as -- \ - --video "https://www.youtube.com/watch?v=MORMZXEaONk" \ - --llama -``` - -Select Llama model: - -```bash -npm run as -- \ - --video "https://www.youtube.com/watch?v=MORMZXEaONk" \ - --llama GEMMA_2_2B - -npm run as -- \ - --video "https://www.youtube.com/watch?v=MORMZXEaONk" \ - --llama LLAMA_3_2_1B - -npm run as -- \ - --video "https://www.youtube.com/watch?v=MORMZXEaONk" \ - --llama PHI_3_5 - -npm run as -- \ - --video "https://www.youtube.com/watch?v=MORMZXEaONk" \ - --llama QWEN_2_5_3B -``` - ### Ollama ```bash @@ -644,7 +619,7 @@ This will start `whisper.cpp`, Ollama, and the AutoShow Commander CLI in their o npm run docker-up ``` -Replace `as` with `docker` to run most other commands explained in this document. Does not support all options at this time, notably `--llama`, `--whisperPython`, and `--whisperDiarization`. +Replace `as` with `docker` to run most other commands explained in this document. Does not support all options at this time, notably `--whisperPython` and `--whisperDiarization`. ```bash npm run docker -- \ diff --git a/docs/server.md b/docs/server.md index 15409f6..1c9769c 100644 --- a/docs/server.md +++ b/docs/server.md @@ -43,7 +43,7 @@ Use LLM. curl --json '{ "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk", "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" }' http://localhost:3000/video ``` @@ -59,7 +59,7 @@ curl --json '{ curl --json '{ "playlistUrl": "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr", "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" }' http://localhost:3000/playlist ``` @@ -75,7 +75,7 @@ curl --json '{ curl --json '{ "filePath": "content/example-urls.md", "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" }' http://localhost:3000/urls ``` @@ -84,7 +84,7 @@ curl --json '{ "filePath": "content/example-urls.md", "prompts": ["titles", "mediumChapters"], "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" }' http://localhost:3000/urls ``` @@ -100,7 +100,7 @@ curl --json '{ curl --json '{ "filePath": "content/audio.mp3", "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" }' http://localhost:3000/file ``` @@ -109,7 +109,7 @@ curl --json '{ "filePath": "content/audio.mp3", "prompts": ["titles"], "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" }' http://localhost:3000/file ``` @@ -125,7 +125,7 @@ curl --json '{ curl --json '{ "rssUrl": "https://feeds.transistor.fm/fsjam-podcast/", "whisperModel": "tiny", - "llm": "llama", + "llm": "ollama", "order": "newest", "skip": 0 }' http://localhost:3000/rss @@ -236,23 +236,6 @@ curl --json '{ }' http://localhost:3000/video ``` -### Octo - -```bash -curl --json '{ - "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk", - "llm": "octo" -}' http://localhost:3000/video -``` - -```bash -curl --json '{ - "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk", - "llm": "octo", - "llmModel": "LLAMA_3_1_8B" -}' http://localhost:3000/video -``` - ## Transcription Options ### Whisper.cpp @@ -277,7 +260,7 @@ curl --json '{ curl --json '{ "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk", "transcriptServices": "deepgram", - "llm": "llama" + "llm": "ollama" }' http://localhost:3000/video ``` @@ -294,7 +277,7 @@ curl --json '{ curl --json '{ "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk", "transcriptServices": "assembly", - "llm": "llama" + "llm": "ollama" }' http://localhost:3000/video ``` @@ -311,7 +294,7 @@ curl --json '{ "youtubeUrl": "https://ajc.pics/audio/fsjam-short.mp3", "transcriptServices": "assembly", "speakerLabels": true, - "llm": "llama" + "llm": "ollama" }' http://localhost:3000/video ``` @@ -336,7 +319,7 @@ curl --json '{ "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk", "prompts": ["titles", "summary", "shortChapters", "takeaways", "questions"], "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" }' http://localhost:3000/video ``` @@ -345,7 +328,7 @@ curl --json '{ "playlistUrl": "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr", "prompts": ["titles", "mediumChapters"], "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" }' http://localhost:3000/playlist ``` @@ -354,6 +337,6 @@ curl --json '{ "playlistUrl": "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr", "prompts": ["titles", "mediumChapters"], "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" }' http://localhost:3000/playlist ``` \ No newline at end of file diff --git a/package.json b/package.json index e14dd55..c722f9e 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,9 @@ "ChatGPT", "Cohere", "Mistral", - "OctoAI", + "Fireworks", + "Together", + "Groq", "Whisper", "Deepgram", "AssemblyAI" @@ -52,7 +54,6 @@ "@fastify/cors": "10.0.1", "@google/generative-ai": "0.21.0", "@mistralai/mistralai": "1.1.0", - "@octoai/sdk": "1.11.0", "assemblyai": "4.7.1", "chalk": "5.3.0", "cohere-ai": "7.14.0", @@ -61,7 +62,6 @@ "fastify": "5.0.0", "file-type": "19.6.0", "inquirer": "12.0.1", - "node-llama-cpp": "3.1.1", "ollama": "0.5.9", "openai": "4.68.4" }, diff --git a/packages/server/README.md b/packages/server/README.md index 5652a9b..7c265bd 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -17,14 +17,14 @@ const TEST_REQ_02 = { const TEST_REQ_03 = { "playlistUrl": "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr", "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" } const TEST_REQ_04 = { "playlistUrl": "https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr", "prompts": ["titles", "mediumChapters"], "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" } ``` @@ -45,14 +45,14 @@ const TEST_REQ_06 = { const TEST_REQ_07 = { "filePath": "content/example-urls.md", "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" } const TEST_REQ_08 = { "filePath": "content/example-urls.md", "prompts": ["titles", "mediumChapters"], "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" } ``` @@ -73,14 +73,14 @@ const TEST_REQ_10 = { const TEST_REQ_11 = { "filePath": "content/audio.mp3", "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" } const TEST_REQ_12 = { "filePath": "content/audio.mp3", "prompts": ["titles"], "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" } ``` @@ -101,7 +101,7 @@ const TEST_REQ_14 = { const TEST_REQ_15 = { "rssUrl": "https://feeds.transistor.fm/fsjam-podcast/", "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" } const TEST_REQ_16 = { @@ -131,7 +131,7 @@ const TEST_REQ_18 = { const TEST_REQ_19 = { "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk", "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" } const TEST_REQ_20 = { @@ -189,17 +189,6 @@ const TEST_REQ_29 = { "llmModel": "MIXTRAL_8x7b" } -const TEST_REQ_30 = { - "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk", - "llm": "octo" -} - -const TEST_REQ_31 = { - "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk", - "llm": "octo", - "llmModel": "LLAMA_3_1_8B" -} - const TEST_REQ_32 = { "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk", "whisperModel": "tiny" @@ -213,7 +202,7 @@ const TEST_REQ_33 = { const TEST_REQ_34 = { "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk", "transcriptServices": "deepgram", - "llm": "llama" + "llm": "ollama" } const TEST_REQ_35 = { @@ -224,7 +213,7 @@ const TEST_REQ_35 = { const TEST_REQ_36 = { "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk", "transcriptServices": "assembly", - "llm": "llama" + "llm": "ollama" } const TEST_REQ_37 = { @@ -237,7 +226,7 @@ const TEST_REQ_38 = { "youtubeUrl": "https://ajc.pics/audio/fsjam-short.mp3", "transcriptServices": "assembly", "speakerLabels": true, - "llm": "llama" + "llm": "ollama" } const TEST_REQ_39 = { @@ -254,6 +243,6 @@ const TEST_REQ_41 = { "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk", "prompts": ["titles", "summary", "shortChapters", "takeaways", "questions"], "whisperModel": "tiny", - "llm": "llama" + "llm": "ollama" } ``` \ No newline at end of file diff --git a/packages/server/index.js b/packages/server/index.ts similarity index 84% rename from packages/server/index.js rename to packages/server/index.ts index 42c869a..91039d4 100644 --- a/packages/server/index.js +++ b/packages/server/index.ts @@ -7,10 +7,11 @@ import { handlePlaylistRequest } from './routes/playlist.js' import { handleURLsRequest } from './routes/urls.js' import { handleFileRequest } from './routes/file.js' import { handleRSSRequest } from './routes/rss.js' +import { l } from '../../src/globals.js' import { env } from 'node:process' // Set the port from environment variable or default to 3000 -const port = env.PORT || 3000 +const port = Number(env.PORT) || 3000 async function start() { // Create a Fastify instance with logging enabled @@ -25,7 +26,7 @@ async function start() { // Log each incoming request fastify.addHook('onRequest', async (request, reply) => { - console.log( + l( `\n[${new Date().toISOString()}] Received ${request.method} request for ${request.url}\n` ) }) @@ -38,9 +39,11 @@ async function start() { fastify.post('/rss', handleRSSRequest) try { + // Start the server and listen on the specified port await fastify.listen({ port }) - console.log(`\nServer running at http://localhost:${port}\n`) + l(`\nServer running at http://localhost:${port}\n`) } catch (err) { + // Log errors and exit if the server fails to start fastify.log.error(err) process.exit(1) } diff --git a/packages/server/routes/file.js b/packages/server/routes/file.js deleted file mode 100644 index b7a44f7..0000000 --- a/packages/server/routes/file.js +++ /dev/null @@ -1,38 +0,0 @@ -// server/routes/file.js - -import { processFile } from '../../../src/commands/processFile.js' -import { reqToOpts } from '../utils/reqToOpts.js' - -// Handler for /file route -const handleFileRequest = async (request, reply) => { - console.log('\nEntered handleFileRequest') - - try { - // Access parsed request body - const requestData = request.body - console.log('\nParsed request body:', requestData) - - // Extract file path - const { filePath } = requestData - - if (!filePath) { - console.log('File path not provided, sending 400') - reply.status(400).send({ error: 'File path is required' }) - return - } - - // Map request data to processing options - const { options, llmServices, transcriptServices } = reqToOpts(requestData) - console.log('\nCalling processFile with params:', { filePath, llmServices, transcriptServices, options }) - - await processFile(filePath, llmServices, transcriptServices, options) - - console.log('\nprocessFile completed successfully') - reply.send({ message: 'File processed successfully.' }) - } catch (error) { - console.error('Error processing file:', error) - reply.status(500).send({ error: 'An error occurred while processing the file' }) - } -} - -export { handleFileRequest } \ No newline at end of file diff --git a/packages/server/routes/file.ts b/packages/server/routes/file.ts new file mode 100644 index 0000000..7b51fe5 --- /dev/null +++ b/packages/server/routes/file.ts @@ -0,0 +1,51 @@ +// server/routes/file.ts + +import { FastifyRequest, FastifyReply } from 'fastify' +import { processFile } from '../../../src/commands/processFile.js' +import { reqToOpts } from '../utils/reqToOpts.js' +import { l, err } from '../../../src/globals.js' + +// Handler for the /file route +export const handleFileRequest = async ( + request: FastifyRequest, + reply: FastifyReply +): Promise => { + l('\nEntered handleFileRequest') + + try { + // Access parsed request body + const requestData = request.body as any + l('\nParsed request body:', requestData) + + // Extract file path from the request data + const { filePath } = requestData + + if (!filePath) { + l('File path not provided, sending 400') + reply.status(400).send({ error: 'File path is required' }) + return + } + + // Map request data to processing options + const { options, llmServices, transcriptServices } = reqToOpts(requestData) + + // Set options.file to filePath + options.file = filePath + + l('\nCalling processFile with params:', { + filePath, + llmServices, + transcriptServices, + options, + }) + + // Call processFile with the mapped options and extracted file path + await processFile(options, filePath, llmServices, transcriptServices) + + l('\nprocessFile completed successfully') + reply.send({ message: 'File processed successfully.' }) + } catch (error) { + err('Error processing file:', error) + reply.status(500).send({ error: 'An error occurred while processing the file' }) + } +} \ No newline at end of file diff --git a/packages/server/routes/playlist.js b/packages/server/routes/playlist.js deleted file mode 100644 index e1c32dc..0000000 --- a/packages/server/routes/playlist.js +++ /dev/null @@ -1,38 +0,0 @@ -// server/routes/playlist.js - -import { processPlaylist } from '../../../src/commands/processPlaylist.js' -import { reqToOpts } from '../utils/reqToOpts.js' - -// Handler for /playlist route -const handlePlaylistRequest = async (request, reply) => { - console.log('\nEntered handlePlaylistRequest') - - try { - // Access parsed request body - const requestData = request.body - console.log('\nParsed request body:', requestData) - - // Extract playlist URL - const { playlistUrl } = requestData - - if (!playlistUrl) { - console.log('Playlist URL not provided, sending 400') - reply.status(400).send({ error: 'Playlist URL is required' }) - return - } - - // Map request data to processing options - const { options, llmServices, transcriptServices } = reqToOpts(requestData) - console.log('\nCalling processPlaylist with params:', { playlistUrl, llmServices, transcriptServices, options }) - - await processPlaylist(playlistUrl, llmServices, transcriptServices, options) - - console.log('\nprocessPlaylist completed successfully') - reply.send({ message: 'Playlist processed successfully.' }) - } catch (error) { - console.error('Error processing playlist:', error) - reply.status(500).send({ error: 'An error occurred while processing the playlist' }) - } -} - -export { handlePlaylistRequest } \ No newline at end of file diff --git a/packages/server/routes/playlist.ts b/packages/server/routes/playlist.ts new file mode 100644 index 0000000..7de53ed --- /dev/null +++ b/packages/server/routes/playlist.ts @@ -0,0 +1,51 @@ +// server/routes/playlist.ts + +import { FastifyRequest, FastifyReply } from 'fastify' +import { processPlaylist } from '../../../src/commands/processPlaylist.js' +import { reqToOpts } from '../utils/reqToOpts.js' +import { l, err } from '../../../src/globals.js' + +// Handler for the /playlist route +export const handlePlaylistRequest = async ( + request: FastifyRequest, + reply: FastifyReply +): Promise => { + l('\nEntered handlePlaylistRequest') + + try { + // Access parsed request body + const requestData = request.body as any + l('\nParsed request body:', requestData) + + // Extract playlist URL from the request data + const { playlistUrl } = requestData + + if (!playlistUrl) { + l('Playlist URL not provided, sending 400') + reply.status(400).send({ error: 'Playlist URL is required' }) + return + } + + // Map request data to processing options + const { options, llmServices, transcriptServices } = reqToOpts(requestData) + + // Set options.playlist to playlistUrl + options.playlist = playlistUrl + + l('\nCalling processPlaylist with params:', { + playlistUrl, + llmServices, + transcriptServices, + options, + }) + + // Call processPlaylist with the mapped options and extracted playlist URL + await processPlaylist(options, playlistUrl, llmServices, transcriptServices) + + l('\nprocessPlaylist completed successfully') + reply.send({ message: 'Playlist processed successfully.' }) + } catch (error) { + err('Error processing playlist:', error) + reply.status(500).send({ error: 'An error occurred while processing the playlist' }) + } +} \ No newline at end of file diff --git a/packages/server/routes/rss.js b/packages/server/routes/rss.js deleted file mode 100644 index 4e28776..0000000 --- a/packages/server/routes/rss.js +++ /dev/null @@ -1,38 +0,0 @@ -// server/routes/rss.js - -import { processRSS } from '../../../src/commands/processRSS.js' -import { reqToOpts } from '../utils/reqToOpts.js' - -// Handler for /rss route -const handleRSSRequest = async (request, reply) => { - console.log('\nEntered handleRSSRequest') - - try { - // Access parsed request body - const requestData = request.body - console.log('\nParsed request body:', requestData) - - // Extract RSS URL - const { rssUrl } = requestData - - if (!rssUrl) { - console.log('RSS URL not provided, sending 400') - reply.status(400).send({ error: 'RSS URL is required' }) - return - } - - // Map request data to processing options - const { options, llmServices, transcriptServices } = reqToOpts(requestData) - console.log('\nCalling processRSS with params:', { rssUrl, llmServices, transcriptServices, options }) - - await processRSS(rssUrl, llmServices, transcriptServices, options) - - console.log('\nprocessRSS completed successfully') - reply.send({ message: 'RSS feed processed successfully.' }) - } catch (error) { - console.error('Error processing RSS request:', error) - reply.status(500).send({ error: 'An error occurred while processing the RSS feed' }) - } -} - -export { handleRSSRequest } \ No newline at end of file diff --git a/packages/server/routes/rss.ts b/packages/server/routes/rss.ts new file mode 100644 index 0000000..be78aa5 --- /dev/null +++ b/packages/server/routes/rss.ts @@ -0,0 +1,51 @@ +// server/routes/rss.ts + +import { FastifyRequest, FastifyReply } from 'fastify' +import { processRSS } from '../../../src/commands/processRSS.js' +import { reqToOpts } from '../utils/reqToOpts.js' +import { l, err } from '../../../src/globals.js' + +// Handler for the /rss route +export const handleRSSRequest = async ( + request: FastifyRequest, + reply: FastifyReply +): Promise => { + l('\nEntered handleRSSRequest') + + try { + // Access parsed request body + const requestData = request.body as any + l('\nParsed request body:', requestData) + + // Extract RSS URL from the request data + const { rssUrl } = requestData + + if (!rssUrl) { + l('RSS URL not provided, sending 400') + reply.status(400).send({ error: 'RSS URL is required' }) + return + } + + // Map request data to processing options + const { options, llmServices, transcriptServices } = reqToOpts(requestData) + + // Set options.rss to rssUrl + options.rss = rssUrl + + l('\nCalling processRSS with params:', { + rssUrl, + llmServices, + transcriptServices, + options, + }) + + // Call processRSS with the mapped options and extracted RSS URL + await processRSS(options, rssUrl, llmServices, transcriptServices) + + l('\nprocessRSS completed successfully') + reply.send({ message: 'RSS feed processed successfully.' }) + } catch (error) { + err('Error processing RSS request:', error) + reply.status(500).send({ error: 'An error occurred while processing the RSS feed' }) + } +} \ No newline at end of file diff --git a/packages/server/routes/urls.js b/packages/server/routes/urls.js deleted file mode 100644 index 45e49ef..0000000 --- a/packages/server/routes/urls.js +++ /dev/null @@ -1,38 +0,0 @@ -// server/routes/urls.js - -import { processURLs } from '../../../src/commands/processURLs.js' -import { reqToOpts } from '../utils/reqToOpts.js' - -// Handler for /urls route -const handleURLsRequest = async (request, reply) => { - console.log('\nEntered handleURLsRequest') - - try { - // Access parsed request body - const requestData = request.body - console.log('\nParsed request body:', requestData) - - // Extract file path - const { filePath } = requestData - - if (!filePath) { - console.log('File path not provided, sending 400') - reply.status(400).send({ error: 'File path is required' }) - return - } - - // Map request data to processing options - const { options, llmServices, transcriptServices } = reqToOpts(requestData) - console.log('\nCalling processURLs with params:', { filePath, llmServices, transcriptServices, options }) - - await processURLs(filePath, llmServices, transcriptServices, options) - - console.log('\nprocessURLs completed successfully') - reply.send({ message: 'URLs processed successfully.' }) - } catch (error) { - console.error('Error processing URLs:', error) - reply.status(500).send({ error: 'An error occurred while processing the URLs' }) - } -} - -export { handleURLsRequest } \ No newline at end of file diff --git a/packages/server/routes/urls.ts b/packages/server/routes/urls.ts new file mode 100644 index 0000000..12c3113 --- /dev/null +++ b/packages/server/routes/urls.ts @@ -0,0 +1,51 @@ +// server/routes/urls.ts + +import { FastifyRequest, FastifyReply } from 'fastify' +import { processURLs } from '../../../src/commands/processURLs.js' +import { reqToOpts } from '../utils/reqToOpts.js' +import { l, err } from '../../../src/globals.js' + +// Handler for the /urls route +export const handleURLsRequest = async ( + request: FastifyRequest, + reply: FastifyReply +): Promise => { + l('\nEntered handleURLsRequest') + + try { + // Access parsed request body + const requestData = request.body as any + l('\nParsed request body:', requestData) + + // Extract file path from the request data + const { filePath } = requestData + + if (!filePath) { + l('File path not provided, sending 400') + reply.status(400).send({ error: 'File path is required' }) + return + } + + // Map request data to processing options + const { options, llmServices, transcriptServices } = reqToOpts(requestData) + + // Set options.urls to filePath + options.urls = filePath + + l('\nCalling processURLs with params:', { + filePath, + llmServices, + transcriptServices, + options, + }) + + // Call processURLs with the mapped options and extracted file path + await processURLs(options, filePath, llmServices, transcriptServices) + + l('\nprocessURLs completed successfully') + reply.send({ message: 'URLs processed successfully.' }) + } catch (error) { + err('Error processing URLs:', error) + reply.status(500).send({ error: 'An error occurred while processing the URLs' }) + } +} \ No newline at end of file diff --git a/packages/server/routes/video.js b/packages/server/routes/video.js deleted file mode 100644 index 9cc892f..0000000 --- a/packages/server/routes/video.js +++ /dev/null @@ -1,38 +0,0 @@ -// server/routes/video.js - -import { processVideo } from '../../../src/commands/processVideo.js' -import { reqToOpts } from '../utils/reqToOpts.js' - -// Handler for /video route -const handleVideoRequest = async (request, reply) => { - console.log('\nEntered handleVideoRequest\n') - - try { - // Access parsed request body - const requestData = request.body - console.log('\nParsed request body:', requestData) - - // Extract YouTube URL - const { youtubeUrl } = requestData - - if (!youtubeUrl) { - console.log('YouTube URL not provided, sending 400') - reply.status(400).send({ error: 'YouTube URL is required' }) - return - } - - // Map request data to processing options - const { options, llmServices, transcriptServices } = reqToOpts(requestData) - console.log('\nCalling processVideo with params:', { youtubeUrl, llmServices, transcriptServices, options }) - - await processVideo(youtubeUrl, llmServices, transcriptServices, options) - - console.log('\nprocessVideo completed successfully') - reply.send({ message: 'Video processed successfully.' }) - } catch (error) { - console.error('Error processing video:', error) - reply.status(500).send({ error: 'An error occurred while processing the video' }) - } -} - -export { handleVideoRequest } \ No newline at end of file diff --git a/packages/server/routes/video.ts b/packages/server/routes/video.ts new file mode 100644 index 0000000..5e6ea07 --- /dev/null +++ b/packages/server/routes/video.ts @@ -0,0 +1,51 @@ +// server/routes/video.ts + +import { FastifyRequest, FastifyReply } from 'fastify' +import { processVideo } from '../../../src/commands/processVideo.js' +import { reqToOpts } from '../utils/reqToOpts.js' +import { l, err } from '../../../src/globals.js' + +// Handler for the /video route +export const handleVideoRequest = async ( + request: FastifyRequest, + reply: FastifyReply +): Promise => { + l('\nEntered handleVideoRequest') + + try { + // Access parsed request body + const requestData = request.body as any + l('\nParsed request body:', requestData) + + // Extract YouTube URL from the request data + const { youtubeUrl } = requestData + + if (!youtubeUrl) { + l('YouTube URL not provided, sending 400') + reply.status(400).send({ error: 'YouTube URL is required' }) + return + } + + // Map request data to processing options + const { options, llmServices, transcriptServices } = reqToOpts(requestData) + + // Set options.video to youtubeUrl + options.video = youtubeUrl + + l('\nCalling processVideo with params:', { + youtubeUrl, + llmServices, + transcriptServices, + options, + }) + + // Call processVideo with the mapped options and extracted URL + await processVideo(options, youtubeUrl, llmServices, transcriptServices) + + l('\nprocessVideo completed successfully') + reply.send({ message: 'Video processed successfully.' }) + } catch (error) { + err('Error processing video:', error) + reply.status(500).send({ error: 'An error occurred while processing the video' }) + } +} \ No newline at end of file diff --git a/packages/server/tests/fetch-all.js b/packages/server/tests/fetch-all.ts similarity index 90% rename from packages/server/tests/fetch-all.js rename to packages/server/tests/fetch-all.ts index d591a93..209f1ce 100644 --- a/packages/server/tests/fetch-all.js +++ b/packages/server/tests/fetch-all.ts @@ -1,7 +1,8 @@ -// server/fetch.js +// server/fetch-all.ts import fs from 'fs/promises' import path from 'path' +import { l, err } from '../../../src/globals.js' const BASE_URL = 'http://localhost:3000' const OUTPUT_DIR = 'content' @@ -27,7 +28,7 @@ const requests = [ data: { playlistUrl: 'https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr', whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/playlist', outputFiles: ['FILE_03A.md', 'FILE_03B.md'], @@ -37,7 +38,7 @@ const requests = [ playlistUrl: 'https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr', prompts: ['titles', 'mediumChapters'], whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/playlist', outputFiles: ['FILE_04A.md', 'FILE_04B.md'], @@ -62,7 +63,7 @@ const requests = [ data: { filePath: 'content/example-urls.md', whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/urls', outputFiles: ['FILE_07A.md', 'FILE_07B.md'], @@ -72,7 +73,7 @@ const requests = [ filePath: 'content/example-urls.md', prompts: ['titles', 'mediumChapters'], whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/urls', outputFiles: ['FILE_08A.md', 'FILE_08B.md'], @@ -97,7 +98,7 @@ const requests = [ data: { filePath: 'content/audio.mp3', whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/file', outputFiles: ['FILE_11.md'], @@ -107,7 +108,7 @@ const requests = [ filePath: 'content/audio.mp3', prompts: ['titles'], whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/file', outputFiles: ['FILE_12.md'], @@ -132,7 +133,7 @@ const requests = [ data: { rssUrl: 'https://ajcwebdev.substack.com/feed/', whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/rss', outputFiles: ['FILE_15.md'], @@ -169,7 +170,7 @@ const requests = [ data: { youtubeUrl: 'https://www.youtube.com/watch?v=MORMZXEaONk', whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/video', outputFiles: ['FILE_19.md'], @@ -259,23 +260,6 @@ const requests = [ endpoint: '/video', outputFiles: ['FILE_29.md'], }, - { - data: { - youtubeUrl: 'https://www.youtube.com/watch?v=MORMZXEaONk', - llm: 'octo', - }, - endpoint: '/video', - outputFiles: ['FILE_30.md'], - }, - { - data: { - youtubeUrl: 'https://www.youtube.com/watch?v=MORMZXEaONk', - llm: 'octo', - llmModel: 'LLAMA_3_1_8B', - }, - endpoint: '/video', - outputFiles: ['FILE_31.md'], - }, { data: { youtubeUrl: 'https://www.youtube.com/watch?v=MORMZXEaONk', @@ -296,7 +280,7 @@ const requests = [ data: { youtubeUrl: 'https://www.youtube.com/watch?v=MORMZXEaONk', transcriptServices: 'deepgram', - llm: 'llama', + llm: 'ollama', }, endpoint: '/video', outputFiles: ['FILE_34.md'], @@ -313,7 +297,7 @@ const requests = [ data: { youtubeUrl: 'https://www.youtube.com/watch?v=MORMZXEaONk', transcriptServices: 'assembly', - llm: 'llama', + llm: 'ollama', }, endpoint: '/video', outputFiles: ['FILE_36.md'], @@ -332,7 +316,7 @@ const requests = [ youtubeUrl: 'https://ajc.pics/audio/fsjam-short.mp3', transcriptServices: 'assembly', speakerLabels: true, - llm: 'llama', + llm: 'ollama', }, endpoint: '/video', outputFiles: ['FILE_38.md'], @@ -358,7 +342,7 @@ const requests = [ youtubeUrl: 'https://www.youtube.com/watch?v=MORMZXEaONk', prompts: ['titles', 'summary', 'shortChapters', 'takeaways', 'questions'], whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/video', outputFiles: ['FILE_41.md'], @@ -377,12 +361,12 @@ const fetchRequest = async (request, index) => { }, body: JSON.stringify(request.data), }) - console.log(`\nRequest ${index + 1} response status:`, response.status) + l(`\nRequest ${index + 1} response status:`, response.status) if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`) } const result = await response.json() - console.log(`Request ${index + 1} result: ${result.message}`) + l(`Request ${index + 1} result: ${result.message}`) // Wait briefly to ensure files are written await new Promise((resolve) => setTimeout(resolve, 1000)) @@ -404,13 +388,13 @@ const fetchRequest = async (request, index) => { const newFileName = outputFiles[i] const newFilePath = path.join(OUTPUT_DIR, newFileName) await fs.rename(oldFilePath, newFilePath) - console.log(`\nFile renamed:\n - Old: ${oldFilePath}\n - New: ${newFilePath}`) + l(`\nFile renamed:\n - Old: ${oldFilePath}\n - New: ${newFilePath}`) } } else { - console.log('No new files to rename for this request.') + l('No new files to rename for this request.') } } catch (error) { - console.error(`Error in request ${index + 1}:`, error) + err(`Error in request ${index + 1}:`, error) } } diff --git a/packages/server/tests/fetch-local.js b/packages/server/tests/fetch-local.ts similarity index 91% rename from packages/server/tests/fetch-local.js rename to packages/server/tests/fetch-local.ts index 37c1fdc..969eba5 100644 --- a/packages/server/tests/fetch-local.js +++ b/packages/server/tests/fetch-local.ts @@ -1,7 +1,8 @@ -// packages/server/tests/fetch-local.js +// packages/server/tests/fetch-local.ts import fs from 'fs/promises' import path from 'path' +import { l, err } from '../../../src/globals.js' const BASE_URL = 'http://localhost:3000' const OUTPUT_DIR = 'content' @@ -27,7 +28,7 @@ const requests = [ data: { playlistUrl: 'https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr', whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/playlist', outputFiles: ['FILE_03A.md', 'FILE_03B.md'], @@ -37,7 +38,7 @@ const requests = [ playlistUrl: 'https://www.youtube.com/playlist?list=PLCVnrVv4KhXPz0SoAVu8Rc1emAdGPbSbr', prompts: ['titles', 'mediumChapters'], whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/playlist', outputFiles: ['FILE_04A.md', 'FILE_04B.md'], @@ -62,7 +63,7 @@ const requests = [ data: { filePath: 'content/example-urls.md', whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/urls', outputFiles: ['FILE_07A.md', 'FILE_07B.md'], @@ -72,7 +73,7 @@ const requests = [ filePath: 'content/example-urls.md', prompts: ['titles', 'mediumChapters'], whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/urls', outputFiles: ['FILE_08A.md', 'FILE_08B.md'], @@ -97,7 +98,7 @@ const requests = [ data: { filePath: 'content/audio.mp3', whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/file', outputFiles: ['FILE_11.md'], @@ -107,7 +108,7 @@ const requests = [ filePath: 'content/audio.mp3', prompts: ['titles'], whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/file', outputFiles: ['FILE_12.md'], @@ -132,7 +133,7 @@ const requests = [ data: { rssUrl: 'https://ajcwebdev.substack.com/feed/', whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/rss', outputFiles: ['FILE_15.md'], @@ -169,7 +170,7 @@ const requests = [ data: { youtubeUrl: 'https://www.youtube.com/watch?v=MORMZXEaONk', whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/video', outputFiles: ['FILE_19.md'], @@ -203,7 +204,7 @@ const requests = [ youtubeUrl: 'https://www.youtube.com/watch?v=MORMZXEaONk', prompts: ['titles', 'summary', 'shortChapters', 'takeaways', 'questions'], whisperModel: 'tiny', - llm: 'llama', + llm: 'ollama', }, endpoint: '/video', outputFiles: ['FILE_23.md'], @@ -222,12 +223,12 @@ const fetchRequest = async (request, index) => { }, body: JSON.stringify(request.data), }) - console.log(`\nRequest ${index + 1} response status:`, response.status) + l(`\nRequest ${index + 1} response status:`, response.status) if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`) } const result = await response.json() - console.log(`Request ${index + 1} result: ${result.message}`) + l(`Request ${index + 1} result: ${result.message}`) // Wait briefly to ensure files are written await new Promise((resolve) => setTimeout(resolve, 1000)) @@ -249,13 +250,13 @@ const fetchRequest = async (request, index) => { const newFileName = outputFiles[i] const newFilePath = path.join(OUTPUT_DIR, newFileName) await fs.rename(oldFilePath, newFilePath) - console.log(`\nFile renamed:\n - Old: ${oldFilePath}\n - New: ${newFilePath}`) + l(`\nFile renamed:\n - Old: ${oldFilePath}\n - New: ${newFilePath}`) } } else { - console.log('No new files to rename for this request.') + l('No new files to rename for this request.') } } catch (error) { - console.error(`Error in request ${index + 1}:`, error) + err(`Error in request ${index + 1}:`, error) } } diff --git a/packages/server/utils/reqToOpts.js b/packages/server/utils/reqToOpts.js deleted file mode 100644 index ee7e42f..0000000 --- a/packages/server/utils/reqToOpts.js +++ /dev/null @@ -1,54 +0,0 @@ -// server/utils/reqToOpts.js - -// Function to map request data to processing options -function reqToOpts(requestData) { - // Define possible options - const [llmServices, transcriptServices, otherOptions] = [ - // List of supported LLM options - ['chatgpt', 'claude', 'cohere', 'mistral', 'octo', 'llama', 'ollama', 'gemini'], - // List of supported transcript services - ['whisper', 'whisperDocker', 'deepgram', 'assembly'], - // List of other supported options - ['speakerLabels', 'prompt', 'noCleanUp', 'order', 'skip', 'info', 'item'] - ] - - // Initialize options object - const options = {} - - // Check if LLM is provided and valid - if (requestData.llm && llmServices.includes(requestData.llm)) { - // Set llmServices - llmServices = requestData.llm - // Set LLM model or true - options[llmServices] = requestData.llmModel || true - } - - // Determine transcript service and default to 'whisper' if not specified - transcriptServices = transcriptServices.includes(requestData.transcriptServices) - ? requestData.transcriptServices - : 'whisper' - - // Set transcript options - if (transcriptServices === 'whisper') { - // Set whisper model - options.whisperModel = requestData.whisperModel || 'base' - // Enable whisper option - options.whisper = options.whisperModel - } else { - // Enable selected transcript service - options[transcriptServices] = true - } - - // Map other options from request data - for (const opt of otherOptions) { - if (requestData[opt] !== undefined) { - // Set option if provided - options[opt] = requestData[opt] - } - } - - // Return mapped options - return { options, llmServices, transcriptServices } -} - -export { reqToOpts } \ No newline at end of file diff --git a/packages/server/utils/reqToOpts.ts b/packages/server/utils/reqToOpts.ts new file mode 100644 index 0000000..3c921d8 --- /dev/null +++ b/packages/server/utils/reqToOpts.ts @@ -0,0 +1,81 @@ +// server/utils/reqToOpts.ts + +import { ProcessingOptions, LLMServices, TranscriptServices } from '../../../src/types.js' + +// Function to map request data to processing options +export function reqToOpts(requestData: any): { + options: ProcessingOptions + llmServices?: LLMServices + transcriptServices?: TranscriptServices +} { + // Define lists of supported options + const llmOptions = [ + 'chatgpt', + 'claude', + 'cohere', + 'mistral', + 'ollama', + 'gemini', + 'fireworks', + 'together', + 'groq', + ] as const + + const transcriptOptions = [ + 'whisper', + 'whisperDocker', + 'whisperPython', + 'whisperDiarization', + 'deepgram', + 'assembly', + ] as const + + const otherOptions = ['speakerLabels', 'prompt', 'noCleanUp', 'order', 'skip', 'info', 'item'] + + // Initialize options object + const options: ProcessingOptions = {} + + // Variables to hold selected services + let llmServices: LLMServices | undefined + let transcriptServices: TranscriptServices | undefined + + // Check if a valid LLM service is provided + if (requestData.llm && llmOptions.includes(requestData.llm)) { + // Set the LLM service + llmServices = requestData.llm as LLMServices + // Set the LLM model or default to true + options[llmServices] = requestData.llmModel || true + } + + // Determine transcript service or default to 'whisper' if not specified + transcriptServices = transcriptOptions.includes(requestData.transcriptServices) + ? (requestData.transcriptServices as TranscriptServices) + : 'whisper' + + // Set transcript options based on the selected service + if (transcriptServices === 'whisper') { + // Set the Whisper model or default to 'base' + options.whisper = requestData.whisperModel || 'base' + } else if (transcriptServices === 'whisperDocker') { + options.whisperDocker = requestData.whisperModel || 'base' + } else if (transcriptServices === 'whisperPython') { + options.whisperPython = requestData.whisperModel || 'base' + } else if (transcriptServices === 'whisperDiarization') { + options.whisperDiarization = requestData.whisperModel || 'base' + } else if (transcriptServices === 'deepgram') { + options.deepgram = true + } else if (transcriptServices === 'assembly') { + options.assembly = true + } + + // Map additional options from the request data + for (const opt of otherOptions) { + if (requestData[opt] !== undefined) { + // Set the option if it is provided + options[opt] = requestData[opt] + } + } + + // Return the mapped options along with selected services + return { options, llmServices, transcriptServices } +} diff --git a/scripts/cleanContent.ts b/scripts/cleanContent.ts index 7f3ef32..5a06ce8 100644 --- a/scripts/cleanContent.ts +++ b/scripts/cleanContent.ts @@ -2,6 +2,7 @@ import { exec } from 'child_process' import { promisify } from 'util' +import { l, err } from '../src/globals.js' const execAsync = promisify(exec) @@ -11,15 +12,15 @@ async function cleanContent() { 'find content -type f -not \\( -name ".gitkeep" -o -name "audio.mp3" -o -name "example-urls.md" \\) -delete' ) if (stderr) { - console.error('Error:', stderr) + err('Error:', stderr) return } - console.log('Files deleted successfully') + l('Files deleted successfully') if (stdout) { - console.log('Output:', stdout) + l('Output:', stdout) } } catch (error) { - console.error('Execution error:', error) + err('Execution error:', error) } } diff --git a/scripts/setup-python.sh b/scripts/setup-python.sh index 4f91c7d..ff216d5 100755 --- a/scripts/setup-python.sh +++ b/scripts/setup-python.sh @@ -6,6 +6,7 @@ if [ ! -d "whisper-diarization" ]; then echo "Cloning the whisper-diarization repository..." git clone https://github.com/MahmoudAshraf97/whisper-diarization.git + rm -rf whisper-diarization/.git else echo "whisper-diarization repository already exists. Skipping clone." fi diff --git a/scripts/setup.sh b/scripts/setup.sh index 3a4a1df..956a8b3 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -57,7 +57,7 @@ else check_ollama_server # Check and pull required models - check_and_pull_model "llama3.2:1b" + check_and_pull_model "llama3.2:1b" && check_and_pull_model "llama3.2:3b" fi # Install npm dependencies @@ -72,7 +72,9 @@ else # Download whisper models echo "Downloading whisper models..." + bash ./whisper.cpp/models/download-ggml-model.sh tiny bash ./whisper.cpp/models/download-ggml-model.sh base + bash ./whisper.cpp/models/download-ggml-model.sh large-v3-turbo # Compile whisper.cpp echo "Compiling whisper.cpp..." @@ -81,14 +83,7 @@ else # Copy Dockerfile echo "Copying Dockerfile..." cp .github/whisper.Dockerfile whisper.cpp/Dockerfile -fi - -# Check if Qwen 2.5 1.5B model exists -if [ -f "./src/llms/models/qwen2.5-1.5b-instruct-q6_k.gguf" ]; then - echo "Qwen 2.5 1.5B model already exists. Skipping download." -else - echo "Downloading Qwen 2.5 1.5B model..." - curl -L "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF/resolve/main/qwen2.5-1.5b-instruct-q6_k.gguf" -o "./src/llms/models/qwen2.5-1.5b-instruct-q6_k.gguf" + rm -rf whisper.cpp/.git fi echo "Setup completed successfully!" \ No newline at end of file diff --git a/src/autoshow.ts b/src/autoshow.ts index 2ed1701..f423775 100644 --- a/src/autoshow.ts +++ b/src/autoshow.ts @@ -18,8 +18,9 @@ import { processPlaylist } from './commands/processPlaylist.js' import { processURLs } from './commands/processURLs.js' import { processFile } from './commands/processFile.js' import { processRSS } from './commands/processRSS.js' +import { validateOption } from './utils/validateOption.js' import { argv, exit } from 'node:process' -import { log, opts, final, ACTION_OPTIONS, LLM_OPTIONS, TRANSCRIPT_OPTIONS } from './models.js' +import { l, err, opts, final, ACTION_OPTIONS, LLM_OPTIONS, TRANSCRIPT_OPTIONS } from './globals.js' import type { ProcessingOptions, HandlerFunction, LLMServices, TranscriptServices } from './types.js' // Initialize the command-line interface using Commander.js @@ -59,11 +60,9 @@ program .option('--claude [model]', 'Use Claude for processing with optional model specification') .option('--cohere [model]', 'Use Cohere for processing with optional model specification') .option('--mistral [model]', 'Use Mistral for processing') - .option('--octo [model]', 'Use Octo for processing') .option('--fireworks [model]', 'Use Fireworks AI for processing with optional model specification') .option('--together [model]', 'Use Together AI for processing with optional model specification') .option('--groq [model]', 'Use Groq for processing with optional model specification') - .option('--llama [model]', 'Use Node Llama for processing with optional model specification') .option('--ollama [model]', 'Use Ollama for processing with optional model specification') .option('--gemini [model]', 'Use Gemini for processing with optional model specification') // Utility options @@ -84,31 +83,6 @@ Report Issues: https://github.com/ajcwebdev/autoshow/issues ` ) -/** - * Helper function to validate that only one option from a list is provided. - * Prevents users from specifying multiple conflicting options simultaneously. - * - * @param optionKeys - The list of option keys to check. - * @param options - The options object. - * @param errorMessage - The prefix of the error message. - * @returns The selected option or undefined. - */ -function getSingleOption( - optionKeys: string[], - options: ProcessingOptions, - errorMessage: string -): string | undefined { - // Filter out which options from the provided list are actually set - const selectedOptions = optionKeys.filter((opt) => options[opt as keyof ProcessingOptions]) - - // If more than one option is selected, throw an error - if (selectedOptions.length > 1) { - console.error(`Error: Multiple ${errorMessage} provided (${selectedOptions.join(', ')}). Please specify only one.`) - exit(1) - } - return selectedOptions[0] as string | undefined -} - /** * Main action for the program. * Handles the processing of options and executes the appropriate command handler. @@ -117,9 +91,9 @@ function getSingleOption( */ program.action(async (options: ProcessingOptions) => { // Log received options for debugging purposes - log(opts(`Options received at beginning of command:\n`)) - log(options) - log(``) + l(opts(`Options received at beginning of command:\n`)) + l(opts(JSON.stringify(options, null, 2))) + l(``) // Define mapping of action types to their handler functions const PROCESS_HANDLERS: Record = { @@ -147,17 +121,14 @@ program.action(async (options: ProcessingOptions) => { } // Validate and get single options for action, LLM, and transcription - const action = getSingleOption(ACTION_OPTIONS, options, 'input option') - const llmKey = getSingleOption(LLM_OPTIONS, options, 'LLM option') + const action = validateOption(ACTION_OPTIONS, options, 'input option') + const llmKey = validateOption(LLM_OPTIONS, options, 'LLM option') const llmServices = llmKey as LLMServices | undefined - const transcriptKey = getSingleOption(TRANSCRIPT_OPTIONS, options, 'transcription option') - const transcriptServices: TranscriptServices | undefined = transcriptKey as TranscriptServices | undefined - - // Set default transcription service to whisper if none provided - const finalTranscriptServices: TranscriptServices = transcriptServices || 'whisper' + const transcriptKey = validateOption(TRANSCRIPT_OPTIONS, options, 'transcription option') + const transcriptServices: TranscriptServices = (transcriptKey as TranscriptServices) || 'whisper' // Set default Whisper model to 'large-v3-turbo' if whisper is selected but no model specified - if (finalTranscriptServices === 'whisper' && !options.whisper) { + if (transcriptServices === 'whisper' && !options.whisper) { options.whisper = 'large-v3-turbo' } @@ -169,16 +140,16 @@ program.action(async (options: ProcessingOptions) => { options, options[action as keyof ProcessingOptions] as string, llmServices, - finalTranscriptServices + transcriptServices ) // Log success message - log(final(`\n================================================================================================`)) - log(final(` ${action} Processing Completed Successfully.`)) - log(final(`================================================================================================\n`)) + l(final(`\n================================================================================================`)) + l(final(` ${action} Processing Completed Successfully.`)) + l(final(`================================================================================================\n`)) exit(0) } catch (error) { // Log error and exit if processing fails - console.error(`Error processing ${action}:`, (error as Error).message) + err(`Error processing ${action}:`, (error as Error).message) exit(1) } } @@ -186,7 +157,7 @@ program.action(async (options: ProcessingOptions) => { // Set up error handling for unknown commands program.on('command:*', function () { - console.error(`Error: Invalid command '${program.args.join(' ')}'. Use --help to see available commands.`) + err(`Error: Invalid command '${program.args.join(' ')}'. Use --help to see available commands.`) exit(1) }) diff --git a/src/commands/processFile.ts b/src/commands/processFile.ts index a9b3c5d..ffca77a 100644 --- a/src/commands/processFile.ts +++ b/src/commands/processFile.ts @@ -10,7 +10,7 @@ import { downloadAudio } from '../utils/downloadAudio.js' import { runTranscription } from '../utils/runTranscription.js' import { runLLM } from '../utils/runLLM.js' import { cleanUpFiles } from '../utils/cleanUpFiles.js' -import { log, opts, wait } from '../models.js' +import { l, err, opts } from '../globals.js' import type { LLMServices, TranscriptServices, ProcessingOptions } from '../types.js' /** @@ -38,8 +38,8 @@ export async function processFile( transcriptServices?: TranscriptServices ): Promise { // Log the processing parameters for debugging purposes - log(opts('Parameters passed to processFile:\n')) - log(wait(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}\n`)) + l(opts('Parameters passed to processFile:\n')) + l(opts(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}\n`)) try { // Generate markdown file with file metadata and get file paths @@ -60,7 +60,7 @@ export async function processFile( } } catch (error) { // Log the error and terminate the process with error code - console.error(`Error processing file: ${(error as Error).message}`) + err(`Error processing file: ${(error as Error).message}`) process.exit(1) } } \ No newline at end of file diff --git a/src/commands/processPlaylist.ts b/src/commands/processPlaylist.ts index 1d99623..81609e0 100644 --- a/src/commands/processPlaylist.ts +++ b/src/commands/processPlaylist.ts @@ -6,17 +6,11 @@ */ import { writeFile } from 'node:fs/promises' -import { execFile } from 'node:child_process' -import { promisify } from 'node:util' import { processVideo } from './processVideo.js' import { extractVideoMetadata } from '../utils/extractVideoMetadata.js' -import { checkDependencies } from '../utils/checkDependencies.js' -import { log, opts, success, wait } from '../models.js' +import { l, err, opts, success, execFilePromise } from '../globals.js' import type { LLMServices, TranscriptServices, ProcessingOptions } from '../types.js' -// Convert execFile to use promises instead of callbacks -const execFilePromise = promisify(execFile) - /** * Processes an entire YouTube playlist by: * 1. Validating system dependencies @@ -42,11 +36,9 @@ export async function processPlaylist( transcriptServices?: TranscriptServices ): Promise { // Log the processing parameters for debugging purposes - log(opts('Parameters passed to processPlaylist:\n')) - log(wait(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}`)) + l(opts('Parameters passed to processPlaylist:\n')) + l(opts(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}`)) try { - // Verify that yt-dlp is installed and available - await checkDependencies(['yt-dlp']) // Extract all video URLs from the playlist using yt-dlp const { stdout, stderr } = await execFilePromise('yt-dlp', [ '--flat-playlist', @@ -56,16 +48,16 @@ export async function processPlaylist( ]) // Log any warnings from yt-dlp if (stderr) { - console.error(`yt-dlp warnings: ${stderr}`) + err(`yt-dlp warnings: ${stderr}`) } // Convert stdout into array of video URLs, removing empty entries const urls = stdout.trim().split('\n').filter(Boolean) // Exit if no videos were found in the playlist if (urls.length === 0) { - console.error('Error: No videos found in the playlist.') + err('Error: No videos found in the playlist.') process.exit(1) } - log(opts(`\nFound ${urls.length} videos in the playlist...`)) + l(opts(`\nFound ${urls.length} videos in the playlist...`)) // Collect metadata for all videos in parallel const metadataPromises = urls.map(extractVideoMetadata) const metadataList = await Promise.all(metadataPromises) @@ -75,25 +67,25 @@ export async function processPlaylist( const jsonContent = JSON.stringify(validMetadata, null, 2) const jsonFilePath = 'content/playlist_info.json' await writeFile(jsonFilePath, jsonContent) - log(success(`Playlist information saved to: ${jsonFilePath}`)) + l(success(`Playlist information saved to: ${jsonFilePath}`)) return } // Process each video sequentially, with error handling for individual videos for (const [index, url] of urls.entries()) { // Visual separator for each video in the console - log(opts(`\n================================================================================================`)) - log(opts(` Processing video ${index + 1}/${urls.length}: ${url}`)) - log(opts(`================================================================================================\n`)) + l(opts(`\n================================================================================================`)) + l(opts(` Processing video ${index + 1}/${urls.length}: ${url}`)) + l(opts(`================================================================================================\n`)) try { await processVideo(options, url, llmServices, transcriptServices) } catch (error) { // Log error but continue processing remaining videos - console.error(`Error processing video ${url}: ${(error as Error).message}`) + err(`Error processing video ${url}: ${(error as Error).message}`) } } } catch (error) { // Handle fatal errors that prevent playlist processing - console.error(`Error processing playlist: ${(error as Error).message}`) + err(`Error processing playlist: ${(error as Error).message}`) process.exit(1) } } \ No newline at end of file diff --git a/src/commands/processRSS.ts b/src/commands/processRSS.ts index 34c74aa..52b7b8d 100644 --- a/src/commands/processRSS.ts +++ b/src/commands/processRSS.ts @@ -6,50 +6,39 @@ */ import { writeFile } from 'node:fs/promises' -import { XMLParser } from 'fast-xml-parser' import { generateMarkdown } from '../utils/generateMarkdown.js' import { downloadAudio } from '../utils/downloadAudio.js' import { runTranscription } from '../utils/runTranscription.js' import { runLLM } from '../utils/runLLM.js' import { cleanUpFiles } from '../utils/cleanUpFiles.js' -import { log, wait, opts } from '../models.js' +import { l, err, wait, opts, parser } from '../globals.js' import type { LLMServices, TranscriptServices, ProcessingOptions, RSSItem } from '../types.js' -/** - * Configure XML parser for RSS feed processing - * Handles attributes without prefixes and allows boolean values - */ -const parser = new XMLParser({ - ignoreAttributes: false, - attributeNamePrefix: '', - allowBooleanAttributes: true, -}) - /** * Validates RSS processing options for consistency and correct values. * * @param options - Configuration options to validate * @throws Will exit process if validation fails */ -function validateOptions(options: ProcessingOptions): void { +function validateRSSOptions(options: ProcessingOptions): void { if (options.last !== undefined) { if (!Number.isInteger(options.last) || options.last < 1) { - console.error('Error: The --last option must be a positive integer.') + err('Error: The --last option must be a positive integer.') process.exit(1) } if (options.skip !== undefined || options.order !== undefined) { - console.error('Error: The --last option cannot be used with --skip or --order.') + err('Error: The --last option cannot be used with --skip or --order.') process.exit(1) } } if (options.skip !== undefined && (!Number.isInteger(options.skip) || options.skip < 0)) { - console.error('Error: The --skip option must be a non-negative integer.') + err('Error: The --skip option must be a non-negative integer.') process.exit(1) } if (options.order !== undefined && !['newest', 'oldest'].includes(options.order)) { - console.error("Error: The --order option must be either 'newest' or 'oldest'.") + err("Error: The --order option must be either 'newest' or 'oldest'.") process.exit(1) } } @@ -61,12 +50,12 @@ function validateOptions(options: ProcessingOptions): void { */ function logProcessingAction(options: ProcessingOptions): void { if (options.item && options.item.length > 0) { - log(wait('\nProcessing specific items:')) - options.item.forEach((url) => log(wait(` - ${url}`))) + l(wait('\nProcessing specific items:')) + options.item.forEach((url) => l(wait(` - ${url}`))) } else if (options.last) { - log(wait(`\nProcessing the last ${options.last} items`)) + l(wait(`\nProcessing the last ${options.last} items`)) } else if (options.skip) { - log(wait(` - Skipping first ${options.skip || 0} items`)) + l(wait(` - Skipping first ${options.skip || 0} items`)) } } @@ -90,7 +79,7 @@ async function fetchRSSFeed(rssUrl: string) { clearTimeout(timeout) if (!response.ok) { - console.error(`HTTP error! status: ${response.status}`) + err(`HTTP error! status: ${response.status}`) process.exit(1) } @@ -98,9 +87,9 @@ async function fetchRSSFeed(rssUrl: string) { return parser.parse(text) } catch (error) { if ((error as Error).name === 'AbortError') { - console.error('Error: Fetch request timed out.') + err('Error: Fetch request timed out.') } else { - console.error(`Error fetching RSS feed: ${(error as Error).message}`) + err(`Error fetching RSS feed: ${(error as Error).message}`) } process.exit(1) } @@ -135,7 +124,7 @@ function extractFeedItems(feed: any): RSSItem[] { })) if (items.length === 0) { - console.error('Error: No audio/video items found in the RSS feed.') + err('Error: No audio/video items found in the RSS feed.') process.exit(1) } @@ -151,7 +140,7 @@ async function saveFeedInfo(items: RSSItem[]): Promise { const jsonContent = JSON.stringify(items, null, 2) const jsonFilePath = 'content/rss_info.json' await writeFile(jsonFilePath, jsonContent) - log(wait(`RSS feed information saved to: ${jsonFilePath}`)) + l(wait(`RSS feed information saved to: ${jsonFilePath}`)) } /** @@ -166,7 +155,7 @@ function selectItemsToProcess(items: RSSItem[], options: ProcessingOptions): RSS if (options.item && options.item.length > 0) { const matchedItems = items.filter((item) => options.item!.includes(item.showLink)) if (matchedItems.length === 0) { - console.error('Error: No matching items found for the provided URLs.') + err('Error: No matching items found for the provided URLs.') process.exit(1) } return matchedItems @@ -185,14 +174,14 @@ function selectItemsToProcess(items: RSSItem[], options: ProcessingOptions): RSS */ function logProcessingStatus(total: number, processing: number, options: ProcessingOptions): void { if (options.item && options.item.length > 0) { - log(wait(`\n - Found ${total} items in the RSS feed.`)) - log(wait(` - Processing ${processing} specified items.`)) + l(wait(`\n - Found ${total} items in the RSS feed.`)) + l(wait(` - Processing ${processing} specified items.`)) } else if (options.last) { - log(wait(`\n - Found ${total} items in the RSS feed.`)) - log(wait(` - Processing the last ${options.last} items.`)) + l(wait(`\n - Found ${total} items in the RSS feed.`)) + l(wait(` - Processing the last ${options.last} items.`)) } else { - log(wait(`\n - Found ${total} item(s) in the RSS feed.`)) - log(wait(` - Processing ${processing} item(s) after skipping ${options.skip || 0}.\n`)) + l(wait(`\n - Found ${total} item(s) in the RSS feed.`)) + l(wait(` - Processing ${processing} item(s) after skipping ${options.skip || 0}.\n`)) } } @@ -210,8 +199,8 @@ async function processItem( llmServices?: LLMServices, transcriptServices?: TranscriptServices ): Promise { - log(opts('Parameters passed to processItem:\n')) - log(wait(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}\n`)) + l(opts('Parameters passed to processItem:\n')) + l(opts(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}\n`)) try { const { frontMatter, finalPath, filename } = await generateMarkdown(options, item) @@ -223,7 +212,7 @@ async function processItem( await cleanUpFiles(finalPath) } } catch (error) { - console.error(`Error processing item ${item.title}: ${(error as Error).message}`) + err(`Error processing item ${item.title}: ${(error as Error).message}`) } } @@ -237,15 +226,15 @@ async function processItems( transcriptServices?: TranscriptServices ): Promise { for (const [index, item] of items.entries()) { - log(opts(`\n========================================================================================`)) - log(opts(` Item ${index + 1}/${items.length} processing: ${item.title}`)) - log(opts(`========================================================================================\n`)) + l(opts(`\n========================================================================================`)) + l(opts(` Item ${index + 1}/${items.length} processing: ${item.title}`)) + l(opts(`========================================================================================\n`)) await processItem(options, item, llmServices, transcriptServices) - log(opts(`\n========================================================================================`)) - log(opts(` ${index + 1}/${items.length} item processing completed successfully`)) - log(opts(`========================================================================================\n`)) + l(opts(`\n========================================================================================`)) + l(opts(` ${index + 1}/${items.length} item processing completed successfully`)) + l(opts(`========================================================================================\n`)) } } @@ -259,11 +248,11 @@ export async function processRSS( llmServices?: LLMServices, transcriptServices?: TranscriptServices ): Promise { - log(opts('Parameters passed to processRSS:\n')) - log(wait(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}`)) + l(opts('Parameters passed to processRSS:\n')) + l(wait(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}`)) try { - validateOptions(options) + validateRSSOptions(options) logProcessingAction(options) const feed = await fetchRSSFeed(rssUrl) @@ -278,7 +267,7 @@ export async function processRSS( logProcessingStatus(items.length, itemsToProcess.length, options) await processItems(itemsToProcess, options, llmServices, transcriptServices) } catch (error) { - console.error(`Error processing RSS feed: ${(error as Error).message}`) + err(`Error processing RSS feed: ${(error as Error).message}`) process.exit(1) } } \ No newline at end of file diff --git a/src/commands/processURLs.ts b/src/commands/processURLs.ts index d69e428..e56875b 100644 --- a/src/commands/processURLs.ts +++ b/src/commands/processURLs.ts @@ -8,8 +8,7 @@ import { readFile, writeFile } from 'node:fs/promises' import { processVideo } from './processVideo.js' import { extractVideoMetadata } from '../utils/extractVideoMetadata.js' -import { checkDependencies } from '../utils/checkDependencies.js' -import { log, wait, opts } from '../models.js' +import { l, err, wait, opts } from '../globals.js' import type { LLMServices, TranscriptServices, ProcessingOptions } from '../types.js' /** @@ -39,12 +38,10 @@ export async function processURLs( transcriptServices?: TranscriptServices ): Promise { // Log the processing parameters for debugging purposes - log(opts('Parameters passed to processURLs:\n')) - log(wait(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}\n`)) + l(opts('Parameters passed to processURLs:\n')) + l(opts(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}\n`)) try { - // Verify that yt-dlp is installed and available - await checkDependencies(['yt-dlp']) // Read the file and extract valid URLs const content = await readFile(filePath, 'utf8') const urls = content.split('\n') @@ -52,10 +49,10 @@ export async function processURLs( .filter(line => line && !line.startsWith('#')) // Exit if no valid URLs were found in the file if (urls.length === 0) { - console.error('Error: No URLs found in the file.') + err('Error: No URLs found in the file.') process.exit(1) } - log(opts(`\nFound ${urls.length} URLs in the file...`)) + l(opts(`\nFound ${urls.length} URLs in the file...`)) // Collect metadata for all videos in parallel const metadataPromises = urls.map(extractVideoMetadata) const metadataList = await Promise.all(metadataPromises) @@ -65,25 +62,25 @@ export async function processURLs( const jsonContent = JSON.stringify(validMetadata, null, 2) const jsonFilePath = 'content/urls_info.json' await writeFile(jsonFilePath, jsonContent) - log(wait(`Video information saved to: ${jsonFilePath}`)) + l(wait(`Video information saved to: ${jsonFilePath}`)) return } // Process each URL sequentially, with error handling for individual videos for (const [index, url] of urls.entries()) { // Visual separator for each video in the console - log(opts(`\n================================================================================================`)) - log(opts(` Processing URL ${index + 1}/${urls.length}: ${url}`)) - log(opts(`================================================================================================\n`)) + l(opts(`\n================================================================================================`)) + l(opts(` Processing URL ${index + 1}/${urls.length}: ${url}`)) + l(opts(`================================================================================================\n`)) try { await processVideo(options, url, llmServices, transcriptServices) } catch (error) { // Log error but continue processing remaining URLs - console.error(`Error processing URL ${url}: ${(error as Error).message}`) + err(`Error processing URL ${url}: ${(error as Error).message}`) } } } catch (error) { // Handle fatal errors that prevent file processing - console.error(`Error reading or processing file ${filePath}: ${(error as Error).message}`) + err(`Error reading or processing file ${filePath}: ${(error as Error).message}`) process.exit(1) } } \ No newline at end of file diff --git a/src/commands/processVideo.ts b/src/commands/processVideo.ts index 7c45fb4..d5874d0 100644 --- a/src/commands/processVideo.ts +++ b/src/commands/processVideo.ts @@ -5,13 +5,12 @@ * @packageDocumentation */ -import { checkDependencies } from '../utils/checkDependencies.js' import { generateMarkdown } from '../utils/generateMarkdown.js' import { downloadAudio } from '../utils/downloadAudio.js' import { runTranscription } from '../utils/runTranscription.js' import { runLLM } from '../utils/runLLM.js' import { cleanUpFiles } from '../utils/cleanUpFiles.js' -import { log, opts, wait } from '../models.js' +import { l, err, opts } from '../globals.js' import type { LLMServices, TranscriptServices, ProcessingOptions } from '../types.js' /** @@ -37,13 +36,10 @@ export async function processVideo( transcriptServices?: TranscriptServices ): Promise { // Log the processing parameters for debugging purposes - log(opts('Parameters passed to processVideo:\n')) - log(wait(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}\n`)) + l(opts('Parameters passed to processVideo:\n')) + l(opts(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}\n`)) try { - // Verify that required system dependencies (yt-dlp) are installed - await checkDependencies(['yt-dlp']) - // Generate markdown file with video metadata and get file paths const { frontMatter, finalPath, filename } = await generateMarkdown(options, url) @@ -62,7 +58,7 @@ export async function processVideo( } } catch (error) { // Log the error details and re-throw for upstream handling - console.error('Error processing video:', (error as Error).message) + err('Error processing video:', (error as Error).message) throw error } } \ No newline at end of file diff --git a/src/models.ts b/src/globals.ts similarity index 76% rename from src/models.ts rename to src/globals.ts index 054d072..c79f8fb 100644 --- a/src/models.ts +++ b/src/globals.ts @@ -5,9 +5,25 @@ * @packageDocumentation */ +import { XMLParser } from 'fast-xml-parser' +import { exec, execFile } from 'node:child_process' +import { promisify } from 'node:util' import chalk from 'chalk' import type { ChalkInstance } from 'chalk' -import type { WhisperModelType, ChatGPTModelType, ClaudeModelType, CohereModelType, GeminiModelType, MistralModelType, OctoModelType, LlamaModelType, OllamaModelType, TogetherModelType, FireworksModelType, GroqModelType } from './types.js' +import type { WhisperModelType, ChatGPTModelType, ClaudeModelType, CohereModelType, GeminiModelType, MistralModelType, OllamaModelType, TogetherModelType, FireworksModelType, GroqModelType } from './types.js' + +export const execPromise = promisify(exec) +export const execFilePromise = promisify(execFile) + +/** + * Configure XML parser for RSS feed processing + * Handles attributes without prefixes and allows boolean values + */ +export const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: '', + allowBooleanAttributes: true, +}) /** * Chalk styling for step indicators in the CLI @@ -49,7 +65,13 @@ export const final: ChalkInstance = chalk.bold.italic * Convenience export for console.log * @type {typeof console.log} */ -export const log: typeof console.log = console.log +export const l: typeof console.log = console.log + +/** + * Convenience export for console.error + * @type {typeof console.log} + */ +export const err: typeof console.error = console.error /** * Available action options for content processing @@ -61,7 +83,7 @@ export const ACTION_OPTIONS = ['video', 'playlist', 'urls', 'file', 'rss'] * Available LLM service options * @type {string[]} */ -export const LLM_OPTIONS = ['chatgpt', 'claude', 'cohere', 'mistral', 'octo', 'llama', 'ollama', 'gemini', 'fireworks', 'together', 'groq'] +export const LLM_OPTIONS = ['chatgpt', 'claude', 'cohere', 'mistral', 'ollama', 'gemini', 'fireworks', 'together', 'groq'] /** * Available transcription service options @@ -159,20 +181,6 @@ export const MISTRAL_MODELS: Record = { MISTRAL_NEMO: "open-mistral-nemo" } -/** - * Mapping of OctoAI model identifiers to their API names. - * @type {Record} - */ -export const OCTO_MODELS: Record = { - LLAMA_3_1_8B: "meta-llama-3.1-8b-instruct", - LLAMA_3_1_70B: "meta-llama-3.1-70b-instruct", - LLAMA_3_1_405B: "meta-llama-3.1-405b-instruct", - MISTRAL_7B: "mistral-7b-instruct", - MIXTRAL_8X_7B: "mixtral-8x7b-instruct", - NOUS_HERMES_MIXTRAL_8X_7B: "nous-hermes-2-mixtral-8x7b-dpo", - WIZARD_2_8X_22B: "wizardlm-2-8x22b", -} - /** * Mapping of Fireworks model identifiers to their API names. * @type {Record} @@ -213,33 +221,6 @@ export const GROQ_MODELS: Record = { MIXTRAL_8X7B_32768: 'mixtral-8x7b-32768', } -/** - * Mapping of local model identifiers to their filenames and download URLs. - * @type {Record} - */ -export const LLAMA_MODELS: Record = { - QWEN_2_5_1B: { - filename: "qwen2.5-1.5b-instruct-q6_k.gguf", - url: "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF/resolve/main/qwen2.5-1.5b-instruct-q6_k.gguf" - }, - QWEN_2_5_3B: { - filename: "qwen2.5-3b-instruct-q6_k.gguf", - url: "https://huggingface.co/Qwen/Qwen2.5-3B-Instruct-GGUF/resolve/main/qwen2.5-3b-instruct-q6_k.gguf" - }, - PHI_3_5: { - filename: "Phi-3.5-mini-instruct-Q6_K.gguf", - url: "https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q6_K.gguf" - }, - LLAMA_3_2_1B: { - filename: "Llama-3.2-1B.i1-Q6_K.gguf", - url: "https://huggingface.co/mradermacher/Llama-3.2-1B-i1-GGUF/resolve/main/Llama-3.2-1B.i1-Q6_K.gguf" - }, - GEMMA_2_2B: { - filename: "gemma-2-2b-it-Q6_K.gguf", - url: "https://huggingface.co/lmstudio-community/gemma-2-2b-it-GGUF/resolve/main/gemma-2-2b-it-Q6_K.gguf" - } -} - /** * Mapping of model identifiers to their corresponding names in Ollama. * @type {Record} diff --git a/src/interactive.ts b/src/interactive.ts index d7c3e10..b11a068 100644 --- a/src/interactive.ts +++ b/src/interactive.ts @@ -2,7 +2,7 @@ import inquirer from 'inquirer' import type { ProcessingOptions, InquirerAnswers, WhisperModelType } from './types.js' -import { log } from './models.js' +import { l } from './globals.js' /** * Prompts the user for input if interactive mode is selected. @@ -122,14 +122,12 @@ export async function handleInteractivePrompt( message: 'Select the Language Model (LLM) you want to use:', choices: [ { name: 'Skip LLM Processing', value: null }, - { name: 'node-llama-cpp (local inference)', value: 'llama' }, { name: 'Ollama (local inference)', value: 'ollama' }, { name: 'OpenAI ChatGPT', value: 'chatgpt' }, { name: 'Anthropic Claude', value: 'claude' }, { name: 'Google Gemini', value: 'gemini' }, { name: 'Cohere', value: 'cohere' }, { name: 'Mistral', value: 'mistral' }, - { name: 'OctoAI', value: 'octo' }, { name: 'Fireworks AI', value: 'fireworks' }, { name: 'Together AI', value: 'together' }, { name: 'Groq', value: 'groq' }, @@ -143,13 +141,6 @@ export async function handleInteractivePrompt( choices: (answers: InquirerAnswers) => { // Return appropriate model choices based on selected LLM service switch (answers.llmServices) { - case 'llama': - return [ - { name: 'LLAMA 3 8B Q4 Model', value: 'LLAMA_3_1_8B_Q4_MODEL' }, - { name: 'LLAMA 3 8B Q6 Model', value: 'LLAMA_3_1_8B_Q6_MODEL' }, - { name: 'GEMMA 2 2B Q4 Model', value: 'GEMMA_2_2B_Q4_MODEL' }, - { name: 'GEMMA 2 2B Q6 Model', value: 'GEMMA_2_2B_Q6_MODEL' }, - ] case 'ollama': return [ { name: 'LLAMA 3 2 1B', value: 'LLAMA_3_2_1B' }, @@ -185,16 +176,6 @@ export async function handleInteractivePrompt( { name: 'Mistral Large', value: 'MISTRAL_LARGE' }, { name: 'Mistral Nemo', value: 'MISTRAL_NEMO' }, ] - case 'octo': - return [ - { name: 'LLAMA 3 1 8B', value: 'LLAMA_3_1_8B' }, - { name: 'LLAMA 3 1 70B', value: 'LLAMA_3_1_70B' }, - { name: 'LLAMA 3 1 405B', value: 'LLAMA_3_1_405B' }, - { name: 'Mistral 7B', value: 'MISTRAL_7B' }, - { name: 'Mixtral 8x7b', value: 'MIXTRAL_8X_7B' }, - { name: 'Nous Hermes Mixtral 8x7b', value: 'NOUS_HERMES_MIXTRAL_8X_7B' }, - { name: 'Wizard 2 8x22b', value: 'WIZARD_2_8X_22B' }, - ] case 'fireworks': return [ { name: 'LLAMA 3 1 405B', value: 'LLAMA_3_1_405B' }, @@ -234,13 +215,11 @@ export async function handleInteractivePrompt( }, when: (answers: InquirerAnswers) => [ - 'llama', 'ollama', 'chatgpt', 'claude', 'cohere', 'mistral', - 'octo', 'fireworks', 'together', 'groq', @@ -283,7 +262,7 @@ export async function handleInteractivePrompt( ['whisper', 'whisperDocker', 'whisperPython', 'whisperDiarization'].includes( answers.transcriptServices as string ), - default: 'large-v2', + default: 'large-v3-turbo', }, // Additional configuration options { @@ -323,7 +302,7 @@ export async function handleInteractivePrompt( ]) // Handle user cancellation if (!answers.confirmAction) { - log('Operation cancelled.') + l('Operation cancelled.') process.exit(0) } // Merge user answers with existing options diff --git a/src/llms/chatgpt.ts b/src/llms/chatgpt.ts index fc74201..9c5948f 100644 --- a/src/llms/chatgpt.ts +++ b/src/llms/chatgpt.ts @@ -3,8 +3,7 @@ import { writeFile } from 'node:fs/promises' import { env } from 'node:process' import { OpenAI } from 'openai' -import { GPT_MODELS } from '../models.js' -import { log, wait } from '../models.js' +import { l, wait, err, GPT_MODELS } from '../globals.js' import type { LLMFunction, ChatGPTModelType } from '../types.js' @@ -54,18 +53,18 @@ export const callChatGPT: LLMFunction = async ( throw new Error('No content generated from the API') } - log(wait(` - Finish Reason: ${finish_reason}\n - ChatGPT Model: ${usedModel}`)) + l(wait(` - Finish Reason: ${finish_reason}\n - ChatGPT Model: ${usedModel}`)) // Check if usage information is available if (usage) { const { prompt_tokens, completion_tokens, total_tokens } = usage - log(wait(` - Token Usage:\n - ${prompt_tokens} prompt tokens\n - ${completion_tokens} completion tokens\n - ${total_tokens} total tokens`)) + l(wait(` - Token Usage:\n - ${prompt_tokens} prompt tokens\n - ${completion_tokens} completion tokens\n - ${total_tokens} total tokens`)) } else { - log(wait(" - Token usage information not available")) + l(wait(" - Token usage information not available")) } } catch (error) { - console.error(`Error in callChatGPT: ${(error as Error).message}`) + err(`Error in callChatGPT: ${(error as Error).message}`) throw error // Re-throw the error for handling in the calling function } } \ No newline at end of file diff --git a/src/llms/claude.ts b/src/llms/claude.ts index 2481ab1..eb3997c 100644 --- a/src/llms/claude.ts +++ b/src/llms/claude.ts @@ -3,8 +3,7 @@ import { writeFile } from 'node:fs/promises' import { env } from 'node:process' import { Anthropic } from '@anthropic-ai/sdk' -import { CLAUDE_MODELS } from '../models.js' -import { log, wait } from '../models.js' +import { l, wait, err, CLAUDE_MODELS } from '../globals.js' import type { LLMFunction, ClaudeModelType } from '../types.js' @@ -58,18 +57,18 @@ export const callClaude: LLMFunction = async ( throw new Error('No text content generated from the API') } - log(wait(` - Stop Reason: ${stop_reason}\n - Model: ${usedModel}`)) + l(wait(` - Stop Reason: ${stop_reason}\n - Model: ${usedModel}`)) // Check if usage information is available if (usage) { const { input_tokens, output_tokens } = usage - log(wait(` - Token Usage:\n - ${input_tokens} input tokens\n - ${output_tokens} output tokens`)) + l(wait(` - Token Usage:\n - ${input_tokens} input tokens\n - ${output_tokens} output tokens`)) } else { - log(wait(" - Token usage information not available")) + l(wait(" - Token usage information not available")) } } catch (error) { - console.error(`Error in callClaude: ${(error as Error).message}`) + err(`Error in callClaude: ${(error as Error).message}`) throw error // Re-throw the error for handling in the calling function } } diff --git a/src/llms/cohere.ts b/src/llms/cohere.ts index edf4fdd..5ae9b50 100644 --- a/src/llms/cohere.ts +++ b/src/llms/cohere.ts @@ -3,8 +3,7 @@ import { writeFile } from 'node:fs/promises' import { env } from 'node:process' import { CohereClient } from 'cohere-ai' -import { COHERE_MODELS } from '../models.js' -import { log, wait } from '../models.js' +import { l, wait, err, COHERE_MODELS } from '../globals.js' import type { LLMFunction, CohereModelType } from '../types.js' @@ -54,18 +53,18 @@ export const callCohere: LLMFunction = async ( throw new Error('No text content generated from the API') } - log(wait(`\n Finish Reason: ${finishReason}\n Model: ${actualModel}`)) + l(wait(`\n Finish Reason: ${finishReason}\n Model: ${actualModel}`)) // Check if token usage information is available if (meta && meta.tokens) { const { inputTokens, outputTokens } = meta.tokens - log(wait(` Token Usage:\n - ${inputTokens} input tokens\n - ${outputTokens} output tokens`)) + l(wait(` Token Usage:\n - ${inputTokens} input tokens\n - ${outputTokens} output tokens`)) } else { - log(wait(" - Token usage information not available")) + l(wait(" - Token usage information not available")) } } catch (error) { - console.error(`Error in callCohere: ${(error as Error).message}`) + err(`Error in callCohere: ${(error as Error).message}`) throw error // Re-throw the error for handling in the calling function } } \ No newline at end of file diff --git a/src/llms/fireworks.ts b/src/llms/fireworks.ts index 060d5df..1a2c45e 100644 --- a/src/llms/fireworks.ts +++ b/src/llms/fireworks.ts @@ -2,7 +2,7 @@ import { writeFile } from 'node:fs/promises' import { env } from 'node:process' -import { log, wait, FIREWORKS_MODELS } from '../models.js' +import { l, wait, err, FIREWORKS_MODELS } from '../globals.js' import type { LLMFunction, FireworksModelType, FireworksResponse } from '../types.js' /** @@ -67,13 +67,13 @@ export const callFireworks: LLMFunction = async ( // Write the generated content to the specified output file await writeFile(tempPath, content) - log(wait(`\n Fireworks response saved to ${tempPath}`)) + l(wait(`\n Fireworks response saved to ${tempPath}`)) // Log finish reason, used model, and token usage - log(wait(`\n Finish Reason: ${finishReason}\n Model Used: ${usedModel}`)) + l(wait(`\n Finish Reason: ${finishReason}\n Model Used: ${usedModel}`)) if (usage) { const { prompt_tokens, completion_tokens, total_tokens } = usage - log( + l( wait( ` Token Usage:\n - ${prompt_tokens} prompt tokens\n - ${completion_tokens} completion tokens\n - ${total_tokens} total tokens` ) @@ -81,7 +81,7 @@ export const callFireworks: LLMFunction = async ( } } catch (error) { // Log any errors that occur during the process - console.error(`Error in callFireworks: ${(error as Error).message}`) + err(`Error in callFireworks: ${(error as Error).message}`) throw error // Re-throw the error for handling by the caller } } \ No newline at end of file diff --git a/src/llms/gemini.ts b/src/llms/gemini.ts index d142e4d..5afde09 100644 --- a/src/llms/gemini.ts +++ b/src/llms/gemini.ts @@ -3,9 +3,7 @@ import { writeFile } from 'node:fs/promises' import { env } from 'node:process' import { GoogleGenerativeAI } from "@google/generative-ai" -import { GEMINI_MODELS } from '../models.js' -import { log, wait } from '../models.js' - +import { l, wait, err, GEMINI_MODELS } from '../globals.js' import type { LLMFunction, GeminiModelType } from '../types.js' /** @@ -63,11 +61,11 @@ export const callGemini: LLMFunction = async ( // Write the generated text to the output file await writeFile(tempPath, text) - log(wait(`\nModel: ${actualModel}`)) + l(wait(`\nModel: ${actualModel}`)) return } catch (error) { - console.error(`Error in callGemini (attempt ${attempt}/${maxRetries}): ${error instanceof Error ? (error as Error).message : String(error)}`) + err(`Error in callGemini (attempt ${attempt}/${maxRetries}): ${error instanceof Error ? (error as Error).message : String(error)}`) // If this is the last attempt, throw the error if (attempt === maxRetries) { diff --git a/src/llms/groq.ts b/src/llms/groq.ts index 2731b6c..3cf698e 100644 --- a/src/llms/groq.ts +++ b/src/llms/groq.ts @@ -2,7 +2,7 @@ import { writeFile } from 'node:fs/promises' import { env } from 'node:process' -import { log, wait, GROQ_MODELS } from '../models.js' +import { l, wait, err, GROQ_MODELS } from '../globals.js' import type { GroqChatCompletionResponse, GroqModelType } from '../types.js' // Define the Groq API URL @@ -66,13 +66,13 @@ export const callGroq = async (promptAndTranscript: string, tempPath: string, mo // Write the generated content to the specified output file await writeFile(tempPath, content) - log(wait(`\n Groq response saved to ${tempPath}`)) + l(wait(`\n Groq response saved to ${tempPath}`)) // Log finish reason, used model, and token usage - log(wait(`\n Finish Reason: ${finishReason}\n Model Used: ${usedModel}`)) + l(wait(`\n Finish Reason: ${finishReason}\n Model Used: ${usedModel}`)) if (usage) { const { prompt_tokens, completion_tokens, total_tokens } = usage - log( + l( wait( ` Token Usage:\n - ${prompt_tokens} prompt tokens\n - ${completion_tokens} completion tokens\n - ${total_tokens} total tokens` ) @@ -80,7 +80,7 @@ export const callGroq = async (promptAndTranscript: string, tempPath: string, mo } } catch (error) { // Log any errors that occur during the process - console.error(`Error in callGroq: ${(error as Error).message}`) + err(`Error in callGroq: ${(error as Error).message}`) throw error // Re-throw the error for handling by the caller } } \ No newline at end of file diff --git a/src/llms/llama.ts b/src/llms/llama.ts deleted file mode 100644 index a849e4d..0000000 --- a/src/llms/llama.ts +++ /dev/null @@ -1,88 +0,0 @@ -// src/llms/llama.ts - -import { writeFile } from 'node:fs/promises' -import { existsSync } from 'node:fs' -import { resolve } from 'node:path' -import { LLAMA_MODELS } from '../models.js' -import { log, success, wait } from '../models.js' -import { getLlama, LlamaModel, LlamaContext, LlamaChatSession } from "node-llama-cpp" -import { createModelDownloader } from 'node-llama-cpp' - -import type { LlamaModelType, LLMFunction } from '../types.js' - -let model: LlamaModel | null = null -let context: LlamaContext | null = null - -/** - * Main function to call the local Llama model using node-llama-cpp API. - * @param promptAndTranscript - The combined prompt and transcript content. - * @param tempPath - The temporary file path to write the LLM output. - * @param modelName - The model name or undefined to use the default model. - * @returns A Promise that resolves when the processing is complete. - * @throws {Error} - If an error occurs during processing. - */ -export const callLlama: LLMFunction = async ( - promptAndTranscript: string, - tempPath: string, - modelName?: string -) => { - try { - // Get the model object from LLAMA_MODELS using the provided model name or default to QWEN_2_5_1B - const selectedModel = LLAMA_MODELS[modelName as LlamaModelType] || LLAMA_MODELS.QWEN_2_5_1B - log(wait(` - filename: ${selectedModel.filename}\n - url: ${selectedModel.url}\n`)) - - // If no valid model is found, throw an error - if (!selectedModel) { - throw new Error(`Invalid model name: ${modelName}`) - } - - // Construct the path where the model file should be stored - const modelDir = resolve('./src/llms/models') - const modelPath = resolve(modelDir, selectedModel.filename) - - // Check if the model file already exists, if not, download it - if (!existsSync(modelPath)) { - log(wait(`\n No model detected, downloading ${selectedModel.filename}...`)) - try { - const downloader = await createModelDownloader({ - modelUri: selectedModel.url, - dirPath: modelDir - }) - await downloader.download() - log(success(' Download completed')) - } catch (err) { - console.error(`Download failed: ${err instanceof Error ? err.message : String(err)}`) - throw new Error('Failed to download the model') - } - } else { - log(wait(` modelPath found:\n - ${modelPath}`)) - } - - // Initialize Llama and load the local model if not already loaded - if (!model || !context) { - const llama = await getLlama() - model = await llama.loadModel({ modelPath }) - context = await model.createContext({ }) - } - - // Create a chat session - const session = new LlamaChatSession({ contextSequence: context.getSequence() }) - - // Generate a response - const response = await session.prompt(promptAndTranscript, { - maxTokens: -1, - temperature: 0.7, - topK: 40, - topP: 0.95, - // repeatPenalty: 1.1 - }) - - // Write the response to the temporary file - await writeFile(tempPath, response) - - log(wait('\n LLM processing completed')) - } catch (error) { - console.error(`Error in callLlama: ${error instanceof Error ? (error as Error).message : String(error)}`) - throw error - } -} \ No newline at end of file diff --git a/src/llms/mistral.ts b/src/llms/mistral.ts index 6ac7ea4..44a9886 100644 --- a/src/llms/mistral.ts +++ b/src/llms/mistral.ts @@ -3,8 +3,7 @@ import { writeFile } from 'node:fs/promises' import { env } from 'node:process' import { Mistral } from '@mistralai/mistralai' -import { MISTRAL_MODELS } from '../models.js' -import { log, wait } from '../models.js' +import { l, wait, err, MISTRAL_MODELS } from '../globals.js' import type { LLMFunction, MistralModelType } from '../types.js' @@ -31,7 +30,7 @@ export const callMistral: LLMFunction = async ( try { // Select the actual model to use, defaulting to MISTRAL_NEMO if the specified model is not found const actualModel = MISTRAL_MODELS[model as MistralModelType] || MISTRAL_MODELS.MISTRAL_NEMO - log(wait(`\n Using Mistral model:\n - ${actualModel}`)) + l(wait(`\n Using Mistral model:\n - ${actualModel}`)) // Make API call to Mistral AI for chat completion const response = await mistral.chat.complete({ @@ -57,12 +56,12 @@ export const callMistral: LLMFunction = async ( // Write the generated content to the specified output file await writeFile(tempPath, content) // Log finish reason, used model, and token usage - log(wait(`\n Finish Reason: ${finishReason}\n Model Used: ${actualModel}`)) - log(wait(` Token Usage:\n - ${promptTokens} prompt tokens\n - ${completionTokens} completion tokens\n - ${totalTokens} total tokens`)) + l(wait(`\n Finish Reason: ${finishReason}\n Model Used: ${actualModel}`)) + l(wait(` Token Usage:\n - ${promptTokens} prompt tokens\n - ${completionTokens} completion tokens\n - ${totalTokens} total tokens`)) } catch (error) { // Log any errors that occur during the process - console.error(`Error in callMistral: ${error instanceof Error ? (error as Error).message : String(error)}`) + err(`Error in callMistral: ${error instanceof Error ? (error as Error).message : String(error)}`) throw error // Re-throw the error for handling by the caller } } \ No newline at end of file diff --git a/src/llms/models/.gitkeep b/src/llms/models/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/llms/octo.ts b/src/llms/octo.ts deleted file mode 100644 index 9895f89..0000000 --- a/src/llms/octo.ts +++ /dev/null @@ -1,57 +0,0 @@ -// src/llms/octo.ts - -import { writeFile } from 'node:fs/promises' -import { env } from 'node:process' -import { OctoAIClient } from '@octoai/sdk' -import { OCTO_MODELS } from '../models.js' -import { log, wait } from '../models.js' - -import type { LLMFunction, OctoModelType } from '../types.js' - -/** - * Main function to call OctoAI API. - * @param promptAndTranscript - The combined prompt and transcript text to process. - * @param tempPath - The temporary file path to write the LLM output. - * @param model - The OctoAI model to use. - * @returns A Promise that resolves when the API call is complete. - * @throws {Error} - If an error occurs during the API call. - */ -export const callOcto: LLMFunction = async (promptAndTranscript: string, tempPath: string, model: string = 'LLAMA_3_1_70B') => { - // Check if the OCTOAI_API_KEY environment variable is set - if (!env.OCTOAI_API_KEY) { - throw new Error('OCTOAI_API_KEY environment variable is not set. Please set it to your OctoAI API key.') - } - // Initialize OctoAI client with API key from environment variables - const octoai = new OctoAIClient({ apiKey: env.OCTOAI_API_KEY }) - - try { - // Select the actual model to use, defaulting to LLAMA_3_1_70B if the specified model is not found - const actualModel = OCTO_MODELS[model as OctoModelType] || OCTO_MODELS.LLAMA_3_1_70B - log(wait(`\n Using OctoAI model:\n - ${actualModel}`)) - - // Make API call to OctoAI for text generation - const response = await octoai.textGen.createChatCompletion({ - model: actualModel, - // max_tokens: ?, // Uncomment and set if you want to limit the response length - messages: [{ role: "user", content: promptAndTranscript }] - }) - - const content = response.choices[0].message.content as string - const finishReason = response.choices[0].finishReason - const usedModel = response.model - const { promptTokens, completionTokens, totalTokens } = response.usage ?? {} - - // Write the generated content to the specified output file - await writeFile(tempPath, content) - log(wait(`\n OctoAI response saved to ${tempPath}`)) - - // Log finish reason, used model, and token usage - log(wait(`\n Finish Reason: ${finishReason}\n Model Used: ${usedModel}`)) - log(wait(` Token Usage:\n - ${promptTokens} prompt tokens\n - ${completionTokens} completion tokens\n - ${totalTokens} total tokens`)) - - } catch (error) { - // Log any errors that occur during the process - console.error(`Error in callOcto: ${(error as Error).message}`) - throw error // Re-throw the error for handling by the caller - } -} \ No newline at end of file diff --git a/src/llms/ollama.ts b/src/llms/ollama.ts index 9fd289e..c2d1b9a 100644 --- a/src/llms/ollama.ts +++ b/src/llms/ollama.ts @@ -2,10 +2,8 @@ import { writeFile } from 'node:fs/promises' import { env } from 'node:process' -import { OLLAMA_MODELS } from '../models.js' -import { log, wait } from '../models.js' +import { OLLAMA_MODELS, l, err, wait } from '../globals.js' import { spawn } from 'child_process' - import type { LLMFunction, OllamaModelType, OllamaResponse, OllamaTagsResponse } from '../types.js' /** @@ -18,11 +16,15 @@ import type { LLMFunction, OllamaModelType, OllamaResponse, OllamaTagsResponse } * @returns A Promise that resolves when the processing is complete. * @throws {Error} - If an error occurs during processing. */ -export const callOllama: LLMFunction = async (promptAndTranscript: string, tempPath: string, modelName: string = 'LLAMA_3_2_1B') => { +export const callOllama: LLMFunction = async ( + promptAndTranscript: string, + tempPath: string, + modelName: string = 'LLAMA_3_2_3B' +) => { try { // Map the model name to the Ollama model identifier - const ollamaModelName = OLLAMA_MODELS[modelName as OllamaModelType] || 'llama3.2:1b' - log(wait(` - modelName: ${modelName}\n - ollamaModelName: ${ollamaModelName}`)) + const ollamaModelName = OLLAMA_MODELS[modelName as OllamaModelType] || 'llama3.2:3b' + l(wait(` - modelName: ${modelName}\n - ollamaModelName: ${ollamaModelName}`)) // Get host and port from environment variables or use defaults const ollamaHost = env.OLLAMA_HOST || 'localhost' @@ -39,14 +41,14 @@ export const callOllama: LLMFunction = async (promptAndTranscript: string, tempP } if (await checkServer()) { - log(wait(' - Ollama server is already running.')) + l(wait('\n Ollama server is already running...')) } else { if (ollamaHost === 'ollama') { // Running inside Docker, do not attempt to start the server throw new Error('Ollama server is not running. Please ensure the Ollama server is running and accessible.') } else { // Not running in Docker, attempt to start the server - log(wait(' - Ollama server is not running. Attempting to start...')) + l(wait('\n Ollama server is not running. Attempting to start...')) const ollamaProcess = spawn('ollama', ['serve'], { detached: true, stdio: 'ignore' @@ -57,7 +59,7 @@ export const callOllama: LLMFunction = async (promptAndTranscript: string, tempP let attempts = 0 while (attempts < 30) { // Wait up to 30 seconds if (await checkServer()) { - log(wait(' - Ollama server is now ready.')) + l(wait(' - Ollama server is now ready.')) break } await new Promise(resolve => setTimeout(resolve, 1000)) @@ -78,7 +80,7 @@ export const callOllama: LLMFunction = async (promptAndTranscript: string, tempP const tagsData = await tagsResponse.json() as OllamaTagsResponse const isModelAvailable = tagsData.models.some(model => model.name === ollamaModelName) if (!isModelAvailable) { - log(wait(`\n Model ${ollamaModelName} is not available, pulling the model...`)) + l(wait(`\n Model ${ollamaModelName} is not available, pulling the model...`)) const pullResponse = await fetch(`http://${ollamaHost}:${ollamaPort}/api/pull`, { method: 'POST', headers: { @@ -104,23 +106,23 @@ export const callOllama: LLMFunction = async (promptAndTranscript: string, tempP try { const response = JSON.parse(line) if (response.status === 'success') { - log(wait(` - Model ${ollamaModelName} has been pulled successfully...\n`)) + l(wait(` - Model ${ollamaModelName} has been pulled successfully...\n`)) break } } catch (parseError) { - console.error(`Error parsing JSON: ${parseError}`) + err(`Error parsing JSON: ${parseError}`) } } } } else { - log(wait(`\n Model ${ollamaModelName} is already available...\n`)) + l(wait(`\n Model ${ollamaModelName} is already available...\n`)) } } catch (error) { - console.error(`Error checking/pulling model: ${error instanceof Error ? error.message : String(error)}`) + err(`Error checking/pulling model: ${error instanceof Error ? error.message : String(error)}`) throw error } - log(wait(` - Sending chat request to http://${ollamaHost}:${ollamaPort} using ${ollamaModelName} model`)) + l(wait(` - Sending chat request to http://${ollamaHost}:${ollamaPort} using ${ollamaModelName} model`)) // Call the Ollama chat API with streaming enabled const response = await fetch(`http://${ollamaHost}:${ollamaPort}/api/chat`, { @@ -162,17 +164,17 @@ export const callOllama: LLMFunction = async (promptAndTranscript: string, tempP const parsedResponse = JSON.parse(line) as OllamaResponse if (parsedResponse.message?.content) { if (isFirstChunk) { - log(wait(` - Receiving streaming response from Ollama...`)) + l(wait(` - Receiving streaming response from Ollama...`)) isFirstChunk = false } fullContent += parsedResponse.message.content } if (parsedResponse.done) { - log(wait(` - Completed receiving response from Ollama.`)) + l(wait(` - Completed receiving response from Ollama.`)) } } catch (parseError) { - console.error(`Error parsing JSON: ${parseError}`) + err(`Error parsing JSON: ${parseError}`) } } } @@ -180,8 +182,8 @@ export const callOllama: LLMFunction = async (promptAndTranscript: string, tempP // Write the full content to the output file await writeFile(tempPath, fullContent) } catch (error) { - console.error(`Error in callOllama: ${error instanceof Error ? error.message : String(error)}`) - console.error(`Stack Trace: ${error instanceof Error ? error.stack : 'No stack trace available'}`) + err(`Error in callOllama: ${error instanceof Error ? error.message : String(error)}`) + err(`Stack Trace: ${error instanceof Error ? error.stack : 'No stack trace available'}`) throw error } } \ No newline at end of file diff --git a/src/llms/together.ts b/src/llms/together.ts index dff3207..0d4e6e5 100644 --- a/src/llms/together.ts +++ b/src/llms/together.ts @@ -2,7 +2,7 @@ import { writeFile } from 'node:fs/promises' import { env } from 'node:process' -import { log, wait, TOGETHER_MODELS } from '../models.js' +import { l, wait, err, TOGETHER_MODELS } from '../globals.js' import type { LLMFunction, TogetherModelType, TogetherResponse } from '../types.js' /** @@ -70,13 +70,13 @@ export const callTogether: LLMFunction = async ( // Write the generated content to the specified output file await writeFile(tempPath, content) - log(wait(`\n Together AI response saved to ${tempPath}`)) + l(wait(`\n Together AI response saved to ${tempPath}`)) // Log finish reason, used model, and token usage - log(wait(`\n Finish Reason: ${finishReason}\n Model Used: ${usedModel}`)) + l(wait(`\n Finish Reason: ${finishReason}\n Model Used: ${usedModel}`)) if (usage) { const { prompt_tokens, completion_tokens, total_tokens } = usage - log( + l( wait( ` Token Usage:\n - ${prompt_tokens} prompt tokens\n - ${completion_tokens} completion tokens\n - ${total_tokens} total tokens` ) @@ -84,7 +84,7 @@ export const callTogether: LLMFunction = async ( } } catch (error) { // Log any errors that occur during the process - console.error(`Error in callTogether: ${(error as Error).message}`) + err(`Error in callTogether: ${(error as Error).message}`) throw error // Re-throw the error for handling by the caller } } \ No newline at end of file diff --git a/src/transcription/assembly.ts b/src/transcription/assembly.ts index d8d0914..f8f9327 100644 --- a/src/transcription/assembly.ts +++ b/src/transcription/assembly.ts @@ -2,7 +2,7 @@ import { createReadStream } from 'node:fs' import { writeFile } from 'node:fs/promises' import { env } from 'node:process' import fetch from 'node-fetch' -import { log, wait, success } from '../models.js' +import { l, wait, success, err } from '../globals.js' import type { ProcessingOptions } from '../types.js' const BASE_URL = 'https://api.assemblyai.com/v2' @@ -15,7 +15,7 @@ const BASE_URL = 'https://api.assemblyai.com/v2' * @throws {Error} - If an error occurs during transcription. */ export async function callAssembly(options: ProcessingOptions, finalPath: string): Promise { - log(wait('\n Using AssemblyAI for transcription...')) + l(wait('\n Using AssemblyAI for transcription...')) // Check if the ASSEMBLY_API_KEY environment variable is set if (!env.ASSEMBLY_API_KEY) { throw new Error('ASSEMBLY_API_KEY environment variable is not set. Please set it to your AssemblyAI API key.') @@ -31,7 +31,7 @@ export async function callAssembly(options: ProcessingOptions, finalPath: string const audioFilePath = `${finalPath}.wav` // Step 1: Upload the audio file - log(wait('\n Uploading audio file to AssemblyAI...')) + l(wait('\n Uploading audio file to AssemblyAI...')) const uploadUrl = `${BASE_URL}/upload` const fileStream = createReadStream(audioFilePath) @@ -54,7 +54,7 @@ export async function callAssembly(options: ProcessingOptions, finalPath: string if (!upload_url) { throw new Error('Upload URL not returned by AssemblyAI.') } - log(success(' Audio file uploaded successfully.')) + l(success(' Audio file uploaded successfully.')) // Step 2: Request transcription const response = await fetch(`${BASE_URL}/transcript`, { @@ -129,16 +129,16 @@ export async function callAssembly(options: ProcessingOptions, finalPath: string // Write the formatted transcript to a file await writeFile(`${finalPath}.txt`, txtContent) - log(wait(`\n Transcript saved...\n - ${finalPath}.txt\n`)) + l(wait(`\n Transcript saved...\n - ${finalPath}.txt\n`)) // Create an empty LRC file to prevent cleanup errors await writeFile(`${finalPath}.lrc`, '') - log(wait(`\n Empty LRC file created:\n - ${finalPath}.lrc\n`)) + l(wait(`\n Empty LRC file created:\n - ${finalPath}.lrc\n`)) return txtContent } catch (error) { // Log any errors that occur during the transcription process - console.error(`Error processing the transcription: ${(error as Error).message}`) + err(`Error processing the transcription: ${(error as Error).message}`) throw error // Re-throw the error for handling in the calling function } } \ No newline at end of file diff --git a/src/transcription/deepgram.ts b/src/transcription/deepgram.ts index 14e5fd8..9998416 100644 --- a/src/transcription/deepgram.ts +++ b/src/transcription/deepgram.ts @@ -2,7 +2,7 @@ import { writeFile, readFile } from 'node:fs/promises' import { env } from 'node:process' -import { log, wait } from '../models.js' +import { l, wait, err } from '../globals.js' import type { ProcessingOptions, DeepgramResponse } from '../types.js' /** @@ -13,10 +13,10 @@ import type { ProcessingOptions, DeepgramResponse } from '../types.js' * @throws {Error} - If an error occurs during transcription. */ export async function callDeepgram(options: ProcessingOptions, finalPath: string): Promise { - log(wait('\n Using Deepgram for transcription...\n')) - // log(`Options received in callDeepgram:\n`) - // log(options) - // log(`finalPath:`, finalPath) + l(wait('\n Using Deepgram for transcription...\n')) + // l(`Options received in callDeepgram:\n`) + // l(options) + // l(`finalPath:`, finalPath) // Check if the DEEPGRAM_API_KEY environment variable is set if (!env.DEEPGRAM_API_KEY) { @@ -77,16 +77,16 @@ export async function callDeepgram(options: ProcessingOptions, finalPath: string // Write the formatted transcript to a file await writeFile(`${finalPath}.txt`, txtContent) - log(wait(`\n Transcript saved:\n - ${finalPath}.txt\n`)) + l(wait(`\n Transcript saved:\n - ${finalPath}.txt\n`)) // Create an empty LRC file to prevent cleanup errors await writeFile(`${finalPath}.lrc`, '') - log(wait(`\n Empty LRC file created:\n - ${finalPath}.lrc\n`)) + l(wait(`\n Empty LRC file created:\n - ${finalPath}.lrc\n`)) return txtContent } catch (error) { // Log any errors that occur during the transcription process - console.error(`Error processing the transcription: ${(error as Error).message}`) + err(`Error processing the transcription: ${(error as Error).message}`) throw error // Re-throw the error for handling in the calling function } } \ No newline at end of file diff --git a/src/transcription/whisper.ts b/src/transcription/whisper.ts index 759c66c..a8e7471 100644 --- a/src/transcription/whisper.ts +++ b/src/transcription/whisper.ts @@ -1,75 +1,285 @@ // src/transcription/whisper.ts -import { readFile, writeFile } from 'node:fs/promises' -import { exec } from 'node:child_process' -import { promisify } from 'node:util' -import { existsSync } from 'node:fs' -import { log, wait, WHISPER_MODELS } from '../models.js' -import type { ProcessingOptions, WhisperModelType } from '../types.js' +/** + * @file Handles transcription using various Whisper implementations. + * Combines multiple Whisper-based transcription methods into a single file. + * @packageDocumentation + */ -const execPromise = promisify(exec) +import { readFile, writeFile, unlink } from 'node:fs/promises' +import { existsSync } from 'node:fs' +import { l, err, wait, success, WHISPER_MODELS, WHISPER_PYTHON_MODELS, execPromise } from '../globals.js' +import type { ProcessingOptions, WhisperModelType, WhisperTranscriptServices } from '../types.js' /** * Main function to handle transcription using Whisper. - * @param {string} finalPath - The base path for the files. - * @param {ProcessingOptions} options - Additional processing options. - * @returns {Promise} - Returns the formatted transcript content. - * @throws {Error} - If an error occurs during transcription. + * @param options - Additional processing options. + * @param finalPath - The base path for the files. + * @param transcriptServices - The Whisper transcription service to use. + * @returns Returns the formatted transcript content. + * @throws If an error occurs during transcription. */ -export async function callWhisper(options: ProcessingOptions, finalPath: string): Promise { - log(wait('\n Using Whisper for transcription...')) +export async function callWhisper( + options: ProcessingOptions, + finalPath: string, + transcriptServices: WhisperTranscriptServices +): Promise { + l(wait(`\n Using ${transcriptServices} for transcription...`)) + try { - // Get the whisper model from options or use 'base' as default - let whisperModel = 'base' - if (typeof options.whisper === 'string') { - whisperModel = options.whisper - } else if (options.whisper !== true) { - throw new Error('Invalid whisper option') - } + // Configuration object mapping each service to its specific settings + const serviceConfig = { + whisper: { + option: options.whisper, + modelList: WHISPER_MODELS, + runner: runWhisperCpp + }, + whisperDocker: { + option: options.whisperDocker, + modelList: WHISPER_MODELS, + runner: runWhisperDocker + }, + whisperPython: { + option: options.whisperPython, + modelList: WHISPER_PYTHON_MODELS, + runner: runWhisperPython + }, + whisperDiarization: { + option: options.whisperDiarization, + modelList: WHISPER_PYTHON_MODELS, + runner: runWhisperDiarization + } + } as const + + // Retrieve the configuration for the specified service + const config = serviceConfig[transcriptServices] + + // Determine the model based on the option type + const whisperModel = typeof config.option === 'string' + ? config.option + : config.option === true + ? 'base' + : (() => { throw new Error(`Invalid ${transcriptServices} option`) })() - if (!(whisperModel in WHISPER_MODELS)) { + // Validate the model + if (!(whisperModel in config.modelList)) { throw new Error(`Unknown model type: ${whisperModel}`) } - // Get the model ggml file name - const modelGGMLName = WHISPER_MODELS[whisperModel as WhisperModelType] + l(wait(`\n - whisperModel: ${whisperModel}`)) + + // Run the appropriate transcription method + await config.runner(options, finalPath, whisperModel) - log(wait(`\n - whisperModel: ${whisperModel}\n - modelGGMLName: ${modelGGMLName}`)) + // Read the generated transcript file (assuming it's always `${finalPath}.txt`) + const txtContent = await readFile(`${finalPath}.txt`, 'utf8') + return txtContent + } catch (error) { + err(`Error in callWhisper with ${transcriptServices}:`, (error as Error).message) + process.exit(1) + } +} - // Setup Whisper - if (!existsSync('./whisper.cpp')) { - log(`\n No whisper.cpp repo found, running git clone and make...\n`) - await execPromise('git clone https://github.com/ggerganov/whisper.cpp.git && make -C whisper.cpp && cp .github/whisper.Dockerfile whisper.cpp/Dockerfile') - log(`\n - whisper.cpp clone and make commands complete.\n`) - } +// Helper functions for each transcription method - // Ensure model is downloaded - if (!existsSync(`./whisper.cpp/models/ggml-${whisperModel}.bin`)) { - log(wait(`\n Model not found, downloading...\n - ${whisperModel}\n`)) - await execPromise(`bash ./whisper.cpp/models/download-ggml-model.sh ${whisperModel}`) - log(wait(' - Model download completed, running transcription...\n')) - } +/** + * Transcribes audio using the local Whisper.cpp implementation. + * @param options - Processing options. + * @param finalPath - Base path for files. + * @param whisperModel - The Whisper model to use. + */ +async function runWhisperCpp(options: ProcessingOptions, finalPath: string, whisperModel: string): Promise { + const modelGGMLName = WHISPER_MODELS[whisperModel as WhisperModelType] + l(wait(` - modelGGMLName: ${modelGGMLName}`)) - // Run transcription - await execPromise(`./whisper.cpp/main -m "whisper.cpp/models/${modelGGMLName}" -f "${finalPath}.wav" -of "${finalPath}" --output-lrc`) - log(wait(`\n Transcript LRC file successfully completed...\n - ${finalPath}.lrc`)) + // Setup Whisper.cpp if not already present + if (!existsSync('./whisper.cpp')) { + l(wait(`\n No whisper.cpp repo found, cloning and compiling...\n`)) + await execPromise('git clone https://github.com/ggerganov/whisper.cpp.git && make -C whisper.cpp') + l(wait(`\n - whisper.cpp clone and compilation complete.\n`)) + } - // Read the generated LRC file - const lrcContent = await readFile(`${finalPath}.lrc`, 'utf8') - // Process and format the LRC content - const txtContent = lrcContent.split('\n') - .filter(line => !line.startsWith('[by:whisper.cpp]')) - .map(line => line.replace(/\[(\d{2,3}):(\d{2})\.(\d{2})\]/g, (_, p1, p2) => `[${p1}:${p2}]`)) - .join('\n') + // Ensure the model is downloaded + if (!existsSync(`./whisper.cpp/models/${modelGGMLName}`)) { + l(wait(`\n Model not found, downloading...\n - ${whisperModel}\n`)) + await execPromise(`bash ./whisper.cpp/models/download-ggml-model.sh ${whisperModel}`) + l(wait(' - Model download completed, running transcription...\n')) + } - // Write the formatted content to a text file - await writeFile(`${finalPath}.txt`, txtContent) - log(wait(` Transcript transformation successfully completed...\n - ${finalPath}.txt\n`)) + // Run transcription + await execPromise(`./whisper.cpp/main -m "whisper.cpp/models/${modelGGMLName}" -f "${finalPath}.wav" -of "${finalPath}" --output-lrc`) + l(success(`\n Transcript LRC file successfully created:\n - ${finalPath}.lrc`)) - // Return the processed content - return txtContent + // Process the LRC file into a TXT file + const lrcContent = await readFile(`${finalPath}.lrc`, 'utf8') + const txtContent = lrcToTxt(lrcContent) + await writeFile(`${finalPath}.txt`, txtContent) + l(success(` Transcript transformation successfully completed:\n - ${finalPath}.txt\n`)) +} + +/** + * Transcribes audio using Whisper.cpp inside a Docker container. + * @param options - Processing options. + * @param finalPath - Base path for files. + * @param whisperModel - The Whisper model to use. + */ +async function runWhisperDocker(options: ProcessingOptions, finalPath: string, whisperModel: string): Promise { + const modelGGMLName = WHISPER_MODELS[whisperModel as WhisperModelType] + const CONTAINER_NAME = 'autoshow-whisper-1' + const modelPathContainer = `/app/models/${modelGGMLName}` + + l(wait(` - modelGGMLName: ${modelGGMLName}`)) + l(wait(` - CONTAINER_NAME: ${CONTAINER_NAME}`)) + l(wait(` - modelPathContainer: ${modelPathContainer}`)) + + // Ensure the Docker container is running + await execPromise(`docker ps | grep ${CONTAINER_NAME}`) + .catch(() => execPromise('docker-compose up -d whisper')) + + // Ensure the model is downloaded inside the container + await execPromise(`docker exec ${CONTAINER_NAME} test -f ${modelPathContainer}`) + .catch(() => execPromise(`docker exec ${CONTAINER_NAME} /app/models/download-ggml-model.sh ${whisperModel}`)) + + // Run transcription inside the container + await execPromise( + `docker exec ${CONTAINER_NAME} /app/main -m ${modelPathContainer} -f "/app/${finalPath}.wav" -of "/app/${finalPath}" --output-lrc` + ) + l(success(`\n Transcript LRC file successfully created:\n - ${finalPath}.lrc`)) + + // Process the LRC file into a TXT file + const lrcContent = await readFile(`${finalPath}.lrc`, 'utf8') + const txtContent = lrcToTxt(lrcContent) + await writeFile(`${finalPath}.txt`, txtContent) + l(success(` Transcript transformation successfully completed:\n - ${finalPath}.txt`)) +} + +/** + * Transcribes audio using the openai-whisper Python library. + * @param options - Processing options. + * @param finalPath - Base path for files. + * @param whisperModel - The Whisper model to use. + */ +async function runWhisperPython(options: ProcessingOptions, finalPath: string, whisperModel: string): Promise { + // Check if ffmpeg is installed + try { + await execPromise('ffmpeg -version') } catch (error) { - console.error('Error in callWhisper:', error) - process.exit(1) + throw new Error('ffmpeg is not installed or not available in PATH') + } + + // Check if Python is installed + try { + await execPromise('python3 --version') + } catch (error) { + throw new Error('Python is not installed or not available in PATH') } -} \ No newline at end of file + + // Check if the openai-whisper package is installed + try { + await execPromise('which whisper') + } catch (error) { + l(wait('\n openai-whisper not found, installing...')) + await execPromise('pip install -U openai-whisper') + l(wait(' - openai-whisper installed')) + } + + // Prepare the command to run the transcription + const command = `whisper "${finalPath}.wav" --model ${whisperModel} --output_dir "content" --output_format srt --language en --word_timestamps True` + + l(wait(`\n Running transcription with command:\n ${command}\n`)) + + // Execute the command + await execPromise(command) + + // Process the SRT file into a TXT file + const srtContent = await readFile(`${finalPath}.srt`, 'utf8') + const txtContent = srtToTxt(srtContent) + await writeFile(`${finalPath}.txt`, txtContent) + l(wait(`\n Transcript transformation successfully completed:\n - ${finalPath}.txt\n`)) + + // Create an empty LRC file to prevent cleanup errors and unlink SRT file + await writeFile(`${finalPath}.lrc`, '') + l(wait(`\n Empty LRC file created:\n - ${finalPath}.lrc\n`)) + await unlink(`${finalPath}.srt`) + l(wait(`\n SRT file deleted:\n - ${finalPath}.srt\n`)) +} + +/** + * Transcribes audio using Whisper with speaker diarization. + * @param options - Processing options. + * @param finalPath - Base path for files. + * @param whisperModel - The Whisper model to use. + */ +async function runWhisperDiarization(options: ProcessingOptions, finalPath: string, whisperModel: string): Promise { + // Check if the virtual environment exists + const venvPythonPath = 'whisper-diarization/venv/bin/python' + if (!existsSync(venvPythonPath)) { + l(wait(`\n Virtual environment not found, running setup script...\n`)) + await execPromise('bash scripts/setup-python.sh') + l(wait(` - whisper-diarization setup complete.\n`)) + } + + // Prepare the command to run the transcription + const command = `${venvPythonPath} whisper-diarization/diarize.py -a ${finalPath}.wav --whisper-model ${whisperModel}` + l(wait(`\n Running transcription with command:\n ${command}\n`)) + + // Execute the command + await execPromise(command) + await unlink(`${finalPath}.txt`) + + // Process the SRT file into a TXT file + const srtContent = await readFile(`${finalPath}.srt`, 'utf8') + const txtContent = srtToTxt(srtContent) + await writeFile(`${finalPath}.txt`, txtContent) + l(wait(`\n Transcript transformation successfully completed:\n - ${finalPath}.txt\n`)) + + // Create an empty LRC file to prevent cleanup errors and unlink SRT file + await writeFile(`${finalPath}.lrc`, '') + l(success(` Empty LRC file created:\n - ${finalPath}.lrc`)) + await unlink(`${finalPath}.srt`) + l(success(` SRT file deleted:\n - ${finalPath}.srt`)) +} + +// Helper functions for processing transcript files + +/** + * Converts LRC content to plain text with timestamps. + * @param lrcContent - The content of the LRC file. + * @returns The converted text content. + */ +function lrcToTxt(lrcContent: string): string { + return lrcContent.split('\n') + .filter(line => !line.startsWith('[by:whisper.cpp]')) + .map(line => line.replace(/\[(\d{2,3}):(\d{2})\.(\d{2})\]/g, (_, p1, p2) => `[${p1}:${p2}]`)) + .join('\n') +} + +/** + * Converts SRT content to plain text with timestamps. + * @param srtContent - The content of the SRT file. + * @returns The converted text content. + */ +function srtToTxt(srtContent: string): string { + const blocks = srtContent.split('\n\n') + return blocks + .map(block => { + const lines = block.split('\n').filter(line => line.trim() !== '') + if (lines.length >= 2) { + const timestampLine = lines[1] + const textLines = lines.slice(2) + const match = timestampLine.match(/(\d{2}):(\d{2}):(\d{2}),\d{3}/) + if (match) { + const hours = parseInt(match[1], 10) + const minutes = parseInt(match[2], 10) + const seconds = match[3] + const totalMinutes = hours * 60 + minutes + const timestamp = `[${String(totalMinutes).padStart(2, '0')}:${seconds}]` + const text = textLines.join(' ') + return `${timestamp} ${text}` + } + } + return null + }) + .filter(line => line !== null) + .join('\n') +} diff --git a/src/transcription/whisperDiarization.ts b/src/transcription/whisperDiarization.ts deleted file mode 100644 index 418c937..0000000 --- a/src/transcription/whisperDiarization.ts +++ /dev/null @@ -1,102 +0,0 @@ -// src/transcription/whisperDiarization.ts - -import { readFile, writeFile, unlink } from 'node:fs/promises' -import { exec } from 'node:child_process' -import { promisify } from 'node:util' -import { existsSync } from 'node:fs' -import { log, wait, WHISPER_PYTHON_MODELS } from '../models.js' -import type { ProcessingOptions } from '../types.js' - -const execPromise = promisify(exec) - -/** - * Main function to handle transcription using openai-whisper Python library. - * @param {ProcessingOptions} options - Additional processing options. - * @param {string} finalPath - The base path for the files. - * @returns {Promise} - Returns the formatted transcript content. - * @throws {Error} - If an error occurs during transcription. - */ -export async function callWhisperDiarization(options: ProcessingOptions, finalPath: string): Promise { - log(wait('\n Using whisper-diarization for transcription...')) - - try { - // Get the whisper model from options or use 'base' as default - let whisperModel: string = 'base' - if (typeof options.whisperDiarization === 'string') { - whisperModel = options.whisperDiarization - } else if (options.whisperDiarization !== true) { - throw new Error('Invalid whisperDiarization option') - } - - if (!(whisperModel in WHISPER_PYTHON_MODELS)) { - throw new Error(`Unknown model type: ${whisperModel}`) - } - - log(wait(`\n - whisperModel: ${whisperModel}`)) - - // Check if the virtual environment exists - const venvPythonPath = 'whisper-diarization/venv/bin/python' - if (!existsSync(venvPythonPath)) { - log(wait(`\n Virtual environment not found, running setup script...\n`)) - await execPromise('bash scripts/setup-python.sh') - log(wait(` - whisper-diarization setup complete.\n`)) - } - - // Prepare the command to run the transcription, example command after interpolation: - // whisper-diarization/venv/bin/python whisper-diarization/diarize.py -a content/audio.wav --whisper-model tiny - const command = `${venvPythonPath} whisper-diarization/diarize.py -a ${finalPath}.wav --whisper-model ${whisperModel}` - log(wait(`\n Running transcription with command:\n\n ${command}\n`)) - - // Execute the command - await execPromise(command) - await unlink(`${finalPath}.txt`) - - // Read the generated transcript file - const srtContent = await readFile(`${finalPath}.srt`, 'utf8') - - // Process and format the SRT content - const blocks = srtContent.split('\n\n') - - const txtContent = blocks - .map(block => { - const lines = block.split('\n').filter(line => line.trim() !== '') - if (lines.length >= 2) { - // lines[0] is the sequence number - // lines[1] is the timestamp line - // lines[2...] are the subtitle text lines - const timestampLine = lines[1] - const textLines = lines.slice(2) - const match = timestampLine.match(/(\d{2}):(\d{2}):(\d{2}),\d{3}/) - if (match) { - const hours = parseInt(match[1], 10) - const minutes = parseInt(match[2], 10) - const seconds = match[3] - const totalMinutes = hours * 60 + minutes - const timestamp = `[${String(totalMinutes).padStart(2, '0')}:${seconds}]` - const text = textLines.join(' ') - return `${timestamp} ${text}` - } - } - return null - }) - .filter(line => line !== null) - .join('\n') - - // Write the formatted content to a text file - await writeFile(`${finalPath}.txt`, txtContent) - log(wait(`\n Transcript transformation successfully completed:\n - ${finalPath}.txt\n`)) - - // Create an empty LRC file to prevent cleanup errors and unlink SRT file - await writeFile(`${finalPath}.lrc`, '') - log(wait(` Empty LRC file created:\n - ${finalPath}.lrc`)) - await unlink(`${finalPath}.srt`) - log(wait(` SRT file deleted:\n - ${finalPath}.srt`)) - - // Return the processed content - return txtContent - - } catch (error) { - console.error('Error in callWhisperDiarization:', (error as Error).message) - process.exit(1) - } -} \ No newline at end of file diff --git a/src/transcription/whisperDocker.ts b/src/transcription/whisperDocker.ts deleted file mode 100644 index 09659dd..0000000 --- a/src/transcription/whisperDocker.ts +++ /dev/null @@ -1,76 +0,0 @@ -// src/transcription/whisperDocker.ts - -import { readFile, writeFile } from 'node:fs/promises' -import { exec } from 'node:child_process' -import { promisify } from 'node:util' -import { join } from 'node:path' -import { log, wait, WHISPER_MODELS } from '../models.js' -import type { ProcessingOptions, WhisperModelType } from '../types.js' - -const execPromise = promisify(exec) - -/** - * Main function to handle transcription using Whisper Docker. - * @param {string} finalPath - The base path for the files. - * @param {ProcessingOptions} options - Additional processing options. - * @returns {Promise} - Returns the formatted transcript content. - * @throws {Error} - If an error occurs during transcription. - */ -export async function callWhisperDocker(options: ProcessingOptions, finalPath: string): Promise { - log(wait('\n Using Whisper Docker for transcription...')) - try { - // Get the whisper model from options or use 'base' as default - let whisperModel = 'base' - if (typeof options.whisperDocker === 'string') { - whisperModel = options.whisperDocker - } else if (options.whisperDocker !== true) { - throw new Error('Invalid whisperDocker option') - } - - if (!(whisperModel in WHISPER_MODELS)) { - throw new Error(`Unknown model type: ${whisperModel}`) - } - - // Get the model ggml file name - const modelGGMLName = WHISPER_MODELS[whisperModel as WhisperModelType] - const CONTAINER_NAME = 'autoshow-whisper-1' - const modelPathContainer = `/app/models/${modelGGMLName}` - - log(wait(` - whisperModel: ${whisperModel}`)) - log(wait(` - modelGGMLName: ${modelGGMLName}`)) - log(wait(` - CONTAINER_NAME: ${CONTAINER_NAME}`)) - log(wait(` - modelPathContainer: ${modelPathContainer}`)) - - // Ensure container is running - await execPromise(`docker ps | grep ${CONTAINER_NAME}`) - .catch(() => execPromise('docker-compose up -d whisper')) - - // Ensure model is downloaded - await execPromise(`docker exec ${CONTAINER_NAME} test -f ${modelPathContainer}`) - .catch(() => execPromise(`docker exec ${CONTAINER_NAME} /app/models/download-ggml-model.sh ${whisperModel}`)) - - // Run transcription - await execPromise( - `docker exec ${CONTAINER_NAME} /app/main -m ${modelPathContainer} -f ${join(`/app`, `${finalPath}.wav`)} -of ${join(`/app`, finalPath)} --output-lrc` - ) - log(wait(`\n Transcript LRC file successfully completed...\n - ${finalPath}.lrc\n`)) - - // Process transcript - const lrcContent = await readFile(`${finalPath}.lrc`, 'utf8') - // Process and format the LRC content - const txtContent = lrcContent.split('\n') - .filter(line => !line.startsWith('[by:whisper.cpp]')) - .map(line => line.replace(/\[(\d{2,3}):(\d{2})\.(\d{2})\]/g, (_, p1, p2) => `[${p1}:${p2}]`)) - .join('\n') - - // Write the formatted content to a text file - await writeFile(`${finalPath}.txt`, txtContent) - log(wait(` Transcript transformation successfully completed...\n - ${finalPath}.txt\n`)) - - // Return the processed content - return txtContent - } catch (error) { - console.error('Error in callWhisperDocker:', error) - process.exit(1) - } -} \ No newline at end of file diff --git a/src/transcription/whisperPython.ts b/src/transcription/whisperPython.ts deleted file mode 100644 index 52374d7..0000000 --- a/src/transcription/whisperPython.ts +++ /dev/null @@ -1,117 +0,0 @@ -// src/transcription/whisperPython.ts - -import { readFile, writeFile, unlink } from 'node:fs/promises' -import { exec } from 'node:child_process' -import { promisify } from 'node:util' -import { log, wait, WHISPER_PYTHON_MODELS } from '../models.js' -import type { ProcessingOptions } from '../types.js' - -const execPromise = promisify(exec) - -/** - * Main function to handle transcription using openai-whisper Python library. - * @param {ProcessingOptions} options - Additional processing options. - * @param {string} finalPath - The base path for the files. - * @returns {Promise} - Returns the formatted transcript content. - * @throws {Error} - If an error occurs during transcription. - */ -export async function callWhisperPython(options: ProcessingOptions, finalPath: string): Promise { - log(wait('\n Using openai-whisper Python library for transcription...')) - - try { - // Get the whisper model from options or use 'base' as default - let whisperModel: string = 'base' - if (typeof options.whisperPython === 'string') { - whisperModel = options.whisperPython - } else if (options.whisperPython !== true) { - throw new Error('Invalid whisperPython option') - } - - if (!(whisperModel in WHISPER_PYTHON_MODELS)) { - throw new Error(`Unknown model type: ${whisperModel}`) - } - - log(wait(`\n - whisperModel: ${whisperModel}`)) - - // Check if ffmpeg is installed - try { - await execPromise('ffmpeg -version') - } catch (error) { - throw new Error('ffmpeg is not installed or not available in PATH') - } - - // Check if Python is installed - try { - await execPromise('python3 --version') - } catch (error) { - throw new Error('Python is not installed or not available in PATH') - } - - // Check if the openai-whisper package is installed - try { - // await execPromise('python3 -c "import whisper"') - await execPromise('which whisper') - } catch (error) { - log(wait('\n openai-whisper not found, installing...')) - // await execPromise('pip install -U openai-whisper') - await execPromise('brew install openai-whisper') - log(wait(' - openai-whisper installed')) - } - - // Prepare the command to run the transcription - const command = `whisper "${finalPath}.wav" --model ${whisperModel} --output_dir "content" --output_format srt --language en --word_timestamps True` - - log(wait(`\n Running transcription with command:\n ${command}\n`)) - - // Execute the command - await execPromise(command) - - // Read the generated transcript file - const srtContent = await readFile(`${finalPath}.srt`, 'utf8') - - // Process and format the SRT content - const blocks = srtContent.split('\n\n') - - const txtContent = blocks - .map(block => { - const lines = block.split('\n').filter(line => line.trim() !== '') - if (lines.length >= 2) { - // lines[0] is the sequence number - // lines[1] is the timestamp line - // lines[2...] are the subtitle text lines - const timestampLine = lines[1] - const textLines = lines.slice(2) - const match = timestampLine.match(/(\d{2}):(\d{2}):(\d{2}),\d{3}/) - if (match) { - const hours = parseInt(match[1], 10) - const minutes = parseInt(match[2], 10) - const seconds = match[3] - const totalMinutes = hours * 60 + minutes - const timestamp = `[${String(totalMinutes).padStart(2, '0')}:${seconds}]` - const text = textLines.join(' ') - return `${timestamp} ${text}` - } - } - return null - }) - .filter(line => line !== null) - .join('\n') - - // Write the formatted content to a text file - await writeFile(`${finalPath}.txt`, txtContent) - log(wait(`\n Transcript transformation successfully completed...\n - ${finalPath}.txt\n`)) - - // Create an empty LRC file to prevent cleanup errors and unlink SRT file - await writeFile(`${finalPath}.lrc`, '') - log(wait(`\n Empty LRC file created:\n - ${finalPath}.lrc\n`)) - await unlink(`${finalPath}.srt`) - log(wait(`\n SRT file deleted:\n - ${finalPath}.srt\n`)) - - // Return the processed content - return txtContent - - } catch (error) { - console.error('Error in callWhisperPython:', (error as Error).message) - process.exit(1) - } -} diff --git a/src/types.ts b/src/types.ts index 6eef56a..2c494ba 100644 --- a/src/types.ts +++ b/src/types.ts @@ -48,8 +48,6 @@ export type ProcessingOptions = { cohere?: string /** Mistral model to use (e.g., 'MISTRAL_LARGE'). */ mistral?: string - /** OctoAI model to use (e.g., 'LLAMA_3_1_8B'). */ - octo?: string /** Fireworks model to use (e.g., ''). */ fireworks?: string /** Together model to use (e.g., ''). */ @@ -58,8 +56,6 @@ export type ProcessingOptions = { groq?: string /** Ollama model to use for local inference (e.g., 'LLAMA_3_2_1B'). */ ollama?: string - /** Llama model to use for local inference (e.g., 'LLAMA_3_1_8B'). */ - llama?: string /** Gemini model to use (e.g., 'GEMINI_1_5_FLASH'). */ gemini?: string /** Array of prompt sections to include (e.g., ['titles', 'summary']). */ @@ -256,6 +252,11 @@ export type TranscriptServices = 'whisper' | 'whisperDocker' | 'whisperPython' | */ export type WhisperModelType = 'tiny' | 'tiny.en' | 'base' | 'base.en' | 'small' | 'small.en' | 'medium' | 'medium.en' | 'large-v1' | 'large-v2' | 'large-v3-turbo' | 'turbo' +/** + * Whisper-specific transcription services. + */ +export type WhisperTranscriptServices = 'whisper' | 'whisperDocker' | 'whisperPython' | 'whisperDiarization' + // LLM Types /** * Object containing different prompts, their instructions to the LLM, and expected example output. @@ -270,7 +271,7 @@ export type PromptSection = { /** * Options for Language Models (LLMs) that can be used in the application. */ -export type LLMServices = 'chatgpt' | 'claude' | 'cohere' | 'mistral' | 'octo' | 'llama' | 'ollama' | 'gemini' | 'fireworks' | 'together' | 'groq' +export type LLMServices = 'chatgpt' | 'claude' | 'cohere' | 'mistral' | 'ollama' | 'gemini' | 'fireworks' | 'together' | 'groq' /** * Options for LLM processing. @@ -332,11 +333,6 @@ export type GeminiModelType = 'GEMINI_1_5_FLASH' | 'GEMINI_1_5_PRO' */ export type MistralModelType = 'MIXTRAL_8x7b' | 'MIXTRAL_8x22b' | 'MISTRAL_LARGE' | 'MISTRAL_NEMO' -/** - * Available OctoAI models. - */ -export type OctoModelType = 'LLAMA_3_1_8B' | 'LLAMA_3_1_70B' | 'LLAMA_3_1_405B' | 'MISTRAL_7B' | 'MIXTRAL_8X_7B' | 'NOUS_HERMES_MIXTRAL_8X_7B' | 'WIZARD_2_8X_22B' - /** * Available Fireworks models. */ @@ -352,11 +348,6 @@ export type TogetherModelType = 'LLAMA_3_2_3B' | 'LLAMA_3_1_405B' | 'LLAMA_3_1_7 */ export type GroqModelType = 'LLAMA_3_1_70B_VERSATILE' | 'LLAMA_3_1_8B_INSTANT' | 'LLAMA_3_2_1B_PREVIEW' | 'LLAMA_3_2_3B_PREVIEW' | 'MIXTRAL_8X7B_32768' -/** - * Local model configurations. - */ -export type LlamaModelType = 'QWEN_2_5_1B' | 'QWEN_2_5_3B' | 'PHI_3_5' | 'LLAMA_3_2_1B' | 'GEMMA_2_2B' - /** * Local model with Ollama. */ diff --git a/src/utils/checkDependencies.ts b/src/utils/checkDependencies.ts deleted file mode 100644 index a9d01dc..0000000 --- a/src/utils/checkDependencies.ts +++ /dev/null @@ -1,54 +0,0 @@ -// src/utils/checkDependencies.ts - -/** - * @file Utility for verifying required command-line dependencies. - * Checks if necessary external tools are installed and accessible in the system PATH. - * @packageDocumentation - */ - -import { execFile } from 'node:child_process' -import { promisify } from 'node:util' - -// Promisify execFile for async/await usage -const execFilePromise = promisify(execFile) - -/** - * Verifies that required command-line dependencies are installed and accessible. - * Attempts to execute each dependency with --version flag to confirm availability. - * - * Common dependencies checked include: - * - yt-dlp: For downloading online content - * - ffmpeg: For audio processing - * - * @param {string[]} dependencies - Array of command names to verify. - * Each command should support the --version flag. - * - * @returns {Promise} Resolves when all dependencies are verified. - * - * @throws {Error} If any dependency is: - * - Not installed - * - Not found in system PATH - * - Not executable - * - Returns non-zero exit code - * - * @example - * try { - * await checkDependencies(['yt-dlp', 'ffmpeg']) - * console.log('All dependencies are available') - * } catch (error) { - * console.error('Missing dependency:', error.message) - * } - */ -export async function checkDependencies(dependencies: string[]): Promise { - for (const command of dependencies) { - try { - // Attempt to execute command with --version flag - await execFilePromise(command, ['--version']) - } catch (error) { - // Throw descriptive error if command check fails - throw new Error( - `Dependency '${command}' is not installed or not found in PATH. Please install it to proceed.` - ) - } - } -} \ No newline at end of file diff --git a/src/utils/cleanUpFiles.ts b/src/utils/cleanUpFiles.ts index 3259dc0..74c20ea 100644 --- a/src/utils/cleanUpFiles.ts +++ b/src/utils/cleanUpFiles.ts @@ -7,7 +7,7 @@ */ import { unlink } from 'node:fs/promises' -import { log, step, success } from '../models.js' +import { l, err, step, success } from '../globals.js' /** * Removes temporary files generated during content processing. @@ -39,11 +39,11 @@ import { log, step, success } from '../models.js' * // - content/my-video-2024-03-21.md * // - content/my-video-2024-03-21.lrc * } catch (error) { - * console.error('Cleanup failed:', error) + * err('Cleanup failed:', error) * } */ export async function cleanUpFiles(id: string): Promise { - log(step('\nStep 5 - Cleaning up temporary files...\n')) + l(step('\nStep 5 - Cleaning up temporary files...\n')) // Define extensions of temporary files to be cleaned up const extensions = [ @@ -53,18 +53,18 @@ export async function cleanUpFiles(id: string): Promise { '.lrc' // Lyrics/subtitles ] - log(success(` Temporary files deleted:`)) + l(success(` Temporary files deleted:`)) // Attempt to delete each file type for (const ext of extensions) { try { // Delete file and log success await unlink(`${id}${ext}`) - log(success(` - ${id}${ext}`)) + l(success(` - ${id}${ext}`)) } catch (error) { // Only log errors that aren't "file not found" (ENOENT) if (error instanceof Error && (error as Error).message !== 'ENOENT') { - console.error(`Error deleting file ${id}${ext}: ${(error as Error).message}`) + err(`Error deleting file ${id}${ext}: ${(error as Error).message}`) } // Silently continue if file doesn't exist } diff --git a/src/utils/downloadAudio.ts b/src/utils/downloadAudio.ts index 3ef8b64..20b73fc 100644 --- a/src/utils/downloadAudio.ts +++ b/src/utils/downloadAudio.ts @@ -11,8 +11,7 @@ import { exec, execFile } from 'node:child_process' import { promisify } from 'node:util' import { readFile, access } from 'node:fs/promises' import { fileTypeFromBuffer } from 'file-type' -import { checkDependencies } from './checkDependencies.js' -import { log, step, success, wait } from '../models.js' +import { l, err, step, success, wait } from '../globals.js' import type { SupportedFileType, ProcessingOptions } from '../types.js' // Promisify node:child_process functions for async/await usage @@ -86,10 +85,8 @@ export async function downloadAudio( // Handle online content (YouTube, RSS feeds, etc.) if (options.video || options.playlist || options.urls || options.rss) { - log(step('\nStep 2 - Downloading URL audio...\n')) + l(step('\nStep 2 - Downloading URL audio...\n')) try { - // Verify yt-dlp is available - await checkDependencies(['yt-dlp']) // Download and convert audio using yt-dlp const { stderr } = await execFilePromise('yt-dlp', [ '--no-warnings', // Suppress warning messages @@ -103,11 +100,11 @@ export async function downloadAudio( ]) // Log any non-fatal warnings from yt-dlp if (stderr) { - console.error(`yt-dlp warnings: ${stderr}`) + err(`yt-dlp warnings: ${stderr}`) } - log(success(` Audio downloaded successfully:\n - ${outputPath}`)) + l(success(` Audio downloaded successfully:\n - ${outputPath}`)) } catch (error) { - console.error( + err( `Error downloading audio: ${ error instanceof Error ? (error as Error).message : String(error) }` @@ -117,7 +114,7 @@ export async function downloadAudio( } // Handle local file processing else if (options.file) { - log(step('\nStep 2 - Processing file audio...\n')) + l(step('\nStep 2 - Processing file audio...\n')) // Define supported media formats const supportedFormats: Set = new Set([ // Audio formats @@ -137,14 +134,14 @@ export async function downloadAudio( fileType ? `Unsupported file type: ${fileType.ext}` : 'Unable to determine file type' ) } - log(wait(` File type detected as ${fileType.ext}, converting to WAV...\n`)) + l(wait(` File type detected as ${fileType.ext}, converting to WAV...\n`)) // Convert to standardized WAV format using ffmpeg await execPromise( `ffmpeg -i "${input}" -ar 16000 -ac 1 -c:a pcm_s16le "${outputPath}"` ) - log(success(` File converted to WAV format successfully:\n - ${outputPath}`)) + l(success(` File converted to WAV format successfully:\n - ${outputPath}`)) } catch (error) { - console.error( + err( `Error processing local file: ${ error instanceof Error ? (error as Error).message : String(error) }` diff --git a/src/utils/extractVideoMetadata.ts b/src/utils/extractVideoMetadata.ts index e57986c..2d54c1d 100644 --- a/src/utils/extractVideoMetadata.ts +++ b/src/utils/extractVideoMetadata.ts @@ -7,7 +7,7 @@ import { execFile } from 'node:child_process' import { promisify } from 'node:util' -import { checkDependencies } from './checkDependencies.js' +import { err } from '../globals.js' import type { VideoMetadata } from '../types.js' // Promisify execFile for async/await usage with yt-dlp @@ -43,17 +43,14 @@ const execFilePromise = promisify(execFile) * @example * try { * const metadata = await extractVideoMetadata('https://www.youtube.com/watch?v=...') - * console.log(metadata.title) // Video title - * console.log(metadata.publishDate) // YYYY-MM-DD + * l(metadata.title) // Video title + * l(metadata.publishDate) // YYYY-MM-DD * } catch (error) { - * console.error('Failed to extract video metadata:', error) + * err('Failed to extract video metadata:', error) * } */ export async function extractVideoMetadata(url: string): Promise { try { - // Verify yt-dlp is available - await checkDependencies(['yt-dlp']) - // Execute yt-dlp with format strings to extract specific metadata fields const { stdout } = await execFilePromise('yt-dlp', [ '--restrict-filenames', // Ensure safe filenames @@ -67,10 +64,14 @@ export async function extractVideoMetadata(url: string): Promise ]) // Split stdout into individual metadata fields - const [showLink, channel, channelURL, title, publishDate, coverImage] = stdout.trim().split('\n') + const [ + showLink, channel, channelURL, title, publishDate, coverImage + ] = stdout.trim().split('\n') // Validate that all required metadata fields are present - if (!showLink || !channel || !channelURL || !title || !publishDate || !coverImage) { + if ( + !showLink || !channel || !channelURL || !title || !publishDate || !coverImage + ) { throw new Error('Incomplete metadata received from yt-dlp.') } @@ -86,7 +87,7 @@ export async function extractVideoMetadata(url: string): Promise } } catch (error) { // Enhanced error handling with type checking - console.error( + err( `Error extracting metadata for ${url}: ${ error instanceof Error ? (error as Error).message : String(error) }` diff --git a/src/utils/generateMarkdown.ts b/src/utils/generateMarkdown.ts index 3586a21..99f7259 100644 --- a/src/utils/generateMarkdown.ts +++ b/src/utils/generateMarkdown.ts @@ -10,8 +10,7 @@ import { execFile } from 'node:child_process' import { promisify } from 'node:util' import { writeFile } from 'node:fs/promises' import { basename, extname } from 'node:path' -import { checkDependencies } from './checkDependencies.js' -import { log, dim, step, success } from '../models.js' +import { l, dim, step, success } from '../globals.js' import type { MarkdownData, ProcessingOptions, RSSItem } from '../types.js' // Promisify the execFile function for use with async/await @@ -97,9 +96,6 @@ export async function generateMarkdown( case !!options.video: case !!options.playlist: case !!options.urls: - // Verify yt-dlp is available for video processing - await checkDependencies(['yt-dlp']) - // Extract video metadata using yt-dlp const { stdout } = await execFilePromise('yt-dlp', [ '--restrict-filenames', @@ -161,7 +157,9 @@ export async function generateMarkdown( case !!options.rss: // Process RSS feed item const item = input as RSSItem - const { publishDate, title: rssTitle, coverImage, showLink, channel: rssChannel, channelURL } = item + const { + publishDate, title: rssTitle, coverImage, showLink, channel: rssChannel, channelURL + } = item // Generate filename using date and sanitized title filename = `${publishDate}-${sanitizeTitle(rssTitle)}` @@ -192,9 +190,9 @@ export async function generateMarkdown( await writeFile(`${finalPath}.md`, frontMatterContent) // Log the generated content and success message - log(dim(frontMatterContent)) - log(step('\nStep 1 - Generating markdown...\n')) - log(success(` Front matter successfully created and saved:\n - ${finalPath}.md`)) + l(dim(frontMatterContent)) + l(step('\nStep 1 - Generating markdown...\n')) + l(success(` Front matter successfully created and saved:\n - ${finalPath}.md`)) // Return the generated markdown data for further processing return { frontMatter: frontMatterContent, finalPath, filename } diff --git a/src/utils/runLLM.ts b/src/utils/runLLM.ts index 13d3697..7013d30 100644 --- a/src/utils/runLLM.ts +++ b/src/utils/runLLM.ts @@ -7,19 +7,17 @@ */ import { readFile, writeFile, unlink } from 'node:fs/promises' -import { callLlama } from '../llms/llama.js' import { callOllama } from '../llms/ollama.js' import { callChatGPT } from '../llms/chatgpt.js' import { callClaude } from '../llms/claude.js' import { callGemini } from '../llms/gemini.js' import { callCohere } from '../llms/cohere.js' import { callMistral } from '../llms/mistral.js' -import { callOcto } from '../llms/octo.js' import { callFireworks } from '../llms/fireworks.js' import { callTogether } from '../llms/together.js' import { callGroq } from '../llms/groq.js' import { generatePrompt } from '../llms/prompt.js' -import { log, step, success, wait } from '../models.js' +import { l, err, step, success, wait } from '../globals.js' import type { LLMServices, ProcessingOptions, LLMFunction, LLMFunctions } from '../types.js' /** @@ -47,14 +45,12 @@ import type { LLMServices, ProcessingOptions, LLMFunction, LLMFunctions } from ' * @param {string} frontMatter - YAML front matter content to include in the output * * @param {LLMServices} [llmServices] - The LLM service to use: - * - llama: Node Llama for local inference * - ollama: Ollama for local inference * - chatgpt: OpenAI's ChatGPT * - claude: Anthropic's Claude * - gemini: Google's Gemini * - cohere: Cohere * - mistral: Mistral AI - * - octo: OctoAI * - fireworks: Fireworks AI * - together: Together AI * - groq: Groq @@ -68,9 +64,9 @@ import type { LLMServices, ProcessingOptions, LLMFunction, LLMFunctions } from ' * - File operations fail * * @example - * // Process with ChatGPT + * // Process with Ollama * await runLLM( - * { prompt: ['summary', 'highlights'], chatgpt: 'GPT_4' }, + * { prompt: ['summary', 'highlights'], ollama: 'LLAMA_3_2_1B' }, * 'content/my-video', * '---\ntitle: My Video\n---', * 'chatgpt' @@ -90,20 +86,18 @@ export async function runLLM( frontMatter: string, llmServices?: LLMServices ): Promise { - log(step(`\nStep 4 - Running LLM processing on transcript...\n`)) + l(step(`\nStep 4 - Running LLM processing on transcript...\n`)) // Map of available LLM service handlers const LLM_FUNCTIONS: LLMFunctions = { - llama: callLlama, // Local inference with Node Llama ollama: callOllama, // Local inference with Ollama chatgpt: callChatGPT, // OpenAI's ChatGPT claude: callClaude, // Anthropic's Claude gemini: callGemini, // Google's Gemini cohere: callCohere, // Cohere mistral: callMistral, // Mistral AI - octo: callOcto, // OctoAI - fireworks: callFireworks, // Fireworks AI - together: callTogether, // Together AI + fireworks: callFireworks, // Fireworks AI + together: callTogether, // Together AI groq: callGroq, // Groq } @@ -117,7 +111,7 @@ export async function runLLM( const promptAndTranscript = `${prompt}${transcript}` if (llmServices) { - log(wait(` Processing with ${llmServices} Language Model...\n`)) + l(wait(` Processing with ${llmServices} Language Model...\n`)) // Get the appropriate LLM handler function const llmFunction: LLMFunction = LLM_FUNCTIONS[llmServices] @@ -128,7 +122,7 @@ export async function runLLM( // Process content with selected LLM const tempPath = `${finalPath}-${llmServices}-temp.md` await llmFunction(promptAndTranscript, tempPath, options[llmServices]) - log(wait(`\n Transcript saved to temporary file:\n - ${tempPath}`)) + l(success(`\n Transcript saved to temporary file:\n - ${tempPath}`)) // Combine results with front matter and original transcript const showNotes = await readFile(tempPath, 'utf8') @@ -139,15 +133,15 @@ export async function runLLM( // Clean up temporary file await unlink(tempPath) - log(success(`\n Generated show notes saved to markdown file:\n - ${finalPath}-${llmServices}-shownotes.md`)) + l(success(`\n Generated show notes saved to markdown file:\n - ${finalPath}-${llmServices}-shownotes.md`)) } else { // Handle case when no LLM is selected - log(wait(' No LLM selected, skipping processing...')) + l(wait(' No LLM selected, skipping processing...')) await writeFile(`${finalPath}-prompt.md`, `${frontMatter}\n${promptAndTranscript}`) - log(success(`\n Prompt and transcript saved to markdown file:\n - ${finalPath}-prompt.md`)) + l(success(`\n Prompt and transcript saved to markdown file:\n - ${finalPath}-prompt.md`)) } } catch (error) { - console.error(`Error running Language Model: ${(error as Error).message}`) + err(`Error running Language Model: ${(error as Error).message}`) throw error } } \ No newline at end of file diff --git a/src/utils/runTranscription.ts b/src/utils/runTranscription.ts index 5001073..51e5837 100644 --- a/src/utils/runTranscription.ts +++ b/src/utils/runTranscription.ts @@ -8,13 +8,10 @@ */ import { callWhisper } from '../transcription/whisper.js' -import { callWhisperPython } from '../transcription/whisperPython.js' -import { callWhisperDocker } from '../transcription/whisperDocker.js' -import { callWhisperDiarization } from '../transcription/whisperDiarization.js' import { callDeepgram } from '../transcription/deepgram.js' import { callAssembly } from '../transcription/assembly.js' -import { log, step } from '../models.js' -import { TranscriptServices, ProcessingOptions } from '../types.js' +import { l, step } from '../globals.js' +import { TranscriptServices, ProcessingOptions, WhisperTranscriptServices } from '../types.js' /** * Orchestrates the transcription process using the specified service. @@ -87,7 +84,7 @@ export async function runTranscription( frontMatter: string, transcriptServices?: TranscriptServices ): Promise { - log(step(`\nStep 3 - Running transcription on audio file using ${transcriptServices}...`)) + l(step(`\nStep 3 - Running transcription on audio file using ${transcriptServices}...`)) // Route to appropriate transcription service switch (transcriptServices) { @@ -102,23 +99,11 @@ export async function runTranscription( break case 'whisper': - // Local Whisper.cpp implementation - await callWhisper(options, finalPath) - break - case 'whisperDocker': - // Containerized Whisper.cpp - await callWhisperDocker(options, finalPath) - break - case 'whisperPython': - // Original Python implementation - await callWhisperPython(options, finalPath) - break - case 'whisperDiarization': - // Whisper with speaker detection - await callWhisperDiarization(options, finalPath) + // Use the unified callWhisper function for all Whisper options + await callWhisper(options, finalPath, transcriptServices as WhisperTranscriptServices) break default: diff --git a/src/utils/validateOption.ts b/src/utils/validateOption.ts new file mode 100644 index 0000000..e2bc882 --- /dev/null +++ b/src/utils/validateOption.ts @@ -0,0 +1,30 @@ +// src/utils/validateOption.ts + +import { exit } from 'node:process' +import { err } from '../globals.js' +import type { ProcessingOptions } from '../types.js' + +/** + * Helper function to validate that only one option from a list is provided. + * Prevents users from specifying multiple conflicting options simultaneously. + * + * @param optionKeys - The list of option keys to check. + * @param options - The options object. + * @param errorMessage - The prefix of the error message. + * @returns The selected option or undefined. + */ +export function validateOption( + optionKeys: string[], + options: ProcessingOptions, + errorMessage: string + ): string | undefined { + // Filter out which options from the provided list are actually set + const selectedOptions = optionKeys.filter((opt) => options[opt as keyof ProcessingOptions]) + + // If more than one option is selected, throw an error + if (selectedOptions.length > 1) { + err(`Error: Multiple ${errorMessage} provided (${selectedOptions.join(', ')}). Please specify only one.`) + exit(1) + } + return selectedOptions[0] as string | undefined + } \ No newline at end of file diff --git a/test/bench.test.ts b/test/bench.test.ts index f78e8c8..1352d64 100644 --- a/test/bench.test.ts +++ b/test/bench.test.ts @@ -1,4 +1,4 @@ -// test/local.test.ts +// test/bench.test.ts import test from 'node:test' import { strictEqual } from 'node:assert/strict' @@ -14,107 +14,145 @@ type Command = { const commands: Command[] = [ { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisper tiny', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', + cmd: 'npm run as -- --file "content/audio.mp3" --whisper tiny', + expectedFile: 'audio-prompt.md', newName: '01_TINY_WHISPERCPP.md' }, - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisper base', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '02_BASE_WHISPERCPP.md' - }, - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisper medium', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '03_MEDIUM_WHISPERCPP.md' - }, - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisper large-v1', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '04_LARGE_V1_WHISPERCPP.md' - }, - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisper large-v2', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '05_LARGE_V2_WHISPERCPP.md' - }, - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisper large-v3-turbo', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '06_LARGE_V3_WHISPERCPP.md' - }, - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperDiarization tiny', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '07_TINY_DIARIZATION.md' - }, - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperDiarization base', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '08_BASE_DIARIZATION.md' - }, - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperDiarization medium', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '09_MEDIUM_DIARIZATION.md' - }, - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperDiarization large-v1', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '10_LARGE_V1_DIARIZATION.md' - }, - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperDiarization large-v2', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '11_LARGE_V2_DIARIZATION.md' - }, - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperPython tiny', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '12_TINY_PYTHON.md' - }, - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperPython base', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '13_BASE_PYTHON.md' - }, - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperPython medium', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '14_MEDIUM_PYTHON.md' - }, - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperPython large-v1', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '15_LARGE_V1_PYTHON.md' - }, - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperPython large-v2', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '16_LARGE_V2_PYTHON.md' - }, - { - // Process a single YouTube video using Autoshow's default settings. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperPython large-v3-turbo', - expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '17_LARGE_V3_TURBO_PYTHON.md' - } + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisper base', + // expectedFile: 'audio-prompt.md', + // newName: '02_BASE_WHISPERCPP.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisper small', + // expectedFile: 'audio-prompt.md', + // newName: '03_SMALL_WHISPERCPP.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisper medium', + // expectedFile: 'audio-prompt.md', + // newName: '04_MEDIUM_WHISPERCPP.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisper large-v1', + // expectedFile: 'audio-prompt.md', + // newName: '05_LARGE_V1_WHISPERCPP.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisper large-v2', + // expectedFile: 'audio-prompt.md', + // newName: '06_LARGE_V2_WHISPERCPP.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisper large-v3-turbo', + // expectedFile: 'audio-prompt.md', + // newName: '07_LARGE_V3_TURBO_WHISPERCPP.md' + // }, + { + cmd: 'npm run as -- --file "content/audio.mp3" --whisperDocker tiny', + expectedFile: 'audio-prompt.md', + newName: '08_TINY_WHISPERCPP_DOCKER.md' + }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperDocker base', + // expectedFile: 'audio-prompt.md', + // newName: '09_BASE_WHISPERCPP_DOCKER.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperDocker small', + // expectedFile: 'audio-prompt.md', + // newName: '10_SMALL_WHISPERCPP_DOCKER.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperDocker medium', + // expectedFile: 'audio-prompt.md', + // newName: '11_MEDIUM_WHISPERCPP_DOCKER.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperDocker large-v1', + // expectedFile: 'audio-prompt.md', + // newName: '12_LARGE_V1_WHISPERCPP_DOCKER.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperDocker large-v2', + // expectedFile: 'audio-prompt.md', + // newName: '13_LARGE_V2_WHISPERCPP_DOCKER.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperDocker large-v3-turbo', + // expectedFile: 'audio-prompt.md', + // newName: '14_LARGE_V3_TURBO_WHISPERCPP_DOCKER.md' + // }, + { + cmd: 'npm run as -- --file "content/audio.mp3" --whisperDiarization tiny', + expectedFile: 'audio-prompt.md', + newName: '15_TINY_DIARIZATION.md' + }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperDiarization base', + // expectedFile: 'audio-prompt.md', + // newName: '16_BASE_DIARIZATION.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperDiarization small', + // expectedFile: 'audio-prompt.md', + // newName: '17_SMALL_DIARIZATION.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperDiarization medium', + // expectedFile: 'audio-prompt.md', + // newName: '18_MEDIUM_DIARIZATION.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperDiarization large-v1', + // expectedFile: 'audio-prompt.md', + // newName: '19_LARGE_V1_DIARIZATION.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperDiarization large-v2', + // expectedFile: 'audio-prompt.md', + // newName: '20_LARGE_V2_DIARIZATION.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperDiarization large-v3-turbo', + // expectedFile: 'audio-prompt.md', + // newName: '21_LARGE_V3_TURBO_DIARIZATION.md' + // }, + { + cmd: 'npm run as -- --file "content/audio.mp3" --whisperPython tiny', + expectedFile: 'audio-prompt.md', + newName: '22_TINY_PYTHON.md' + }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperPython base', + // expectedFile: 'audio-prompt.md', + // newName: '23_BASE_PYTHON.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperPython small', + // expectedFile: 'audio-prompt.md', + // newName: '24_SMALL_PYTHON.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperPython medium', + // expectedFile: 'audio-prompt.md', + // newName: '25_MEDIUM_PYTHON.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperPython large-v1', + // expectedFile: 'audio-prompt.md', + // newName: '26_LARGE_V1_PYTHON.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperPython large-v2', + // expectedFile: 'audio-prompt.md', + // newName: '27_LARGE_V2_PYTHON.md' + // }, + // { + // cmd: 'npm run as -- --file "content/audio.mp3" --whisperPython large-v3-turbo', + // expectedFile: 'audio-prompt.md', + // newName: '28_LARGE_V3_TURBO_PYTHON.md' + // } ] test('Autoshow Command Tests', async (t) => { diff --git a/test/integrations.test.ts b/test/integrations.test.ts index 0d45ad9..1a38eae 100644 --- a/test/integrations.test.ts +++ b/test/integrations.test.ts @@ -76,46 +76,52 @@ const commands = [ newName: '11---ep0-fsjam-podcast-mistral-shownotes.md' }, { - // Process a video using OctoAI for LLM operations - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --octo', - expectedFile: '2024-09-24-ep0-fsjam-podcast-octo-shownotes.md', - newName: '12---ep0-fsjam-podcast-octo-shownotes.md' + // Process a video using Fireworks for LLM operations + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --fireworks', + expectedFile: '2024-09-24-ep0-fsjam-podcast-fireworks-shownotes.md', + newName: '12---ep0-fsjam-podcast-fireworks-shownotes.md' }, { - // Process video with Octo using LLAMA_3_1_8B model. - cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --octo LLAMA_3_1_8B', - expectedFile: '2024-09-24-ep0-fsjam-podcast-octo-shownotes.md', - newName: '13---ep0-fsjam-podcast-octo-shownotes.md' + // Process a video using Together for LLM operations + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --together', + expectedFile: '2024-09-24-ep0-fsjam-podcast-together-shownotes.md', + newName: '13---ep0-fsjam-podcast-together-shownotes.md' + }, + { + // Process a video using BLANK for LLM operations + cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --groq', + expectedFile: '2024-09-24-ep0-fsjam-podcast-groq-shownotes.md', + newName: '14---ep0-fsjam-podcast-groq-shownotes.md' }, { // Process a video using Deepgram for transcription cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --deepgram', expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '14---ep0-fsjam-podcast-prompt.md' + newName: '15---ep0-fsjam-podcast-prompt.md' }, { // Process video using Deepgram and Llama. cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --deepgram --ollama', expectedFile: '2024-09-24-ep0-fsjam-podcast-ollama-shownotes.md', - newName: '15---ep0-fsjam-podcast-ollama-shownotes.md' + newName: '16---ep0-fsjam-podcast-ollama-shownotes.md' }, { // Process a video using AssemblyAI for transcription cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --assembly', expectedFile: '2024-09-24-ep0-fsjam-podcast-prompt.md', - newName: '16---ep0-fsjam-podcast-prompt.md' + newName: '17---ep0-fsjam-podcast-prompt.md' }, { // Process video using AssemblyAI and Llama. cmd: 'npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --assembly --ollama', expectedFile: '2024-09-24-ep0-fsjam-podcast-ollama-shownotes.md', - newName: '17---ep0-fsjam-podcast-ollama-shownotes.md' + newName: '18---ep0-fsjam-podcast-ollama-shownotes.md' }, { // Process an audio file using AssemblyAI with speaker labels cmd: 'npm run as -- --video "https://ajc.pics/audio/fsjam-short.mp3" --assembly --speakerLabels', expectedFile: '2024-05-08-fsjam-short-prompt.md', - newName: '18---fsjam-short-prompt.md' + newName: '19---fsjam-short-prompt.md' } ] diff --git a/test/local.test.ts b/test/local.test.ts index eb32b95..ecab151 100644 --- a/test/local.test.ts +++ b/test/local.test.ts @@ -36,14 +36,14 @@ const commands = [ newName: 'FILE_04.md' }, { - // Process local audio file with title prompts, Whisper 'tiny' model, and Llama. - cmd: 'npm run as -- --file "content/audio.mp3" --prompt titles --whisper tiny --llama', - expectedFile: 'audio-llama-shownotes.md', + // Process local audio file with title prompts, Whisper 'tiny' model, and Ollama. + cmd: 'npm run as -- --file "content/audio.mp3" --prompt titles --whisper tiny --ollama', + expectedFile: 'audio-ollama-shownotes.md', newName: 'FILE_05.md' }, { - // Process a local audio file with Ollama using LLAMA_3_2_3B model. - cmd: 'npm run as -- --file "content/audio.mp3" --ollama LLAMA_3_2_3B', + // Process a local audio file with Ollama using LLAMA_3_2_1B model. + cmd: 'npm run as -- --file "content/audio.mp3" --ollama LLAMA_3_2_1B', expectedFile: 'audio-ollama-shownotes.md', newName: 'FILE_06.md' },