Merge pull request #33 from ajcwebdev/new

TypeScript

.env.example

@@ -1,8 +1,6 @@
-LLAMA_MODEL="gemma-2-2b-it-IQ4_XS.gguf"
-HUGGING_FACE_URL="https://huggingface.co/lmstudio-community/gemma-2-2b-it-GGUF"
-
 OPENAI_API_KEY=""
 ANTHROPIC_API_KEY=""
+GEMINI_API_KEY=""
 COHERE_API_KEY=""
 MISTRAL_API_KEY=""
 OCTOAI_API_KEY=""

.gitignore

@@ -8,4 +8,8 @@ src/llms/models
 .idea
 build
 deno.lock
-out
+out
+types
+dist
+NEW.md
+TODO.md

docker-compose.yml

@@ -15,6 +15,8 @@ services:
     environment:
       - OLLAMA_HOST=ollama
       - OLLAMA_PORT=11434
+    networks:
+      - autoshownet
   whisper:
     build:
       context: ./whisper.cpp
@@ -25,8 +27,11 @@ services:
     command: tail -f /dev/null
     tty: true
     stdin_open: true
+    networks:
+      - autoshownet
   ollama:
     image: ollama/ollama
+    command: ["ollama", "serve", "--address", "0.0.0.0"]  # Listen on all interfaces
     ports:
       - "11434:11434"
     volumes:
@@ -35,4 +40,9 @@ services:
       test: ["CMD", "curl", "-f", "http://localhost:11434/healthz"]
       interval: 10s
       timeout: 5s
-      retries: 5
+      retries: 5
+    networks:
+      - autoshownet
+networks:
+  autoshownet:
+    driver: bridge

docs/examples.md

@@ -81,7 +81,15 @@ npm run as -- --file "content/audio.mp3"
 Process RSS feed from newest to oldest (default behavior):
 
 ```bash
-npm run as -- --rss "https://feeds.transistor.fm/fsjam-podcast/"
+npm run as -- --rss "https://ajcwebdev.substack.com/feed"
+```
+
+Process a certain number of the most recent items, for example the last three episodes released on the feed:
+
+```bash
+npm run as -- \
+  --rss "https://feeds.transistor.fm/fsjam-podcast/" \
+  --last 3
 ```
 
 Process RSS feed from oldest to newest:
@@ -243,6 +251,32 @@ npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --octo WIZAR
 npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --llama
 ```
 
+Select Llama model:
+
+```bash
+npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --llama GEMMA_2_2B
+npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --llama LLAMA_3_2_1B
+npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --llama PHI_3_5
+npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --llama QWEN_2_5_3B
+```
+
+### Ollama
+
+```bash
+npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --ollama
+```
+
+Select Ollama model:
+
+```bash
+npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --ollama LLAMA_3_2_1B
+npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --ollama LLAMA_3_2_3B
+npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --ollama GEMMA_2_2B
+npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --ollama PHI_3_5
+npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --ollama QWEN_2_5_1B
+npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --ollama QWEN_2_5_3B
+```
+
 ## Transcription Options
 
 ### Whisper.cpp
@@ -263,7 +297,7 @@ npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisper sm
 npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisper medium
 
 # large-v2 model
-npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisper large
+npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisper large-v2
 ```
 
 Run `whisper.cpp` in a Docker container with `--whisperDocker`:
@@ -357,7 +391,7 @@ npm run as -- --video "https://www.youtube.com/watch?v=MORMZXEaONk" --prompt tit
 This will run both `whisper.cpp` and the AutoShow Commander CLI in their own Docker containers.
 
 ```bash
-docker-compose run autoshow --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperDocker base
+docker compose run autoshow --video "https://www.youtube.com/watch?v=MORMZXEaONk" --whisperDocker base
 ```
 
 Currently working on the `llama.cpp` Docker integration so the entire project can be encapsulated in one local Docker Compose file.
@@ -426,6 +460,7 @@ export MD="LLM.md" && export COMMANDS="src/commands" && export UTILS="src/utils"
   echo ''$CLOSE'### Call Cohere'$OPEN'' >> $MD && cat $LLMS/cohere.js >> $MD && \
   echo ''$CLOSE'### Call Gemini'$OPEN'' >> $MD && cat $LLMS/gemini.js >> $MD && \
   echo ''$CLOSE'### Call Llama.cpp'$OPEN'' >> $MD && cat $LLMS/llama.js >> $MD && \
+  echo ''$CLOSE'### Call Ollama'$OPEN'' >> $MD && cat $LLMS/ollama.js >> $MD && \
   echo ''$CLOSE'### Call Mistral'$OPEN'' >> $MD && cat $LLMS/mistral.js >> $MD && \
   echo ''$CLOSE'### Call Octo'$OPEN'' >> $MD && cat $LLMS/octo.js >> $MD && \
   echo ''$CLOSE'## Docker Files\n\n```Dockerfile' >> $MD && cat .github/whisper.Dockerfile >> $MD && \

docs/server.md

@@ -269,14 +269,14 @@ curl --json '{
 ```bash
 curl --json '{
   "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk",
-  "transcriptService": "deepgram"
+  "transcriptServices": "deepgram"
 }' http://localhost:3000/video
 ```
 
 ```bash
 curl --json '{
   "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk",
-  "transcriptService": "deepgram",
+  "transcriptServices": "deepgram",
   "llm": "llama"
 }' http://localhost:3000/video
 ```
@@ -286,30 +286,30 @@ curl --json '{
 ```bash
 curl --json '{
   "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk",
-  "transcriptService": "assembly"
+  "transcriptServices": "assembly"
 }' http://localhost:3000/video
 ```
 
 ```bash
 curl --json '{
   "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk",
-  "transcriptService": "assembly",
+  "transcriptServices": "assembly",
   "llm": "llama"
 }' http://localhost:3000/video
 ```
 
 ```bash
 curl --json '{
   "youtubeUrl": "https://ajc.pics/audio/fsjam-short.mp3",
-  "transcriptService": "assembly",
+  "transcriptServices": "assembly",
   "speakerLabels": true
 }' http://localhost:3000/video
 ```
 
 ```bash
 curl --json '{
   "youtubeUrl": "https://ajc.pics/audio/fsjam-short.mp3",
-  "transcriptService": "assembly",
+  "transcriptServices": "assembly",
   "speakerLabels": true,
   "llm": "llama"
 }' http://localhost:3000/video

package.json

@@ -12,25 +12,30 @@
     "Deepgram",
     "AssemblyAI"
   ],
-  "main": "autoshow.js",
+  "main": "dist/autoshow.js",
   "type": "module",
   "bin": {
-    "autoshow": "./autoshow.js"
+    "autoshow": "./dist/autoshow.js"
   },
   "scripts": {
     "setup": "bash ./setup.sh",
-    "autoshow": "node --env-file=.env --no-warnings src/autoshow.js",
-    "as": "node --env-file=.env --no-warnings src/autoshow.js",
-    "bun-as": "bun --env-file=.env --no-warnings src/autoshow.js",
-    "deno-as": "deno run --allow-sys --allow-read --allow-run --allow-write --allow-env src/autoshow.js",
-    "v": "node --env-file=.env --no-warnings src/autoshow.js --whisper large --video",
-    "u": "node --env-file=.env --no-warnings src/autoshow.js --whisper large --urls",
-    "p": "node --env-file=.env --no-warnings src/autoshow.js --whisper large --playlist",
-    "f": "node --env-file=.env --no-warnings src/autoshow.js --whisper large --file",
-    "serve": "node --env-file=.env --no-warnings --watch packages/server/index.js",
-    "fetch": "node --env-file=.env --no-warnings packages/server/fetch.js",
-    "test-local": "node --test test/local.test.js",
-    "test-all": "node --test test/all.test.js"
+    "build": "npx tsc",
+    "autoshow": "npm run build && node --env-file=.env --no-warnings dist/autoshow.js",
+    "as": "npm run build && node --env-file=.env --no-warnings dist/autoshow.js",
+    "bun-as": "npm run build && bun --env-file=.env --no-warnings dist/autoshow.js",
+    "deno-as": "npm run build && deno run --allow-sys --allow-read --allow-run --allow-write --allow-env dist/autoshow.js",
+    "v": "npm run build && node --env-file=.env --no-warnings dist/autoshow.js --whisper large-v2 --video",
+    "u": "npm run build && node --env-file=.env --no-warnings dist/autoshow.js --whisper large-v2 --urls",
+    "p": "npm run build && node --env-file=.env --no-warnings dist/autoshow.js --whisper large-v2 --playlist",
+    "f": "npm run build && node --env-file=.env --no-warnings dist/autoshow.js --whisper large-v2 --file",
+    "r": "npm run build && node --env-file=.env --no-warnings dist/autoshow.js --whisper large-v2 --rss",
+    "last3": "npm run build && node --env-file=.env --no-warnings dist/autoshow.js --whisper large-v2 --last 3 --rss",
+    "serve": "npm run build && node --env-file=.env --no-warnings --watch packages/server/index.js",
+    "fetch-local": "npm run build && node --env-file=.env --no-warnings packages/server/tests/fetch-local.js",
+    "fetch-all": "npm run build && node --env-file=.env --no-warnings packages/server/tests/fetch-all.js",
+    "t": "npm run build && node --test test/local.test.js",
+    "test-local": "npm run build && node --test test/local.test.js",
+    "test-all": "npm run build && node --test test/all.test.js"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.26.0",
@@ -39,16 +44,22 @@
     "@google/generative-ai": "^0.17.1",
     "@mistralai/mistralai": "^1.0.2",
     "@octoai/sdk": "^1.5.1",
+    "@types/inquirer": "^9.0.7",
     "assemblyai": "^4.6.1",
+    "chalk": "^5.3.0",
     "cohere-ai": "^7.12.0",
     "commander": "^12.1.0",
     "fast-xml-parser": "^4.4.1",
     "fastify": "^5.0.0",
     "ffmpeg-static": "^5.2.0",
     "file-type": "^19.4.1",
     "inquirer": "^10.2.2",
-    "node-llama-cpp": "^3.0.0-beta.44",
+    "node-llama-cpp": "^3.1.0",
     "ollama": "^0.5.9",
-    "openai": "^4.55.7"
+    "openai": "^4.55.7",
+    "typescript": "^5.6.2"
+  },
+  "devDependencies": {
+    "typescript": "^5.6.2"
   }
 }

packages/server/README.md

@@ -207,35 +207,35 @@ const TEST_REQ_32 = {
 
 const TEST_REQ_33 = {
   "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk",
-  "transcriptService": "deepgram"
+  "transcriptServices": "deepgram"
 }
 
 const TEST_REQ_34 = {
   "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk",
-  "transcriptService": "deepgram",
+  "transcriptServices": "deepgram",
   "llm": "llama"
 }
 
 const TEST_REQ_35 = {
   "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk",
-  "transcriptService": "assembly"
+  "transcriptServices": "assembly"
 }
 
 const TEST_REQ_36 = {
   "youtubeUrl": "https://www.youtube.com/watch?v=MORMZXEaONk",
-  "transcriptService": "assembly",
+  "transcriptServices": "assembly",
   "llm": "llama"
 }
 
 const TEST_REQ_37 = {
   "youtubeUrl": "https://ajc.pics/audio/fsjam-short.mp3",
-  "transcriptService": "assembly",
+  "transcriptServices": "assembly",
   "speakerLabels": true
 }
 
 const TEST_REQ_38 = {
   "youtubeUrl": "https://ajc.pics/audio/fsjam-short.mp3",
-  "transcriptService": "assembly",
+  "transcriptServices": "assembly",
   "speakerLabels": true,
   "llm": "llama"
 }

packages/server/index.js

@@ -26,7 +26,7 @@ async function start() {
   // Log each incoming request
   fastify.addHook('onRequest', async (request, reply) => {
     console.log(
-      `[${new Date().toISOString()}] Received ${request.method} request for ${request.url}`
+      `\n[${new Date().toISOString()}] Received ${request.method} request for ${request.url}\n`
     )
   })
 
@@ -39,7 +39,7 @@ async function start() {
 
   try {
     await fastify.listen({ port })
-    console.log(`Server running at http://localhost:${port}`)
+    console.log(`\nServer running at http://localhost:${port}\n`)
   } catch (err) {
     fastify.log.error(err)
     process.exit(1)

packages/server/routes/file.js

@@ -5,12 +5,12 @@ import { reqToOpts } from '../utils/reqToOpts.js'
 
 // Handler for /file route
 const handleFileRequest = async (request, reply) => {
-  console.log('Entered handleFileRequest')
+  console.log('\nEntered handleFileRequest')
 
   try {
     // Access parsed request body
     const requestData = request.body
-    console.log('Parsed request body:', requestData)
+    console.log('\nParsed request body:', requestData)
 
     // Extract file path
     const { filePath } = requestData
@@ -22,12 +22,12 @@ const handleFileRequest = async (request, reply) => {
     }
 
     // Map request data to processing options
-    const { options, llmOpt, transcriptOpt } = reqToOpts(requestData)
-    console.log('Calling processFile with params:', { filePath, llmOpt, transcriptOpt, options })
+    const { options, llmServices, transcriptServices } = reqToOpts(requestData)
+    console.log('\nCalling processFile with params:', { filePath, llmServices, transcriptServices, options })
 
-    await processFile(filePath, llmOpt, transcriptOpt, options)
+    await processFile(filePath, llmServices, transcriptServices, options)
 
-    console.log('processFile completed successfully')
+    console.log('\nprocessFile completed successfully')
     reply.send({ message: 'File processed successfully.' })
   } catch (error) {
     console.error('Error processing file:', error)