Skip to content

Commit

Permalink
refactor tests, logging, and global exports
Browse files Browse the repository at this point in the history
  • Loading branch information
@ajcwebdev
ajcwebdev committed Oct 7, 2024
1 parent b5a1e68 commit da9d493
Show file tree
Hide file tree
Showing 19 changed files with 249 additions and 302 deletions.
4 changes: 3 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,6 @@ build
deno.lock
out
types
dist
dist
NEW.md
TODO.md
3 changes: 2 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
"serve": "npm run build && node --env-file=.env --no-warnings --watch packages/server/index.js",
"fetch-local": "npm run build && node --env-file=.env --no-warnings packages/server/tests/fetch-local.js",
"fetch-all": "npm run build && node --env-file=.env --no-warnings packages/server/tests/fetch-all.js",
"t": "npm run build && node --test test/local.test.js",
"test-local": "npm run build && node --test test/local.test.js",
"test-all": "npm run build && node --test test/all.test.js"
},
Expand All @@ -53,7 +54,7 @@
"ffmpeg-static": "^5.2.0",
"file-type": "^19.4.1",
"inquirer": "^10.2.2",
"node-llama-cpp": "^3.0.0-beta.44",
"node-llama-cpp": "^3.1.0",
"ollama": "^0.5.9",
"openai": "^4.55.7",
"typescript": "^5.6.2"
Expand Down
78 changes: 43 additions & 35 deletions src/autoshow.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import { processURLs } from './commands/processURLs.js'
import { processFile } from './commands/processFile.js'
import { processRSS } from './commands/processRSS.js'
import { argv, exit } from 'node:process'
import { log, opts } from './models.js'
import { log, opts, final, ACTION_OPTIONS, LLM_OPTIONS, TRANSCRIPT_OPTIONS } from './models.js'
import type { ProcessingOptions, HandlerFunction, LLMServices, TranscriptServices } from './types.js'

// Initialize the command-line interface
Expand Down Expand Up @@ -72,6 +72,22 @@ Report Issues: https://github.com/ajcwebdev/autoshow/issues
`
)

/**
* Helper function to validate that only one option from a list is provided.
* @param {string[]} optionKeys - The list of option keys to check.
* @param {ProcessingOptions} options - The options object.
* @param {string} errorMessage - The prefix of the error message.
* @returns {string | undefined} - The selected option or undefined.
*/
function getSingleOption(optionKeys: string[], options: ProcessingOptions, errorMessage: string): string | undefined {
const selectedOptions = optionKeys.filter((opt) => options[opt as keyof ProcessingOptions])
if (selectedOptions.length > 1) {
console.error(`Error: Multiple ${errorMessage} provided (${selectedOptions.join(', ')}). Please specify only one.`)
exit(1)
}
return selectedOptions[0] as string | undefined
}

/**
* Main action for the program.
* @param {ProcessingOptions} options - The command-line options provided by the user.
Expand All @@ -90,12 +106,8 @@ program.action(async (options: ProcessingOptions) => {
rss: processRSS,
}

const ACTION_OPTIONS = ['video', 'playlist', 'urls', 'file', 'rss']
const LLM_OPTIONS = ['chatgpt', 'claude', 'cohere', 'mistral', 'octo', 'llama', 'ollama', 'gemini']
const TRANSCRIPT_OPTIONS = ['whisper', 'whisperDocker', 'deepgram', 'assembly']

const { video, playlist, urls, file, rss, interactive } = options
const noActionProvided = [video, playlist, urls, file, rss].every((opt) => !opt)
const { interactive } = options
const noActionProvided = ACTION_OPTIONS.every((opt) => !options[opt as keyof ProcessingOptions])

if (interactive || noActionProvided) {
options = await handleInteractivePrompt(options)
Expand All @@ -106,25 +118,16 @@ program.action(async (options: ProcessingOptions) => {
options.item = [options.item]
}

const actionsProvided = ACTION_OPTIONS.filter((opt) => options[opt as keyof ProcessingOptions])
if (actionsProvided.length > 1) {
console.error(`Error: Multiple input options provided (${actionsProvided.join(', ')}). Please specify only one input option.`)
exit(1)
}

const selectedLLMs = LLM_OPTIONS.filter((opt) => options[opt as keyof ProcessingOptions])
if (selectedLLMs.length > 1) {
console.error(`Error: Multiple LLM options provided (${selectedLLMs.join(', ')}). Please specify only one LLM option.`)
exit(1)
}
const llmServices = selectedLLMs[0] as LLMServices | undefined
// Validate and retrieve single action option
const action = getSingleOption(ACTION_OPTIONS, options, 'input option')

// Validate and retrieve single LLM option
const llmKey = getSingleOption(LLM_OPTIONS, options, 'LLM option')
const llmServices = llmKey as LLMServices | undefined

const selectedTranscripts = TRANSCRIPT_OPTIONS.filter((opt) => options[opt as keyof ProcessingOptions])
if (selectedTranscripts.length > 1) {
console.error(`Error: Multiple transcription options provided (${selectedTranscripts.join(', ')}). Please specify only one transcription option.`)
exit(1)
}
const transcriptServices = selectedTranscripts[0] as TranscriptServices | undefined
// Validate and retrieve single transcription option
const transcriptKey = getSingleOption(TRANSCRIPT_OPTIONS, options, 'transcription option')
const transcriptServices: TranscriptServices | undefined = transcriptKey as TranscriptServices | undefined

// Set default transcription service if not provided
const finalTranscriptServices: TranscriptServices = transcriptServices || 'whisper'
Expand All @@ -134,16 +137,21 @@ program.action(async (options: ProcessingOptions) => {
options.whisper = 'base'
}

// Execute the appropriate handler based on the action
for (const [key, handler] of Object.entries(PROCESS_HANDLERS)) {
if (options[key as keyof ProcessingOptions]) {
try {
await handler(options, options[key as keyof ProcessingOptions] as string, llmServices, finalTranscriptServices)
exit(0)
} catch (error) {
console.error(`Error processing ${key}:`, (error as Error).message)
exit(1)
}
if (action) {
try {
await PROCESS_HANDLERS[action](
options,
options[action as keyof ProcessingOptions] as string,
llmServices,
finalTranscriptServices
)
log(final(`\n==================================================`))
log(final(` ${action} Processing Completed Successfully.`))
log(final(`==================================================\n`))
exit(0)
} catch (error) {
console.error(`Error processing ${action}:`, (error as Error).message)
exit(1)
}
}
})
Expand Down
30 changes: 8 additions & 22 deletions src/commands/processFile.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { downloadAudio } from '../utils/downloadAudio.js'
import { runTranscription } from '../utils/runTranscription.js'
import { runLLM } from '../utils/runLLM.js'
import { cleanUpFiles } from '../utils/cleanUpFiles.js'
import { log, final } from '../models.js'
import { log, opts, wait } from '../models.js'
import type { LLMServices, TranscriptServices, ProcessingOptions } from '../types.js'

/**
Expand All @@ -22,30 +22,16 @@ export async function processFile(
llmServices?: LLMServices,
transcriptServices?: TranscriptServices
): Promise<void> {
// log(`Options received in processFile:\n`)
// log(options)
// log(`filePath:`, filePath)
// log(`llmServices:`, llmServices)
// log(`transcriptServices:`, transcriptServices)
log(opts('Parameters passed to processFile:\n'))
log(wait(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}\n`))
try {
// Generate markdown for the file
const { frontMatter, finalPath, filename } = await generateMarkdown(options, filePath)

// Convert the audio or video file to the required format
await downloadAudio(options, filePath, filename)

// Run transcription on the file
await runTranscription(options, finalPath, frontMatter, transcriptServices)

// Process the transcript with the selected Language Model
await runLLM(options, finalPath, frontMatter, llmServices)

// Clean up temporary files if the noCleanUp option is not set
if (!options.noCleanUp) {
const { frontMatter, finalPath, filename } = await generateMarkdown(options, filePath) // Generate markdown for the file
await downloadAudio(options, filePath, filename) // Convert the audio or video file to the required format
await runTranscription(options, finalPath, frontMatter, transcriptServices) // Run transcription on the file
await runLLM(options, finalPath, frontMatter, llmServices) // Process the transcript with the selected Language Model
if (!options.noCleanUp) { // Clean up temporary files if the noCleanUp option is not set
await cleanUpFiles(finalPath)
}

log(final('\nLocal file processing completed successfully.\n'))
} catch (error) {
console.error(`Error processing file: ${(error as Error).message}`)
process.exit(1) // Exit with an error code
Expand Down
19 changes: 8 additions & 11 deletions src/commands/processPlaylist.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { promisify } from 'node:util'
import { processVideo } from './processVideo.js'
import { extractVideoMetadata } from '../utils/extractVideoMetadata.js'
import { checkDependencies } from '../utils/checkDependencies.js'
import { log, final, wait } from '../models.js'
import { log, opts, success, wait } from '../models.js'
import type { LLMServices, TranscriptServices, ProcessingOptions } from '../types.js'

const execFilePromise = promisify(execFile)
Expand All @@ -24,11 +24,8 @@ export async function processPlaylist(
llmServices?: LLMServices,
transcriptServices?: TranscriptServices
): Promise<void> {
// log(`Options received in processPlaylist:\n`)
// log(options)
// log(`playlistUrl:`, playlistUrl)
// log(`llmServices:`, llmServices)
// log(`transcriptServices:`, transcriptServices)
log(opts('Parameters passed to processPlaylist:\n'))
log(wait(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}`))
try {
// Check for required dependencies
await checkDependencies(['yt-dlp'])
Expand All @@ -52,7 +49,7 @@ export async function processPlaylist(
process.exit(1) // Exit with an error code
}

log(wait(` Found ${urls.length} videos in the playlist...`))
log(opts(`\nFound ${urls.length} videos in the playlist...`))

// Extract metadata for all videos
const metadataPromises = urls.map(extractVideoMetadata)
Expand All @@ -64,22 +61,22 @@ export async function processPlaylist(
const jsonContent = JSON.stringify(validMetadata, null, 2)
const jsonFilePath = 'content/playlist_info.json'
await writeFile(jsonFilePath, jsonContent)
log(wait(`Playlist information saved to: ${jsonFilePath}`))
log(success(`Playlist information saved to: ${jsonFilePath}`))
return
}

// Process each video in the playlist
for (const [index, url] of urls.entries()) {
log(wait(`\n Processing video ${index + 1}/${urls.length}:\n - ${url}\n`))
log(opts(`\n==============================================================`))
log(opts(` Processing video ${index + 1}/${urls.length}: ${url}`))
log(opts(`==============================================================\n`))
try {
await processVideo(options, url, llmServices, transcriptServices)
} catch (error) {
console.error(`Error processing video ${url}: ${(error as Error).message}`)
// Continue processing the next video
}
}

log(final('\nPlaylist processing completed successfully.\n'))
} catch (error) {
console.error(`Error processing playlist: ${(error as Error).message}`)
process.exit(1) // Exit with an error code
Expand Down
62 changes: 24 additions & 38 deletions src/commands/processRSS.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { downloadAudio } from '../utils/downloadAudio.js'
import { runTranscription } from '../utils/runTranscription.js'
import { runLLM } from '../utils/runLLM.js'
import { cleanUpFiles } from '../utils/cleanUpFiles.js'
import { log, final, wait } from '../models.js'
import { log, final, wait, opts } from '../models.js'

import type { LLMServices, TranscriptServices, ProcessingOptions, RSSItem } from '../types.js'

Expand All @@ -32,30 +32,16 @@ async function processItem(
llmServices?: LLMServices,
transcriptServices?: TranscriptServices
): Promise<void> {
// log(`Options received in processItem:\n`)
// log(options)
// log(`item\n\n`, item)
// log(`llmServices:`, llmServices)
// log(`transcriptServices:`, transcriptServices)
log(opts('Parameters passed to processItem:\n'))
log(wait(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}\n`))
try {
// Generate markdown for the item
const { frontMatter, finalPath, filename } = await generateMarkdown(options, item)

// Download audio
await downloadAudio(options, item.showLink, filename)

// Run transcription
await runTranscription(options, finalPath, frontMatter, transcriptServices)

// Process with Language Model
await runLLM(options, finalPath, frontMatter, llmServices)

// Clean up temporary files if necessary
if (!options.noCleanUp) {
const { frontMatter, finalPath, filename } = await generateMarkdown(options, item) // Generate markdown for the item
await downloadAudio(options, item.showLink, filename) // Download audio
await runTranscription(options, finalPath, frontMatter, transcriptServices) // Run transcription
await runLLM(options, finalPath, frontMatter, llmServices) // Process with Language Model
if (!options.noCleanUp) { // Clean up temporary files if necessary
await cleanUpFiles(finalPath)
}

log(final(`\nItem processing completed successfully: ${item.title}`))
} catch (error) {
console.error(`Error processing item ${item.title}: ${(error as Error).message}`)
// Continue processing the next item
Expand All @@ -76,11 +62,8 @@ export async function processRSS(
llmServices?: LLMServices,
transcriptServices?: TranscriptServices
): Promise<void> {
log(`Options received in processRSS:\n`)
log(options)
log(`rssUrl:`, rssUrl)
log(`llmServices:`, llmServices)
log(`transcriptServices:`, transcriptServices)
log(opts('Parameters passed to processRSS:\n'))
log(` - llmServices: ${llmServices}\n - transcriptServices: ${transcriptServices}`)
try {
// Validate that --last is a positive integer if provided
if (options.last !== undefined) {
Expand Down Expand Up @@ -115,11 +98,11 @@ export async function processRSS(
if (options.item && options.item.length > 0) {
// If specific items are provided, list them
log(wait('\nProcessing specific items:'))
options.item.forEach((url) => log(` - ${url}`))
options.item.forEach((url) => log(wait(` - ${url}`)))
} else if (options.last) {
console.log(`\nProcessing the last ${options.last} items`)
} else {
console.log(` - Skipping first ${options.skip || 0} items`)
log(wait(`\nProcessing the last ${options.last} items`))
} else if (options.skip) {
log(wait(` - Skipping first ${options.skip || 0} items`))
}

// Fetch the RSS feed with a timeout
Expand Down Expand Up @@ -213,28 +196,31 @@ export async function processRSS(
process.exit(1) // Exit with an error code
}
itemsToProcess = matchedItems
log(wait(` Found ${items.length} items in the RSS feed.`))
log(wait(`\n - Found ${items.length} items in the RSS feed.`))
log(wait(` - Processing ${itemsToProcess.length} specified items.`))
} else if (options.last) {
// Process the most recent N items
itemsToProcess = items.slice(0, options.last)
log(wait(` Found ${items.length} items in the RSS feed.`))
log(wait(`\n - Found ${items.length} items in the RSS feed.`))
log(wait(` - Processing the last ${options.last} items.`))
} else {
// Sort items based on the specified order and apply skip
const sortedItems = options.order === 'oldest' ? items.slice().reverse() : items
itemsToProcess = sortedItems.slice(options.skip || 0)
log(wait(` Found ${items.length} items in the RSS feed.`))
log(wait(` - Processing ${itemsToProcess.length} items after skipping ${options.skip || 0}.\n`))
log(wait(`\n - Found ${items.length} item(s) in the RSS feed.`))
log(wait(` - Processing ${itemsToProcess.length} item(s) after skipping ${options.skip || 0}.\n`))
}

// Process each item in the feed
for (const [index, item] of itemsToProcess.entries()) {
log(wait(` Processing item ${index + 1}/${itemsToProcess.length}:\n - ${item.title}\n`))
log(opts(`\n==============================================================`))
log(opts(` Item ${index + 1}/${itemsToProcess.length} processing: ${item.title}`))
log(opts(`==============================================================\n`))
await processItem(options, item, llmServices, transcriptServices)
log(final(`\n==============================================================`))
log(final(` ${index + 1}/${itemsToProcess.length} item processing completed successfully`))
log(final(`==============================================================\n`))
}

log(final('\nRSS feed processing completed successfully.\n'))
} catch (error) {
console.error(`Error processing RSS feed: ${(error as Error).message}`)
process.exit(1) // Exit with an error code
Expand Down
Loading

0 comments on commit da9d493

Please sign in to comment.