Merge pull request #55 from ajcwebdev/info

Pass Multiple RSS Feeds to `--rss` and `create_clips` script

.gitignore

@@ -11,10 +11,10 @@ build
 deno.lock
 out
 dist
-NEW.md
-TODO.md
+LLM.md
 nemo_msdd_configs
 temp_outputs
 tsconfig.tsbuildinfo
 show_notes.db
-ollama.log
+ollama.log
+llm-txt.md

scripts/cleanContent.ts

@@ -2,7 +2,7 @@
 
 import { exec } from 'node:child_process'
 import { promisify } from 'node:util'
-import { l, err } from '../src/globals'
+import { l, err } from '../src/types/globals'
 
 const execAsync = promisify(exec)
 

scripts/create_clips.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+# Usage:
+#
+#   ./scripts/create_clips.sh <markdown_file> <video_file>
+
+# Example:
+# 
+#   ./scripts/create_clips.sh content/2021-05-10-thoughts-on-lambda-school-layoffs.md content/2021-05-10-thoughts-on-lambda-school-layoffs.wav
+
+# Check for correct number of arguments
+if [ $# -ne 2 ]; then
+    echo "Usage: $0 <markdown_file> <video_file>"
+    exit 1
+fi
+
+# Input arguments
+markdown_file="$1"
+video_file="$2"
+
+# Arrays to hold timestamps and titles
+timestamps=()
+titles=()
+
+# Read the markdown file
+while read -r line; do
+    if [[ "$line" =~ ^###[[:space:]]([0-9]{1,2}:[0-9]{2}(:[0-9]{2})?)[[:space:]]-[[:space:]](.*)$ ]]; then
+        timestamp="${BASH_REMATCH[1]}"
+        title="${BASH_REMATCH[3]}"
+        timestamps+=("$timestamp")
+        titles+=("$title")
+    fi
+done < "$markdown_file"
+
+# Check if we have any chapters
+if [ ${#timestamps[@]} -eq 0 ]; then
+    echo "No chapters found in $markdown_file"
+    exit 1
+fi
+
+# Get total duration of the video in seconds
+total_duration=$(ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 "$video_file")
+total_duration_seconds=$(printf "%.0f" "$total_duration")
+
+# Function to convert timestamp to seconds
+timestamp_to_seconds() {
+    local time_str="$1"
+    local h=0
+    local m=0
+    local s=0
+    IFS=':' read -r -a time_parts <<< "$time_str"
+    if [ ${#time_parts[@]} -eq 3 ]; then
+        h="${time_parts[0]}"
+        m="${time_parts[1]}"
+        s="${time_parts[2]}"
+    elif [ ${#time_parts[@]} -eq 2 ]; then
+        h=0
+        m="${time_parts[0]}"
+        s="${time_parts[1]}"
+    else
+        echo "Invalid time format: $time_str"
+        exit 1
+    fi
+    total_seconds=$((10#$h * 3600 + 10#$m * 60 + 10#$s))
+    echo "$total_seconds"
+}
+
+# Function to sanitize titles for filenames
+sanitize() {
+    local s="$1"
+    s="$(echo "$s" | tr '[:upper:]' '[:lower:]')"  # Convert to lowercase
+    s="$(echo "$s" | sed 's/[^a-z0-9]/-/g')"       # Replace non-alphanumerics with dashes
+    s="$(echo "$s" | sed 's/--*/-/g')"             # Replace multiple dashes with a single dash
+    s="$(echo "$s" | sed 's/^-//' | sed 's/-$//')" # Trim leading and trailing dashes
+    echo "$s"
+}
+
+# Loop over chapters
+num_chapters=${#timestamps[@]}
+
+for ((i=0; i<num_chapters; i++)); do
+    start_time="${timestamps[i]}"
+    title="${titles[i]}"
+    sanitized_title="$(sanitize "$title")"
+    start_seconds=$(timestamp_to_seconds "$start_time")
+
+    if [ $i -lt $((num_chapters - 1)) ]; then
+        end_time="${timestamps[i+1]}"
+        end_seconds=$(timestamp_to_seconds "$end_time")
+    else
+        end_seconds=$total_duration_seconds
+    fi
+
+    duration_seconds=$((end_seconds - start_seconds))
+
+    if [ $duration_seconds -le 0 ]; then
+        echo "Invalid duration for clip: $title"
+        continue
+    fi
+
+    # Now, use ffmpeg to extract the clip
+    output_file="${sanitized_title}.mp4"
+    echo "Extracting clip: $output_file (Start: $start_time, Duration: $duration_seconds seconds)"
+    ffmpeg -y -ss "$start_time" -i "$video_file" -t "$duration_seconds" -c copy "$output_file"
+done

src/cli/commander.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-// src/autoshow.ts
+// src/cli/commander.ts
 
 /**
  * Autoshow CLI Application
@@ -21,7 +21,7 @@ import { processFile } from '../commands/processFile'
 import { processRSS } from '../commands/processRSS'
 import { validateOption } from '../utils/validateOption'
 import { argv, exit } from 'node:process'
-import { l, err, opts, final, ACTION_OPTIONS, LLM_OPTIONS, TRANSCRIPT_OPTIONS } from '../globals'
+import { l, err, opts, final, ACTION_OPTIONS, LLM_OPTIONS, TRANSCRIPT_OPTIONS } from '../types/globals'
 import type { ProcessingOptions, HandlerFunction, LLMServices, TranscriptServices } from '../types/main'
 
 // Initialize the command-line interface using Commander.js
@@ -45,18 +45,6 @@ function isValidAction(action: string | undefined): action is ValidAction {
   return Boolean(action && action in PROCESS_HANDLERS)
 }
 
-/**
- * Collector function to handle multiple RSS feed URLs.
- *
- * @param value - The current RSS URL provided by the user.
- * @param previous - The array of previously collected RSS URLs.
- * @returns The updated array of RSS URLs.
- */
-function collectRss(value: string, previous?: string[]): string[] {
-  // If 'previous' is undefined, initialize it as an empty array
-  return previous ? previous.concat([value]) : [value]
-}
-
 /**
  * Defines the command-line interface options and descriptions.
  * Sets up all available commands and their respective flags.
@@ -72,12 +60,7 @@ program
   .option('-c, --channel <channelUrl>', 'Process all videos in a YouTube channel')
   .option('-u, --urls <filePath>', 'Process YouTube videos from a list of URLs in a file')
   .option('-f, --file <filePath>', 'Process a local audio or video file')
-  // Modify the --rss option to accept multiple values without a default value
-  .option<string[]>(
-    '-r, --rss <rssURL>',
-    'Process a podcast RSS feed',
-    collectRss
-  )
+  .option('-r, --rss <rssURLs...>', 'Process one or more podcast RSS feeds')
   // RSS feed specific options
   .option('--item <itemUrls...>', 'Process specific items in the RSS feed by providing their audio URLs')
   .option('--order <order>', 'Specify the order for RSS feed processing (newest or oldest)')

src/cli/interactive.ts

@@ -1,12 +1,12 @@
-// src/interactive.ts
+// src/cli/interactive.ts
 
 import inquirer from 'inquirer'
 import {
   l, PROCESS_CHOICES, TRANSCRIPT_SERVICES, WHISPER_MODELS,
   LLM_SERVICES, LLM_OPTIONS, OLLAMA_MODELS, GPT_MODELS,
   CLAUDE_MODELS, GEMINI_MODELS, COHERE_MODELS, MISTRAL_MODELS,
   FIREWORKS_MODELS, TOGETHER_MODELS, GROQ_MODELS, PROMPT_CHOICES
-} from '../globals'
+} from '../types/globals'
 import type { LLMServices, ProcessingOptions, InquirerAnswers, WhisperModelType } from '../types/main'
 
 /**

src/commands/processChannel.ts

@@ -7,7 +7,7 @@
 
 import { writeFile } from 'node:fs/promises'
 import { processVideo } from './processVideo'
-import { l, err, opts, success, execFilePromise, wait } from '../globals'
+import { l, err, opts, success, execFilePromise, wait } from '../types/globals'
 import type { LLMServices, TranscriptServices, ProcessingOptions, VideoMetadata } from '../types/main'
 
 /**

src/commands/processFile.ts

@@ -10,7 +10,7 @@ import { downloadAudio } from '../utils/downloadAudio'
 import { runTranscription } from '../utils/runTranscription'
 import { runLLM } from '../utils/runLLM'
 import { cleanUpFiles } from '../utils/cleanUpFiles'
-import { l, err, opts } from '../globals'
+import { l, err, opts } from '../types/globals'
 import { readFile } from 'fs/promises'
 import { db } from '../server/db'
 import type { LLMServices, TranscriptServices, ProcessingOptions } from '../types/main'

src/commands/processPlaylist.ts

@@ -7,7 +7,7 @@
 
 import { writeFile } from 'node:fs/promises'
 import { processVideo } from './processVideo'
-import { l, err, opts, success, execFilePromise } from '../globals'
+import { l, err, opts, success, execFilePromise } from '../types/globals'
 import { sanitizeTitle } from '../utils/generateMarkdown'
 import type { LLMServices, TranscriptServices, ProcessingOptions, VideoMetadata } from '../types/main'
 

src/commands/processRSS.ts

@@ -11,7 +11,7 @@ import { downloadAudio } from '../utils/downloadAudio'
 import { runTranscription } from '../utils/runTranscription'
 import { runLLM } from '../utils/runLLM'
 import { cleanUpFiles } from '../utils/cleanUpFiles'
-import { l, err, wait, opts, parser } from '../globals'
+import { l, err, wait, opts, parser } from '../types/globals'
 import { db } from '../server/db'
 import type { LLMServices, TranscriptServices, ProcessingOptions, RSSItem } from '../types/main'
 
@@ -128,14 +128,13 @@ function extractFeedItems(feed: any): { items: RSSItem[], channelTitle: string }
       }
 
       return {
-        showLink: item.enclosure.url || '',
+        showLink: item.enclosure?.url || '',
         channel: channelTitle || '',
         channelURL: channelLink || '',
         title: item.title || '',
-        description: item.description || '',
+        description: '',
         publishDate,
-        coverImage: item['itunes:image']?.href || channelImage || '',
-        audioURL: item.enclosure?.url || ''
+        coverImage: item['itunes:image']?.href || channelImage || ''
       }
     })
 

src/commands/processURLs.ts

@@ -7,7 +7,7 @@
 
 import { readFile, writeFile } from 'node:fs/promises'
 import { processVideo } from './processVideo'
-import { l, err, wait, opts, execFilePromise } from '../globals'
+import { l, err, wait, opts, execFilePromise } from '../types/globals'
 import type { LLMServices, TranscriptServices, ProcessingOptions, VideoMetadata } from '../types/main'
 
 /**

src/commands/processVideo.ts

@@ -10,7 +10,7 @@ import { downloadAudio } from '../utils/downloadAudio'
 import { runTranscription } from '../utils/runTranscription'
 import { runLLM } from '../utils/runLLM'
 import { cleanUpFiles } from '../utils/cleanUpFiles'
-import { l, err, opts } from '../globals'
+import { l, err, opts } from '../types/globals'
 import { readFile } from 'fs/promises'
 import { db } from '../server/db'
 import type { LLMServices, TranscriptServices, ProcessingOptions } from '../types/main'

src/llms/chatgpt.ts

@@ -3,7 +3,7 @@
 import { writeFile } from 'node:fs/promises'
 import { env } from 'node:process'
 import { OpenAI } from 'openai'
-import { l, wait, err, GPT_MODELS } from '../globals'
+import { l, wait, err, GPT_MODELS } from '../types/globals'
 import type { LLMFunction, ChatGPTModelType } from '../types/llm-types'
 
 /**

src/llms/claude.ts

@@ -3,7 +3,7 @@
 import { writeFile } from 'node:fs/promises'
 import { env } from 'node:process'
 import { Anthropic } from '@anthropic-ai/sdk'
-import { l, wait, err, CLAUDE_MODELS } from '../globals'
+import { l, wait, err, CLAUDE_MODELS } from '../types/globals'
 import type { LLMFunction, ClaudeModelType } from '../types/llm-types'
 
 /**

src/llms/cohere.ts

@@ -3,7 +3,7 @@
 import { writeFile } from 'node:fs/promises'
 import { env } from 'node:process'
 import { CohereClient } from 'cohere-ai'
-import { l, wait, err, COHERE_MODELS } from '../globals'
+import { l, wait, err, COHERE_MODELS } from '../types/globals'
 import type { LLMFunction, CohereModelType } from '../types/llm-types'
 
 /**

src/llms/fireworks.ts

@@ -2,7 +2,7 @@
 
 import { writeFile } from 'node:fs/promises'
 import { env } from 'node:process'
-import { l, wait, err, FIREWORKS_MODELS } from '../globals'
+import { l, wait, err, FIREWORKS_MODELS } from '../types/globals'
 import type { LLMFunction, FireworksModelType, FireworksResponse } from '../types/llm-types'
 
 /**

src/llms/gemini.ts

@@ -3,7 +3,7 @@
 import { writeFile } from 'node:fs/promises'
 import { env } from 'node:process'
 import { GoogleGenerativeAI } from "@google/generative-ai"
-import { l, wait, err, GEMINI_MODELS } from '../globals'
+import { l, wait, err, GEMINI_MODELS } from '../types/globals'
 import type { LLMFunction, GeminiModelType } from '../types/llm-types'
 
 /**

src/llms/groq.ts

@@ -2,7 +2,7 @@
 
 import { writeFile } from 'node:fs/promises'
 import { env } from 'node:process'
-import { l, wait, err, GROQ_MODELS } from '../globals'
+import { l, wait, err, GROQ_MODELS } from '../types/globals'
 import type { GroqChatCompletionResponse, GroqModelType } from '../types/llm-types'
 
 // Define the Groq API URL

src/llms/mistral.ts

@@ -3,7 +3,7 @@
 import { writeFile } from 'node:fs/promises'
 import { env } from 'node:process'
 import { Mistral } from '@mistralai/mistralai'
-import { l, wait, err, MISTRAL_MODELS } from '../globals'
+import { l, wait, err, MISTRAL_MODELS } from '../types/globals'
 import type { LLMFunction, MistralModelType } from '../types/llm-types'
 
 /**

src/llms/ollama.ts

@@ -2,7 +2,7 @@
 
 import { writeFile } from 'node:fs/promises'
 import { env } from 'node:process'
-import { OLLAMA_MODELS, l, err, wait } from '../globals'
+import { OLLAMA_MODELS, l, err, wait } from '../types/globals'
 import { spawn } from 'node:child_process'
 import type { LLMFunction, OllamaModelType, OllamaResponse, OllamaTagsResponse } from '../types/llm-types'
 

src/llms/together.ts

@@ -2,7 +2,7 @@
 
 import { writeFile } from 'node:fs/promises'
 import { env } from 'node:process'
-import { l, wait, err, TOGETHER_MODELS } from '../globals'
+import { l, wait, err, TOGETHER_MODELS } from '../types/globals'
 import type { LLMFunction, TogetherModelType, TogetherResponse } from '../types/llm-types'
 
 /**

src/server/index.ts

@@ -1,4 +1,4 @@
-// server/index.js
+// src/server/index.js
 
 import Fastify from 'fastify'
 import cors from '@fastify/cors'
@@ -7,7 +7,7 @@ import { handlePlaylistRequest } from './routes/playlist'
 import { handleURLsRequest } from './routes/urls'
 import { handleFileRequest } from './routes/file'
 import { handleRSSRequest } from './routes/rss'
-import { l } from '../../src/globals'
+import { l } from '../../src/types/globals'
 import { env } from 'node:process'
 import { getShowNotes } from './routes/showNotes'
 import { getShowNote } from './routes/showNote'

src/server/routes/file.ts

@@ -1,9 +1,9 @@
-// server/routes/file.ts
+// src/server/routes/file.ts
 
 import type { FastifyRequest, FastifyReply } from 'fastify'
 import { processFile } from '../../../src/commands/processFile'
 import { reqToOpts } from '../utils/reqToOpts'
-import { l, err } from '../../../src/globals'
+import { l, err } from '../../../src/types/globals'
 
 // Handler for the /file route
 export const handleFileRequest = async (

src/server/routes/playlist.ts

@@ -1,9 +1,9 @@
-// server/routes/playlist.ts
+// src/server/routes/playlist.ts
 
 import type { FastifyRequest, FastifyReply } from 'fastify'
 import { processPlaylist } from '../../../src/commands/processPlaylist'
 import { reqToOpts } from '../utils/reqToOpts'
-import { l, err } from '../../../src/globals'
+import { l, err } from '../../../src/types/globals'
 
 // Handler for the /playlist route
 export const handlePlaylistRequest = async (

src/server/routes/rss.ts

@@ -1,9 +1,9 @@
-// server/routes/rss.ts
+// src/server/routes/rss.ts
 
 import type { FastifyRequest, FastifyReply } from 'fastify'
 import { processRSS } from '../../../src/commands/processRSS'
 import { reqToOpts } from '../utils/reqToOpts'
-import { l, err } from '../../../src/globals'
+import { l, err } from '../../../src/types/globals'
 
 // Handler for the /rss route
 export const handleRSSRequest = async (