From 5bb7703918b5dd775762df1b38bbec2a57933948 Mon Sep 17 00:00:00 2001 From: Phil Date: Fri, 24 Jul 2026 22:28:45 +0100 Subject: [PATCH] Updated script and added new config file --- podcast-downloader.sh | 759 +++++++++++++++++++----------------------- podcast_settings.conf | 23 ++ podcasts.json | 24 -- 3 files changed, 369 insertions(+), 437 deletions(-) create mode 100644 podcast_settings.conf delete mode 100644 podcasts.json diff --git a/podcast-downloader.sh b/podcast-downloader.sh index 1ec5cab..d603cf4 100644 --- a/podcast-downloader.sh +++ b/podcast-downloader.sh @@ -1,443 +1,376 @@ -#!/bin/bash +#!/usr/bin/env bash +set -u +set -o pipefail -# YouTube Playlist Downloader with yt-dlp and Database -# Downloads videos from multiple playlists, maintains a DB of downloaded videos, -# and automatically skips already-downloaded content +CONFIG_FILE="/opt/podcast-pownloader/PODCAST_NAME/podcast_settings.conf" -# Color output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' +if [[ ! -f "$CONFIG_FILE" ]]; then + echo "ERROR: Configuration file not found at $CONFIG_FILE" + exit 1 +fi -# Configuration file path -CONFIG_FILE="${XDG_CONFIG_HOME:-$HOME/.config}/youtube_downloader/config.json" -DEFAULT_CONFIG_DIR="$HOME/.youtube_downloader" +source "$CONFIG_FILE" -# Function to log messages -log_info() { - echo -e "${GREEN}[INFO]${NC} $1" | tee -a "$LOG_FILE" +validate_config() { + local required_vars=( + "DOWNLOAD_DIR" "FINAL_DIR" "LOG_DIR" "LOG_FILE" + "DOWNLOADS_LOG" "DOWNLOADS_LOCK" "MAX_LOG_SIZE" + "QUALITY" "VIDEO_FORMAT" "AUDIO_FORMAT" + "SLEEP_INTERVAL" "RATE_LIMIT" "CONNECTION_TIMEOUT" "MAX_RETRIES" + ) + for var in "${required_vars[@]}"; do + if [[ -z "${!var:-}" ]]; then + echo "ERROR: Required variable '$var' not set in $CONFIG_FILE" + exit 1 + fi + done } -log_error() { - echo -e "${RED}[ERROR]${NC} $1" | tee -a "$LOG_FILE" -} +validate_config -log_warning() { - echo -e "${YELLOW}[WARNING]${NC} $1" | tee -a "$LOG_FILE" -} +mkdir -p "$DOWNLOAD_DIR" "$FINAL_DIR" "$LOG_DIR" "$(dirname "$DOWNLOADS_LOG")" -log_debug() { - echo -e "${BLUE}[DEBUG]${NC} $1" | tee -a "$LOG_FILE" -} +[[ -f "$LOG_FILE" ]] || : > "$LOG_FILE" -# Load configuration from JSON file -load_config() { - if [ ! -f "$CONFIG_FILE" ]; then - log_error "Config file not found: $CONFIG_FILE" - echo "Please create the config file first. You can copy the example from the script comments." - exit 1 +if [[ ! -f "$DOWNLOADS_LOG" ]]; then + { + echo "# YouTube Downloads Database - $(date)" + echo "# Format: VIDEO_ID|TITLE|FILENAME|DOWNLOADED_DATE" + } >> "$DOWNLOADS_LOG" +fi + +rotate_log() { + local size + size=$(stat -c%s "$LOG_FILE" 2>/dev/null || stat -f%z "$LOG_FILE" 2>/dev/null || echo 0) + if [[ "$size" -gt "$MAX_LOG_SIZE" ]]; then + mv "$LOG_FILE" "$LOG_FILE.$(date +%s)" + echo "# YouTube Downloader Log - $(date)" > "$LOG_FILE" + log_message "Log rotated (previous log too large)" fi - - # Check if jq is installed - if ! command -v jq &> /dev/null; then - log_error "jq is not installed. Please install it first." - echo "On Ubuntu/Debian: sudo apt-get install jq" - echo "On macOS: brew install jq" - exit 1 - fi - - # Parse configuration - DB_DIR=$(jq -r '.general.db_dir' "$CONFIG_FILE" | sed "s|\$HOME|$HOME|g") - DOWNLOAD_DIR=$(jq -r '.general.temp_download_dir' "$CONFIG_FILE" | sed "s|\$HOME|$HOME|g") - VIDEO_FORMAT=$(jq -r '.general.video_format' "$CONFIG_FILE") - OUTPUT_TEMPLATE=$(jq -r '.general.output_template' "$CONFIG_FILE") - AUDIO_ONLY=$(jq -r '.general.audio_only' "$CONFIG_FILE") - DOWNLOAD_DELAY=$(jq -r '.general.download_delay' "$CONFIG_FILE") - - # Set derived paths - DB_FILE="$DB_DIR/downloads.db" - LOG_FILE="$DB_DIR/downloader.log" - - log_debug "Configuration loaded from: $CONFIG_FILE" } -# Get playlist count -get_playlist_count() { - jq -r '.playlists | length' "$CONFIG_FILE" +log_message() { + local timestamp + timestamp=$(date '+%Y-%m-%d %H:%M:%S') + echo "[$timestamp] $1" | tee -a "$LOG_FILE" + rotate_log } -# Get specific playlist data -get_playlist_data() { - local index="$1" - local field="$2" - jq -r ".playlists[$index].$field" "$CONFIG_FILE" +sanitize_filename() { + local name="$1" + name="${name//\//-}" + name="${name//:/-}" + name="${name//\*/-}" + name="${name//\?/}" + name="${name//\"/}" + name="${name///}" + name="${name//|/-}" + name="${name//\\/}" + echo "$name" | tr -s ' ' ' ' | sed 's/^ *//; s/ *$//' } -# Get all enabled playlists -get_enabled_playlists() { - jq -r '.playlists[] | select(.enabled == true) | [.name, .url, .destination] | @tsv' "$CONFIG_FILE" +acquire_lock() { + local timeout=30 + local elapsed=0 + while [[ $elapsed -lt $timeout ]]; do + if mkdir "$DOWNLOADS_LOCK" 2>/dev/null; then + return 0 + fi + sleep 1 + ((elapsed++)) + done + return 1 } -# Initialize database -init_database() { - # Create directory if it doesn't exist - mkdir -p "$DB_DIR" || { - log_error "Failed to create database directory: $DB_DIR" - exit 1 - } - - # Check if sqlite3 is installed - if ! command -v sqlite3 &> /dev/null; then - log_error "sqlite3 is not installed. Please install it first." - echo "On Ubuntu/Debian: sudo apt-get install sqlite3" - echo "On macOS: brew install sqlite3" - exit 1 - fi - - # Create database and tables if they don't exist - sqlite3 "$DB_FILE" </dev/null || true } -# Check if video is already downloaded -is_video_downloaded() { - local video_id="$1" - local result=$(sqlite3 "$DB_FILE" "SELECT COUNT(*) FROM downloaded_videos WHERE video_id='$video_id' AND status='completed';") - [ "$result" -gt 0 ] -} +is_downloaded() { + local video_id="${1:-}" + [[ -z "$video_id" ]] && return 1 -# Add video to database -add_to_database() { - local video_id="$1" - local title="$2" - local url="$3" - local playlist_name="$4" - local file_path="$5" - local file_size="$6" - - # Escape single quotes in title - title="${title//\'/\'\'}" - - sqlite3 "$DB_FILE" </dev/null -} - -# Start a new download session -start_session() { - local playlist_name="$1" - local playlist_url="$2" - - local session_id=$(sqlite3 "$DB_FILE" \ - "INSERT INTO download_sessions (playlist_name, playlist_url) VALUES ('$playlist_name', '$playlist_url'); SELECT last_insert_rowid();") - echo "$session_id" -} - -# Update session with results -end_session() { - local session_id="$1" - local downloaded="$2" - local skipped="$3" - local errors="$4" - - sqlite3 "$DB_FILE" </dev/null || echo "$total_size bytes")" - echo "Last download: $last_download" - echo "Total download sessions: $total_sessions" - echo "" - - # Show statistics per playlist - log_info "===== Per-Playlist Statistics =====" - sqlite3 -header -column "$DB_FILE" \ - "SELECT playlist_name, COUNT(*) as videos, ROUND(SUM(file_size)/1024/1024, 2) as size_mb FROM downloaded_videos WHERE status='completed' GROUP BY playlist_name;" -} - -# List recently downloaded videos -list_recent() { - local limit="${1:-10}" - log_info "===== Last $limit Downloads =====" - - sqlite3 -header -column "$DB_FILE" \ - "SELECT playlist_name, title, download_date FROM downloaded_videos WHERE status='completed' ORDER BY download_date DESC LIMIT $limit;" -} - -# Validate inputs -validate_setup() { - local destination_dir="$1" - - if [ ! -d "$destination_dir" ]; then - log_warning "Destination directory does not exist. Creating: $destination_dir" - mkdir -p "$destination_dir" || { - log_error "Failed to create destination directory: $destination_dir" - return 1 - } - fi - - mkdir -p "$DOWNLOAD_DIR" || { - log_error "Failed to create download directory: $DOWNLOAD_DIR" + acquire_lock || { + log_message "WARNING: Could not acquire lock for is_downloaded check" return 1 } - + + grep -q -F -- "${video_id}|" "$DOWNLOADS_LOG" + local result=$? + + release_lock + return $result +} + +record_download() { + local video_id="${1:-}" + local title="${2:-}" + local filename="${3:-}" + + acquire_lock || { + log_message "ERROR: Could not acquire lock for recording download" + return 1 + } + + local timestamp + timestamp=$(date '+%Y-%m-%d %H:%M:%S') + + echo "${video_id}|${title}|${filename}|${timestamp}" >> "$DOWNLOADS_LOG" + local result=$? + + release_lock + return $result +} + +process_downloaded_video() { + local video_id="${1:-}" + local title="${2:-}" + local temp_dir="${3:-}" + + if [[ -z "$video_id" || -z "$temp_dir" ]]; then + log_message "ERROR: Missing video_id or temp_dir in process_downloaded_video" + return 1 + fi + + if is_downloaded "$video_id"; then + log_message "SKIP: '$title' (already in database)" + find "$temp_dir" -maxdepth 1 -type f -name "${video_id}.*" -delete 2>/dev/null + return 0 + fi + + local video_file audio_file + video_file=$(find "$temp_dir" -maxdepth 1 -type f -name "${video_id}.video.*" 2>/dev/null | head -n 1) + audio_file=$(find "$temp_dir" -maxdepth 1 -type f -name "${video_id}.audio.mp3" 2>/dev/null | head -n 1) + + if [[ -z "${video_file:-}" || ! -f "$video_file" ]]; then + log_message "ERROR: Video file not found for '$title' (id: $video_id)" + return 1 + fi + + if [[ -z "${audio_file:-}" || ! -f "$audio_file" ]]; then + log_message "ERROR: Audio MP3 file not found for '$title' (id: $video_id)" + return 1 + fi + + local clean_title base_name final_video final_audio + clean_title=$(sanitize_filename "$title") + base_name="$clean_title" + + final_video="$FINAL_DIR/${base_name}.${VIDEO_FORMAT}" + final_audio="$FINAL_DIR/${base_name}.mp3" + + if [[ -e "$final_video" || -e "$final_audio" ]]; then + base_name="${clean_title}_${video_id}" + final_video="$FINAL_DIR/${base_name}.${VIDEO_FORMAT}" + final_audio="$FINAL_DIR/${base_name}.mp3" + fi + + if mv "$video_file" "$final_video" && mv "$audio_file" "$final_audio"; then + if record_download "$video_id" "$title" "${base_name}.${VIDEO_FORMAT} + ${base_name}.mp3"; then + log_message "MOVED: '$title' -> '${base_name}.${VIDEO_FORMAT}' and '${base_name}.mp3'" + return 0 + else + log_message "ERROR: Failed to record download for '$title'" + return 1 + fi + else + log_message "ERROR: Failed to move files for '$title' to $FINAL_DIR" + return 1 + fi +} + +wait_for_next() { + local remaining=$SLEEP_INTERVAL + while [[ $remaining -gt 0 ]]; do + printf "\rWaiting %ss before next download...\033[K" "$remaining" + sleep 1 + ((remaining--)) + done + echo +} + +format_time() { + local seconds=$1 + local hours=$((seconds / 3600)) + local minutes=$(((seconds % 3600) / 60)) + local secs=$((seconds % 60)) + + if [[ $hours -gt 0 ]]; then + printf "%dh %dm %ds" "$hours" "$minutes" "$secs" + elif [[ $minutes -gt 0 ]]; then + printf "%dm %ds" "$minutes" "$secs" + else + printf "%ds" "$secs" + fi +} + +estimate_download_time() { + local playlist_url="${1:-}" + [[ -z "$playlist_url" ]] && return 0 + + log_message "Estimating playlist size..." + local video_count + video_count=$(yt-dlp --no-warnings --flat-playlist --print "%(id)s" "$playlist_url" 2>/dev/null \ + | grep -E '^[a-zA-Z0-9_-]{11}$' | wc -l) + + if [[ $video_count -gt 0 ]]; then + local estimated_seconds=$(( (video_count * 300) + (video_count * SLEEP_INTERVAL) )) + log_message "Estimated: $video_count videos, approximately $(format_time "$estimated_seconds") total time" + fi +} + +show_log() { + echo + echo "=== Downloaded Videos (Last 20) ===" + echo + tail -n 20 "$DOWNLOADS_LOG" 2>/dev/null | grep -E '^[a-zA-Z0-9_-]{11}\|' | \ + awk -F'|' '{printf "%-15s | %-50s | %s\n", $1, substr($2,1,50), $4}' || echo "No downloads yet" +} + +download_playlist() { + local playlist_url="${1:-}" + local temp_dir="" + local video_list="" + local current=0 + local video_count=0 + + if [[ -z "$playlist_url" ]]; then + log_message "ERROR: No playlist URL provided" + echo "Usage: $0 [youtube_playlist_url]" + return 1 + fi + + cleanup() { + [[ -n "${video_list:-}" && -f "$video_list" ]] && rm -f "$video_list" + [[ -n "${temp_dir:-}" && -d "$temp_dir" ]] && rm -rf "$temp_dir" + } + trap cleanup EXIT + + log_message "Starting download for playlist: $playlist_url" + log_message "Settings: Quality=$QUALITY, Rate Limit=$RATE_LIMIT, Sleep Interval=${SLEEP_INTERVAL}s" + + temp_dir=$(mktemp -d) || { + log_message "ERROR: Failed to create temporary directory" + return 1 + } + + video_list=$(mktemp) || { + log_message "ERROR: Failed to create temporary file" + return 1 + } + + log_message "Fetching playlist information..." + + yt-dlp \ + --no-warnings \ + --flat-playlist \ + --socket-timeout "$CONNECTION_TIMEOUT" \ + --retries "$MAX_RETRIES" \ + --print "%(id)s" \ + "$playlist_url" 2>>"$LOG_FILE" | grep -E '^[a-zA-Z0-9_-]{11}$' > "$video_list" + + if [[ ! -s "$video_list" ]]; then + log_message "ERROR: Failed to fetch playlist or playlist is empty" + return 1 + fi + + video_count=$(wc -l < "$video_list") + log_message "Found $video_count videos to process" + + while IFS= read -r video_id; do + ((current++)) + + [[ -z "${video_id:-}" ]] && continue + + if ! [[ "$video_id" =~ ^[a-zA-Z0-9_-]{11}$ ]]; then + log_message "[$current/$video_count] WARNING: Invalid video ID format: '$video_id'" + continue + fi + + title=$(yt-dlp \ + --no-warnings \ + --skip-download \ + --socket-timeout "$CONNECTION_TIMEOUT" \ + --retries "$MAX_RETRIES" \ + --print "%(title)s" \ + "https://www.youtube.com/watch?v=$video_id" 2>>"$LOG_FILE" | head -n 1) + + [[ -z "$title" || "$title" == "NA" ]] && title="$video_id" + + if is_downloaded "$video_id"; then + log_message "[$current/$video_count] SKIP: '$title' (already downloaded)" + continue + fi + + log_message "[$current/$video_count] Downloading: '$title'" + + yt-dlp \ + --no-warnings \ + --format "$QUALITY" \ + --merge-output-format "$VIDEO_FORMAT" \ + --limit-rate "$RATE_LIMIT" \ + --socket-timeout "$CONNECTION_TIMEOUT" \ + --retries "$MAX_RETRIES" \ + --no-continue \ + --output "$temp_dir/%(id)s.video.%(ext)s" \ + "https://www.youtube.com/watch?v=$video_id" \ + 2>>"$LOG_FILE" + video_status=$? + + yt-dlp \ + --no-warnings \ + --extract-audio \ + --audio-format mp3 \ + --audio-quality 0 \ + --socket-timeout "$CONNECTION_TIMEOUT" \ + --retries "$MAX_RETRIES" \ + --no-continue \ + --output "$temp_dir/%(id)s.audio.%(ext)s" \ + "https://www.youtube.com/watch?v=$video_id" \ + 2>>"$LOG_FILE" + audio_status=$? + + if [[ $video_status -eq 0 && $audio_status -eq 0 ]]; then + if process_downloaded_video "$video_id" "$title" "$temp_dir"; then + log_message "[$current/$video_count] Successfully processed '$title'" + [[ $current -lt $video_count ]] && wait_for_next + else + log_message "[$current/$video_count] ERROR: Processing failed for '$title'" + fi + else + log_message "[$current/$video_count] ERROR: Download failed for '$title' (video: $video_status, audio: $audio_status)" + fi + done < "$video_list" + + log_message "Playlist download completed" return 0 } -# Main download function for a single playlist -download_playlist() { - local playlist_name="$1" - local playlist_url="$2" - local destination_dir="$3" - - log_info "==========================================" - log_info "Starting download for: $playlist_name" - log_info "Playlist URL: $playlist_url" - log_info "Destination: $destination_dir" - log_info "==========================================" - - # Validate setup - if ! validate_setup "$destination_dir"; then - return 1 +main() { + local start_time end_time duration formatted_duration + start_time=$(date +%s) + + local playlist_url="${1:-${PLAYLIST_URL:-}}" + if [[ -z "$playlist_url" ]]; then + echo "ERROR: No playlist URL provided and PLAYLIST_URL is not set in the config" + exit 1 fi - - local session_id=$(start_session "$playlist_name" "$playlist_url") - log_debug "Session ID: $session_id" - - local download_count=0 - local skip_count=0 - local error_count=0 - - # Get all videos in the playlist - local playlist_data=$(get_playlist_videos "$playlist_url") - - if [ -z "$playlist_data" ]; then - log_error "Failed to fetch playlist. Check URL and internet connection." - end_session "$session_id" 0 0 1 - return 1 - fi - - # Process each video - while IFS='|' read -r video_id title video_url; do - [ -z "$video_id" ] && continue - - if is_video_downloaded "$video_id"; then - log_info "Skipping (already downloaded): $title" - skip_count=$((skip_count + 1)) - continue - fi - - log_info "Downloading: $title (ID: $video_id)" - - # Download the video - local temp_file="$DOWNLOAD_DIR/${title}.%(ext)s" - - if yt-dlp \ - -f "$VIDEO_FORMAT" \ - -o "$temp_file" \ - --quiet \ - --no-warnings \ - "$video_url" 2>/dev/null; then - - # Find the actual downloaded file - local downloaded_file=$(find "$DOWNLOAD_DIR" -name "${title}.*" -type f -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2-) - - if [ -f "$downloaded_file" ]; then - local file_size=$(stat -f%z "$downloaded_file" 2>/dev/null || stat -c%s "$downloaded_file" 2>/dev/null) - local file_name=$(basename "$downloaded_file") - - # Move to destination - if mv "$downloaded_file" "$destination_dir/" 2>/dev/null; then - log_info "Moved to destination: $file_name" - - # Add to database - add_to_database "$video_id" "$title" "$video_url" "$playlist_name" "$destination_dir/$file_name" "$file_size" - - download_count=$((download_count + 1)) - else - log_error "Failed to move: $file_name" - error_count=$((error_count + 1)) - fi - else - log_error "Downloaded file not found" - error_count=$((error_count + 1)) - fi - else - log_error "Download failed: $title" - error_count=$((error_count + 1)) - fi - - # Small delay between downloads to avoid rate limiting - sleep "$DOWNLOAD_DELAY" - - done <<< "$playlist_data" - - # Cleanup - rm -rf "$DOWNLOAD_DIR" 2>/dev/null - - # End session - end_session "$session_id" "$download_count" "$skip_count" "$error_count" - - # Summary - echo "" - log_info "===== Download Summary for: $playlist_name =====" - echo "New videos downloaded: $download_count" - echo "Videos skipped (already have): $skip_count" - echo "Errors: $error_count" - echo "" + + estimate_download_time "$playlist_url" + download_playlist "$playlist_url" + local result=$? + + show_log + + end_time=$(date +%s) + duration=$((end_time - start_time)) + formatted_duration=$(format_time "$duration") + + log_message "Total execution time: $formatted_duration" + log_message "Full downloads database saved to: $DOWNLOADS_LOG" + return $result } -# Download from all enabled playlists -download_all_playlists() { - log_info "Starting downloads from all enabled playlists..." - - local total_downloaded=0 - local total_skipped=0 - local total_errors=0 - - get_enabled_playlists | while IFS=$'\t' read -r playlist_name playlist_url destination_dir; do - if download_playlist "$playlist_name" "$playlist_url" "$destination_dir"; then - # Note: We can't increment variables in subshells, so we'll just log per-playlist - : - fi - done - - log_info "All playlist downloads completed!" -} +main "$@" -# Download from a specific playlist -download_specific_playlist() { - local playlist_name="$1" - - local count=$(get_playlist_count) - local found=false - - for ((i=0; i