Updated script and added new config file
This commit is contained in:
+346
-413
@@ -1,443 +1,376 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
# YouTube Playlist Downloader with yt-dlp and Database
|
||||
# Downloads videos from multiple playlists, maintains a DB of downloaded videos,
|
||||
# and automatically skips already-downloaded content
|
||||
CONFIG_FILE="/opt/podcast-pownloader/PODCAST_NAME/podcast_settings.conf"
|
||||
|
||||
# Color output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
if [[ ! -f "$CONFIG_FILE" ]]; then
|
||||
echo "ERROR: Configuration file not found at $CONFIG_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Configuration file path
|
||||
CONFIG_FILE="${XDG_CONFIG_HOME:-$HOME/.config}/youtube_downloader/config.json"
|
||||
DEFAULT_CONFIG_DIR="$HOME/.youtube_downloader"
|
||||
source "$CONFIG_FILE"
|
||||
|
||||
# Function to log messages
|
||||
log_info() {
|
||||
echo -e "${GREEN}[INFO]${NC} $1" | tee -a "$LOG_FILE"
|
||||
validate_config() {
|
||||
local required_vars=(
|
||||
"DOWNLOAD_DIR" "FINAL_DIR" "LOG_DIR" "LOG_FILE"
|
||||
"DOWNLOADS_LOG" "DOWNLOADS_LOCK" "MAX_LOG_SIZE"
|
||||
"QUALITY" "VIDEO_FORMAT" "AUDIO_FORMAT"
|
||||
"SLEEP_INTERVAL" "RATE_LIMIT" "CONNECTION_TIMEOUT" "MAX_RETRIES"
|
||||
)
|
||||
for var in "${required_vars[@]}"; do
|
||||
if [[ -z "${!var:-}" ]]; then
|
||||
echo "ERROR: Required variable '$var' not set in $CONFIG_FILE"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1" | tee -a "$LOG_FILE"
|
||||
}
|
||||
validate_config
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $1" | tee -a "$LOG_FILE"
|
||||
}
|
||||
mkdir -p "$DOWNLOAD_DIR" "$FINAL_DIR" "$LOG_DIR" "$(dirname "$DOWNLOADS_LOG")"
|
||||
|
||||
log_debug() {
|
||||
echo -e "${BLUE}[DEBUG]${NC} $1" | tee -a "$LOG_FILE"
|
||||
}
|
||||
[[ -f "$LOG_FILE" ]] || : > "$LOG_FILE"
|
||||
|
||||
# Load configuration from JSON file
|
||||
load_config() {
|
||||
if [ ! -f "$CONFIG_FILE" ]; then
|
||||
log_error "Config file not found: $CONFIG_FILE"
|
||||
echo "Please create the config file first. You can copy the example from the script comments."
|
||||
exit 1
|
||||
if [[ ! -f "$DOWNLOADS_LOG" ]]; then
|
||||
{
|
||||
echo "# YouTube Downloads Database - $(date)"
|
||||
echo "# Format: VIDEO_ID|TITLE|FILENAME|DOWNLOADED_DATE"
|
||||
} >> "$DOWNLOADS_LOG"
|
||||
fi
|
||||
|
||||
rotate_log() {
|
||||
local size
|
||||
size=$(stat -c%s "$LOG_FILE" 2>/dev/null || stat -f%z "$LOG_FILE" 2>/dev/null || echo 0)
|
||||
if [[ "$size" -gt "$MAX_LOG_SIZE" ]]; then
|
||||
mv "$LOG_FILE" "$LOG_FILE.$(date +%s)"
|
||||
echo "# YouTube Downloader Log - $(date)" > "$LOG_FILE"
|
||||
log_message "Log rotated (previous log too large)"
|
||||
fi
|
||||
|
||||
# Check if jq is installed
|
||||
if ! command -v jq &> /dev/null; then
|
||||
log_error "jq is not installed. Please install it first."
|
||||
echo "On Ubuntu/Debian: sudo apt-get install jq"
|
||||
echo "On macOS: brew install jq"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Parse configuration
|
||||
DB_DIR=$(jq -r '.general.db_dir' "$CONFIG_FILE" | sed "s|\$HOME|$HOME|g")
|
||||
DOWNLOAD_DIR=$(jq -r '.general.temp_download_dir' "$CONFIG_FILE" | sed "s|\$HOME|$HOME|g")
|
||||
VIDEO_FORMAT=$(jq -r '.general.video_format' "$CONFIG_FILE")
|
||||
OUTPUT_TEMPLATE=$(jq -r '.general.output_template' "$CONFIG_FILE")
|
||||
AUDIO_ONLY=$(jq -r '.general.audio_only' "$CONFIG_FILE")
|
||||
DOWNLOAD_DELAY=$(jq -r '.general.download_delay' "$CONFIG_FILE")
|
||||
|
||||
# Set derived paths
|
||||
DB_FILE="$DB_DIR/downloads.db"
|
||||
LOG_FILE="$DB_DIR/downloader.log"
|
||||
|
||||
log_debug "Configuration loaded from: $CONFIG_FILE"
|
||||
}
|
||||
|
||||
# Get playlist count
|
||||
get_playlist_count() {
|
||||
jq -r '.playlists | length' "$CONFIG_FILE"
|
||||
log_message() {
|
||||
local timestamp
|
||||
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
echo "[$timestamp] $1" | tee -a "$LOG_FILE"
|
||||
rotate_log
|
||||
}
|
||||
|
||||
# Get specific playlist data
|
||||
get_playlist_data() {
|
||||
local index="$1"
|
||||
local field="$2"
|
||||
jq -r ".playlists[$index].$field" "$CONFIG_FILE"
|
||||
sanitize_filename() {
|
||||
local name="$1"
|
||||
name="${name//\//-}"
|
||||
name="${name//:/-}"
|
||||
name="${name//\*/-}"
|
||||
name="${name//\?/}"
|
||||
name="${name//\"/}"
|
||||
name="${name//</}"
|
||||
name="${name//>/}"
|
||||
name="${name//|/-}"
|
||||
name="${name//\\/}"
|
||||
echo "$name" | tr -s ' ' ' ' | sed 's/^ *//; s/ *$//'
|
||||
}
|
||||
|
||||
# Get all enabled playlists
|
||||
get_enabled_playlists() {
|
||||
jq -r '.playlists[] | select(.enabled == true) | [.name, .url, .destination] | @tsv' "$CONFIG_FILE"
|
||||
acquire_lock() {
|
||||
local timeout=30
|
||||
local elapsed=0
|
||||
while [[ $elapsed -lt $timeout ]]; do
|
||||
if mkdir "$DOWNLOADS_LOCK" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
((elapsed++))
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Initialize database
|
||||
init_database() {
|
||||
# Create directory if it doesn't exist
|
||||
mkdir -p "$DB_DIR" || {
|
||||
log_error "Failed to create database directory: $DB_DIR"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Check if sqlite3 is installed
|
||||
if ! command -v sqlite3 &> /dev/null; then
|
||||
log_error "sqlite3 is not installed. Please install it first."
|
||||
echo "On Ubuntu/Debian: sudo apt-get install sqlite3"
|
||||
echo "On macOS: brew install sqlite3"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create database and tables if they don't exist
|
||||
sqlite3 "$DB_FILE" <<EOF
|
||||
CREATE TABLE IF NOT EXISTS downloaded_videos (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
video_id TEXT UNIQUE NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
playlist_name TEXT,
|
||||
download_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
file_path TEXT,
|
||||
file_size INTEGER,
|
||||
status TEXT DEFAULT 'completed'
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS download_sessions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
playlist_name TEXT NOT NULL,
|
||||
playlist_url TEXT NOT NULL,
|
||||
session_start TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
session_end TIMESTAMP,
|
||||
videos_downloaded INTEGER DEFAULT 0,
|
||||
videos_skipped INTEGER DEFAULT 0,
|
||||
errors INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_video_id ON downloaded_videos(video_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_status ON downloaded_videos(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_playlist_name ON downloaded_videos(playlist_name);
|
||||
EOF
|
||||
|
||||
log_info "Database initialized at: $DB_FILE"
|
||||
release_lock() {
|
||||
rmdir "$DOWNLOADS_LOCK" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Check if video is already downloaded
|
||||
is_video_downloaded() {
|
||||
local video_id="$1"
|
||||
local result=$(sqlite3 "$DB_FILE" "SELECT COUNT(*) FROM downloaded_videos WHERE video_id='$video_id' AND status='completed';")
|
||||
[ "$result" -gt 0 ]
|
||||
}
|
||||
is_downloaded() {
|
||||
local video_id="${1:-}"
|
||||
[[ -z "$video_id" ]] && return 1
|
||||
|
||||
# Add video to database
|
||||
add_to_database() {
|
||||
local video_id="$1"
|
||||
local title="$2"
|
||||
local url="$3"
|
||||
local playlist_name="$4"
|
||||
local file_path="$5"
|
||||
local file_size="$6"
|
||||
|
||||
# Escape single quotes in title
|
||||
title="${title//\'/\'\'}"
|
||||
|
||||
sqlite3 "$DB_FILE" <<EOF
|
||||
INSERT OR REPLACE INTO downloaded_videos (video_id, title, url, playlist_name, file_path, file_size, status)
|
||||
VALUES ('$video_id', '$title', '$url', '$playlist_name', '$file_path', $file_size, 'completed');
|
||||
EOF
|
||||
|
||||
log_debug "Added to database: $title (ID: $video_id)"
|
||||
}
|
||||
|
||||
# Get list of all videos in playlist with their IDs
|
||||
get_playlist_videos() {
|
||||
local playlist_url="$1"
|
||||
yt-dlp --quiet --no-warnings --print "%(id)s|%(title)s|%(webpage_url)s" \
|
||||
--flat-playlist "$playlist_url" 2>/dev/null
|
||||
}
|
||||
|
||||
# Start a new download session
|
||||
start_session() {
|
||||
local playlist_name="$1"
|
||||
local playlist_url="$2"
|
||||
|
||||
local session_id=$(sqlite3 "$DB_FILE" \
|
||||
"INSERT INTO download_sessions (playlist_name, playlist_url) VALUES ('$playlist_name', '$playlist_url'); SELECT last_insert_rowid();")
|
||||
echo "$session_id"
|
||||
}
|
||||
|
||||
# Update session with results
|
||||
end_session() {
|
||||
local session_id="$1"
|
||||
local downloaded="$2"
|
||||
local skipped="$3"
|
||||
local errors="$4"
|
||||
|
||||
sqlite3 "$DB_FILE" <<EOF
|
||||
UPDATE download_sessions
|
||||
SET session_end=CURRENT_TIMESTAMP,
|
||||
videos_downloaded=$downloaded,
|
||||
videos_skipped=$skipped,
|
||||
errors=$errors
|
||||
WHERE id=$session_id;
|
||||
EOF
|
||||
}
|
||||
|
||||
# Get database statistics
|
||||
show_stats() {
|
||||
log_info "===== Database Statistics ====="
|
||||
|
||||
local total=$(sqlite3 "$DB_FILE" "SELECT COUNT(*) FROM downloaded_videos WHERE status='completed';")
|
||||
local total_size=$(sqlite3 "$DB_FILE" "SELECT SUM(file_size) FROM downloaded_videos WHERE status='completed';")
|
||||
local last_download=$(sqlite3 "$DB_FILE" "SELECT download_date FROM downloaded_videos WHERE status='completed' ORDER BY download_date DESC LIMIT 1;")
|
||||
local total_sessions=$(sqlite3 "$DB_FILE" "SELECT COUNT(*) FROM download_sessions;")
|
||||
|
||||
echo "Total videos downloaded: $total"
|
||||
echo "Total storage used: $(numfmt --to=iec-i --suffix=B $total_size 2>/dev/null || echo "$total_size bytes")"
|
||||
echo "Last download: $last_download"
|
||||
echo "Total download sessions: $total_sessions"
|
||||
echo ""
|
||||
|
||||
# Show statistics per playlist
|
||||
log_info "===== Per-Playlist Statistics ====="
|
||||
sqlite3 -header -column "$DB_FILE" \
|
||||
"SELECT playlist_name, COUNT(*) as videos, ROUND(SUM(file_size)/1024/1024, 2) as size_mb FROM downloaded_videos WHERE status='completed' GROUP BY playlist_name;"
|
||||
}
|
||||
|
||||
# List recently downloaded videos
|
||||
list_recent() {
|
||||
local limit="${1:-10}"
|
||||
log_info "===== Last $limit Downloads ====="
|
||||
|
||||
sqlite3 -header -column "$DB_FILE" \
|
||||
"SELECT playlist_name, title, download_date FROM downloaded_videos WHERE status='completed' ORDER BY download_date DESC LIMIT $limit;"
|
||||
}
|
||||
|
||||
# Validate inputs
|
||||
validate_setup() {
|
||||
local destination_dir="$1"
|
||||
|
||||
if [ ! -d "$destination_dir" ]; then
|
||||
log_warning "Destination directory does not exist. Creating: $destination_dir"
|
||||
mkdir -p "$destination_dir" || {
|
||||
log_error "Failed to create destination directory: $destination_dir"
|
||||
return 1
|
||||
}
|
||||
fi
|
||||
|
||||
mkdir -p "$DOWNLOAD_DIR" || {
|
||||
log_error "Failed to create download directory: $DOWNLOAD_DIR"
|
||||
acquire_lock || {
|
||||
log_message "WARNING: Could not acquire lock for is_downloaded check"
|
||||
return 1
|
||||
}
|
||||
|
||||
|
||||
grep -q -F -- "${video_id}|" "$DOWNLOADS_LOG"
|
||||
local result=$?
|
||||
|
||||
release_lock
|
||||
return $result
|
||||
}
|
||||
|
||||
record_download() {
|
||||
local video_id="${1:-}"
|
||||
local title="${2:-}"
|
||||
local filename="${3:-}"
|
||||
|
||||
acquire_lock || {
|
||||
log_message "ERROR: Could not acquire lock for recording download"
|
||||
return 1
|
||||
}
|
||||
|
||||
local timestamp
|
||||
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
echo "${video_id}|${title}|${filename}|${timestamp}" >> "$DOWNLOADS_LOG"
|
||||
local result=$?
|
||||
|
||||
release_lock
|
||||
return $result
|
||||
}
|
||||
|
||||
process_downloaded_video() {
|
||||
local video_id="${1:-}"
|
||||
local title="${2:-}"
|
||||
local temp_dir="${3:-}"
|
||||
|
||||
if [[ -z "$video_id" || -z "$temp_dir" ]]; then
|
||||
log_message "ERROR: Missing video_id or temp_dir in process_downloaded_video"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if is_downloaded "$video_id"; then
|
||||
log_message "SKIP: '$title' (already in database)"
|
||||
find "$temp_dir" -maxdepth 1 -type f -name "${video_id}.*" -delete 2>/dev/null
|
||||
return 0
|
||||
fi
|
||||
|
||||
local video_file audio_file
|
||||
video_file=$(find "$temp_dir" -maxdepth 1 -type f -name "${video_id}.video.*" 2>/dev/null | head -n 1)
|
||||
audio_file=$(find "$temp_dir" -maxdepth 1 -type f -name "${video_id}.audio.mp3" 2>/dev/null | head -n 1)
|
||||
|
||||
if [[ -z "${video_file:-}" || ! -f "$video_file" ]]; then
|
||||
log_message "ERROR: Video file not found for '$title' (id: $video_id)"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -z "${audio_file:-}" || ! -f "$audio_file" ]]; then
|
||||
log_message "ERROR: Audio MP3 file not found for '$title' (id: $video_id)"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local clean_title base_name final_video final_audio
|
||||
clean_title=$(sanitize_filename "$title")
|
||||
base_name="$clean_title"
|
||||
|
||||
final_video="$FINAL_DIR/${base_name}.${VIDEO_FORMAT}"
|
||||
final_audio="$FINAL_DIR/${base_name}.mp3"
|
||||
|
||||
if [[ -e "$final_video" || -e "$final_audio" ]]; then
|
||||
base_name="${clean_title}_${video_id}"
|
||||
final_video="$FINAL_DIR/${base_name}.${VIDEO_FORMAT}"
|
||||
final_audio="$FINAL_DIR/${base_name}.mp3"
|
||||
fi
|
||||
|
||||
if mv "$video_file" "$final_video" && mv "$audio_file" "$final_audio"; then
|
||||
if record_download "$video_id" "$title" "${base_name}.${VIDEO_FORMAT} + ${base_name}.mp3"; then
|
||||
log_message "MOVED: '$title' -> '${base_name}.${VIDEO_FORMAT}' and '${base_name}.mp3'"
|
||||
return 0
|
||||
else
|
||||
log_message "ERROR: Failed to record download for '$title'"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
log_message "ERROR: Failed to move files for '$title' to $FINAL_DIR"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
wait_for_next() {
|
||||
local remaining=$SLEEP_INTERVAL
|
||||
while [[ $remaining -gt 0 ]]; do
|
||||
printf "\rWaiting %ss before next download...\033[K" "$remaining"
|
||||
sleep 1
|
||||
((remaining--))
|
||||
done
|
||||
echo
|
||||
}
|
||||
|
||||
format_time() {
|
||||
local seconds=$1
|
||||
local hours=$((seconds / 3600))
|
||||
local minutes=$(((seconds % 3600) / 60))
|
||||
local secs=$((seconds % 60))
|
||||
|
||||
if [[ $hours -gt 0 ]]; then
|
||||
printf "%dh %dm %ds" "$hours" "$minutes" "$secs"
|
||||
elif [[ $minutes -gt 0 ]]; then
|
||||
printf "%dm %ds" "$minutes" "$secs"
|
||||
else
|
||||
printf "%ds" "$secs"
|
||||
fi
|
||||
}
|
||||
|
||||
estimate_download_time() {
|
||||
local playlist_url="${1:-}"
|
||||
[[ -z "$playlist_url" ]] && return 0
|
||||
|
||||
log_message "Estimating playlist size..."
|
||||
local video_count
|
||||
video_count=$(yt-dlp --no-warnings --flat-playlist --print "%(id)s" "$playlist_url" 2>/dev/null \
|
||||
| grep -E '^[a-zA-Z0-9_-]{11}$' | wc -l)
|
||||
|
||||
if [[ $video_count -gt 0 ]]; then
|
||||
local estimated_seconds=$(( (video_count * 300) + (video_count * SLEEP_INTERVAL) ))
|
||||
log_message "Estimated: $video_count videos, approximately $(format_time "$estimated_seconds") total time"
|
||||
fi
|
||||
}
|
||||
|
||||
show_log() {
|
||||
echo
|
||||
echo "=== Downloaded Videos (Last 20) ==="
|
||||
echo
|
||||
tail -n 20 "$DOWNLOADS_LOG" 2>/dev/null | grep -E '^[a-zA-Z0-9_-]{11}\|' | \
|
||||
awk -F'|' '{printf "%-15s | %-50s | %s\n", $1, substr($2,1,50), $4}' || echo "No downloads yet"
|
||||
}
|
||||
|
||||
download_playlist() {
|
||||
local playlist_url="${1:-}"
|
||||
local temp_dir=""
|
||||
local video_list=""
|
||||
local current=0
|
||||
local video_count=0
|
||||
|
||||
if [[ -z "$playlist_url" ]]; then
|
||||
log_message "ERROR: No playlist URL provided"
|
||||
echo "Usage: $0 [youtube_playlist_url]"
|
||||
return 1
|
||||
fi
|
||||
|
||||
cleanup() {
|
||||
[[ -n "${video_list:-}" && -f "$video_list" ]] && rm -f "$video_list"
|
||||
[[ -n "${temp_dir:-}" && -d "$temp_dir" ]] && rm -rf "$temp_dir"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
log_message "Starting download for playlist: $playlist_url"
|
||||
log_message "Settings: Quality=$QUALITY, Rate Limit=$RATE_LIMIT, Sleep Interval=${SLEEP_INTERVAL}s"
|
||||
|
||||
temp_dir=$(mktemp -d) || {
|
||||
log_message "ERROR: Failed to create temporary directory"
|
||||
return 1
|
||||
}
|
||||
|
||||
video_list=$(mktemp) || {
|
||||
log_message "ERROR: Failed to create temporary file"
|
||||
return 1
|
||||
}
|
||||
|
||||
log_message "Fetching playlist information..."
|
||||
|
||||
yt-dlp \
|
||||
--no-warnings \
|
||||
--flat-playlist \
|
||||
--socket-timeout "$CONNECTION_TIMEOUT" \
|
||||
--retries "$MAX_RETRIES" \
|
||||
--print "%(id)s" \
|
||||
"$playlist_url" 2>>"$LOG_FILE" | grep -E '^[a-zA-Z0-9_-]{11}$' > "$video_list"
|
||||
|
||||
if [[ ! -s "$video_list" ]]; then
|
||||
log_message "ERROR: Failed to fetch playlist or playlist is empty"
|
||||
return 1
|
||||
fi
|
||||
|
||||
video_count=$(wc -l < "$video_list")
|
||||
log_message "Found $video_count videos to process"
|
||||
|
||||
while IFS= read -r video_id; do
|
||||
((current++))
|
||||
|
||||
[[ -z "${video_id:-}" ]] && continue
|
||||
|
||||
if ! [[ "$video_id" =~ ^[a-zA-Z0-9_-]{11}$ ]]; then
|
||||
log_message "[$current/$video_count] WARNING: Invalid video ID format: '$video_id'"
|
||||
continue
|
||||
fi
|
||||
|
||||
title=$(yt-dlp \
|
||||
--no-warnings \
|
||||
--skip-download \
|
||||
--socket-timeout "$CONNECTION_TIMEOUT" \
|
||||
--retries "$MAX_RETRIES" \
|
||||
--print "%(title)s" \
|
||||
"https://www.youtube.com/watch?v=$video_id" 2>>"$LOG_FILE" | head -n 1)
|
||||
|
||||
[[ -z "$title" || "$title" == "NA" ]] && title="$video_id"
|
||||
|
||||
if is_downloaded "$video_id"; then
|
||||
log_message "[$current/$video_count] SKIP: '$title' (already downloaded)"
|
||||
continue
|
||||
fi
|
||||
|
||||
log_message "[$current/$video_count] Downloading: '$title'"
|
||||
|
||||
yt-dlp \
|
||||
--no-warnings \
|
||||
--format "$QUALITY" \
|
||||
--merge-output-format "$VIDEO_FORMAT" \
|
||||
--limit-rate "$RATE_LIMIT" \
|
||||
--socket-timeout "$CONNECTION_TIMEOUT" \
|
||||
--retries "$MAX_RETRIES" \
|
||||
--no-continue \
|
||||
--output "$temp_dir/%(id)s.video.%(ext)s" \
|
||||
"https://www.youtube.com/watch?v=$video_id" \
|
||||
2>>"$LOG_FILE"
|
||||
video_status=$?
|
||||
|
||||
yt-dlp \
|
||||
--no-warnings \
|
||||
--extract-audio \
|
||||
--audio-format mp3 \
|
||||
--audio-quality 0 \
|
||||
--socket-timeout "$CONNECTION_TIMEOUT" \
|
||||
--retries "$MAX_RETRIES" \
|
||||
--no-continue \
|
||||
--output "$temp_dir/%(id)s.audio.%(ext)s" \
|
||||
"https://www.youtube.com/watch?v=$video_id" \
|
||||
2>>"$LOG_FILE"
|
||||
audio_status=$?
|
||||
|
||||
if [[ $video_status -eq 0 && $audio_status -eq 0 ]]; then
|
||||
if process_downloaded_video "$video_id" "$title" "$temp_dir"; then
|
||||
log_message "[$current/$video_count] Successfully processed '$title'"
|
||||
[[ $current -lt $video_count ]] && wait_for_next
|
||||
else
|
||||
log_message "[$current/$video_count] ERROR: Processing failed for '$title'"
|
||||
fi
|
||||
else
|
||||
log_message "[$current/$video_count] ERROR: Download failed for '$title' (video: $video_status, audio: $audio_status)"
|
||||
fi
|
||||
done < "$video_list"
|
||||
|
||||
log_message "Playlist download completed"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Main download function for a single playlist
|
||||
download_playlist() {
|
||||
local playlist_name="$1"
|
||||
local playlist_url="$2"
|
||||
local destination_dir="$3"
|
||||
|
||||
log_info "=========================================="
|
||||
log_info "Starting download for: $playlist_name"
|
||||
log_info "Playlist URL: $playlist_url"
|
||||
log_info "Destination: $destination_dir"
|
||||
log_info "=========================================="
|
||||
|
||||
# Validate setup
|
||||
if ! validate_setup "$destination_dir"; then
|
||||
return 1
|
||||
main() {
|
||||
local start_time end_time duration formatted_duration
|
||||
start_time=$(date +%s)
|
||||
|
||||
local playlist_url="${1:-${PLAYLIST_URL:-}}"
|
||||
if [[ -z "$playlist_url" ]]; then
|
||||
echo "ERROR: No playlist URL provided and PLAYLIST_URL is not set in the config"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local session_id=$(start_session "$playlist_name" "$playlist_url")
|
||||
log_debug "Session ID: $session_id"
|
||||
|
||||
local download_count=0
|
||||
local skip_count=0
|
||||
local error_count=0
|
||||
|
||||
# Get all videos in the playlist
|
||||
local playlist_data=$(get_playlist_videos "$playlist_url")
|
||||
|
||||
if [ -z "$playlist_data" ]; then
|
||||
log_error "Failed to fetch playlist. Check URL and internet connection."
|
||||
end_session "$session_id" 0 0 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Process each video
|
||||
while IFS='|' read -r video_id title video_url; do
|
||||
[ -z "$video_id" ] && continue
|
||||
|
||||
if is_video_downloaded "$video_id"; then
|
||||
log_info "Skipping (already downloaded): $title"
|
||||
skip_count=$((skip_count + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
log_info "Downloading: $title (ID: $video_id)"
|
||||
|
||||
# Download the video
|
||||
local temp_file="$DOWNLOAD_DIR/${title}.%(ext)s"
|
||||
|
||||
if yt-dlp \
|
||||
-f "$VIDEO_FORMAT" \
|
||||
-o "$temp_file" \
|
||||
--quiet \
|
||||
--no-warnings \
|
||||
"$video_url" 2>/dev/null; then
|
||||
|
||||
# Find the actual downloaded file
|
||||
local downloaded_file=$(find "$DOWNLOAD_DIR" -name "${title}.*" -type f -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2-)
|
||||
|
||||
if [ -f "$downloaded_file" ]; then
|
||||
local file_size=$(stat -f%z "$downloaded_file" 2>/dev/null || stat -c%s "$downloaded_file" 2>/dev/null)
|
||||
local file_name=$(basename "$downloaded_file")
|
||||
|
||||
# Move to destination
|
||||
if mv "$downloaded_file" "$destination_dir/" 2>/dev/null; then
|
||||
log_info "Moved to destination: $file_name"
|
||||
|
||||
# Add to database
|
||||
add_to_database "$video_id" "$title" "$video_url" "$playlist_name" "$destination_dir/$file_name" "$file_size"
|
||||
|
||||
download_count=$((download_count + 1))
|
||||
else
|
||||
log_error "Failed to move: $file_name"
|
||||
error_count=$((error_count + 1))
|
||||
fi
|
||||
else
|
||||
log_error "Downloaded file not found"
|
||||
error_count=$((error_count + 1))
|
||||
fi
|
||||
else
|
||||
log_error "Download failed: $title"
|
||||
error_count=$((error_count + 1))
|
||||
fi
|
||||
|
||||
# Small delay between downloads to avoid rate limiting
|
||||
sleep "$DOWNLOAD_DELAY"
|
||||
|
||||
done <<< "$playlist_data"
|
||||
|
||||
# Cleanup
|
||||
rm -rf "$DOWNLOAD_DIR" 2>/dev/null
|
||||
|
||||
# End session
|
||||
end_session "$session_id" "$download_count" "$skip_count" "$error_count"
|
||||
|
||||
# Summary
|
||||
echo ""
|
||||
log_info "===== Download Summary for: $playlist_name ====="
|
||||
echo "New videos downloaded: $download_count"
|
||||
echo "Videos skipped (already have): $skip_count"
|
||||
echo "Errors: $error_count"
|
||||
echo ""
|
||||
|
||||
estimate_download_time "$playlist_url"
|
||||
download_playlist "$playlist_url"
|
||||
local result=$?
|
||||
|
||||
show_log
|
||||
|
||||
end_time=$(date +%s)
|
||||
duration=$((end_time - start_time))
|
||||
formatted_duration=$(format_time "$duration")
|
||||
|
||||
log_message "Total execution time: $formatted_duration"
|
||||
log_message "Full downloads database saved to: $DOWNLOADS_LOG"
|
||||
return $result
|
||||
}
|
||||
|
||||
# Download from all enabled playlists
|
||||
download_all_playlists() {
|
||||
log_info "Starting downloads from all enabled playlists..."
|
||||
|
||||
local total_downloaded=0
|
||||
local total_skipped=0
|
||||
local total_errors=0
|
||||
|
||||
get_enabled_playlists | while IFS=$'\t' read -r playlist_name playlist_url destination_dir; do
|
||||
if download_playlist "$playlist_name" "$playlist_url" "$destination_dir"; then
|
||||
# Note: We can't increment variables in subshells, so we'll just log per-playlist
|
||||
:
|
||||
fi
|
||||
done
|
||||
|
||||
log_info "All playlist downloads completed!"
|
||||
}
|
||||
main "$@"
|
||||
|
||||
# Download from a specific playlist
|
||||
download_specific_playlist() {
|
||||
local playlist_name="$1"
|
||||
|
||||
local count=$(get_playlist_count)
|
||||
local found=false
|
||||
|
||||
for ((i=0; i<count; i++)); do
|
||||
local name=$(get_playlist_data "$i" "name")
|
||||
if [ "$name" = "$playlist_name" ]; then
|
||||
local url=$(get_playlist_data "$i" "url")
|
||||
local destination=$(get_playlist_data "$i" "destination")
|
||||
local enabled=$(get_playlist_data "$i" "enabled")
|
||||
|
||||
if [ "$enabled" != "true" ]; then
|
||||
log_warning "Playlist '$playlist_name' is disabled in config"
|
||||
return 1
|
||||
fi
|
||||
|
||||
download_playlist "$name" "$url" "$destination"
|
||||
found=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$found" = false ]; then
|
||||
log_error "Playlist not found: $playlist_name"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# List all playlists
|
||||
list_playlists() {
|
||||
log_info "===== Configured Playlists ====="
|
||||
|
||||
local count=$(get_playlist_count)
|
||||
|
||||
for ((i=0; i<count; i++)); do
|
||||
local name=$(get_playlist_data "$i" "name")
|
||||
local url=$(get_playlist_data "$i" "url")
|
||||
local destination=$(get_playlist_data "$i" "destination")
|
||||
local enabled=$(get_playlist_data "$i" "enabled")
|
||||
|
||||
local status="${GREEN}enabled${NC}"
|
||||
[ "$enabled" != "true" ] && status="${RED}disabled${NC}"
|
||||
|
||||
echo ""
|
||||
echo -e "Name: $name (${status})"
|
||||
echo "URL: $url"
|
||||
echo "Destination: $destination"
|
||||
done
|
||||
}
|
||||
|
||||
# Display help
|
||||
show_help() {
|
||||
cat << EOF
|
||||
YouTube Playlist Downloader with Database
|
||||
|
||||
Usage: $0 [--cron] [COMMAND] [OPTIONS]
|
||||
|
||||
Global Options:
|
||||
--cron Run in cron mode (disables colors, logs to file only)
|
||||
|
||||
Commands:
|
||||
run [PLAYLIST_NAME] Download new videos from specified playlist, or all if not specified
|
||||
list [N] List the last N downloaded videos (default: 10)
|
||||
playlists List all configured playlists
|
||||
stats Show database statistics
|
||||
reset Clear the entire database (WARNING: irreversible)
|
||||
help Show this help message
|
||||
|
||||
Examples:
|
||||
$0 run # Download all enabled playlists
|
||||
$0 run "My First Playlist" # Download specific playlist
|
||||
$0 stats # Show statistics
|
||||
$0 list 20 # Show last 20 downloads
|
||||
$0 --cron run # Run in cron mode
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# Directory paths
|
||||
DOWNLOAD_DIR="/tmp/"
|
||||
FINAL_DIR="/location/to/save/directory"
|
||||
LOG_FILE="/opt/podcast-pownloader/PODCAST_NAME/.youtube_downloader.log"
|
||||
LOG_DIR="$(dirname "$LOG_FILE")"
|
||||
|
||||
DOWNLOADS_LOG="/opt/podcast-pownloader/PODCAST_NAME/downloaded_episodes.log"
|
||||
DOWNLOADS_LOCK="/opt/podcast-pownloader/PODCAST_NAME/.podcast_downloads.lock"
|
||||
MAX_LOG_SIZE=$((10 * 1024 * 1024))
|
||||
|
||||
# Quality and format settings
|
||||
QUALITY="137+140"
|
||||
AUDIO_FORMAT="m4a"
|
||||
VIDEO_FORMAT="mp4"
|
||||
|
||||
# Download behavior
|
||||
RATE_LIMIT="3M"
|
||||
SLEEP_INTERVAL=120
|
||||
MAX_RETRIES=3
|
||||
CONNECTION_TIMEOUT=30
|
||||
|
||||
# Playlist
|
||||
PLAYLIST_URL="https://www.youtube.com/playlist?list=URL"
|
||||
@@ -1,24 +0,0 @@
|
||||
{
|
||||
"general": {
|
||||
"db_dir": "$HOME/.youtube_downloader",
|
||||
"temp_download_dir": "/tmp/youtube_downloads",
|
||||
"video_format": "best[ext=mp4]",
|
||||
"output_template": "%(title)s.%(ext)s",
|
||||
"audio_only": false,
|
||||
"download_delay": 2
|
||||
},
|
||||
"playlists": [
|
||||
{
|
||||
"name": "My First Playlist",
|
||||
"url": "https://www.youtube.com/playlist?list=PLAYLIST_ID_1",
|
||||
"destination": "/path/to/first/location",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"name": "My Second Playlist",
|
||||
"url": "https://www.youtube.com/playlist?list=PLAYLIST_ID_2",
|
||||
"destination": "/path/to/second/location",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user