diff --git a/README.md b/README.md index 99cf5c9..e7fdc91 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,539 @@ # Podcast-Downloader -A script to check for podcasts, download them and move them to a specified folder \ No newline at end of file + +A **bash script** that automatically downloads videos from YouTube playlists using `yt-dlp`, maintains a local SQLite database of downloaded content, and prevents duplicate downloads. Supports multiple playlists with individual destination folders and **cron job integration** for automated scheduling. + +--- + +## Features + +| Feature | Description | +|---------|-------------| +| **Multi-Playlist Support** | Configure and download from multiple YouTube playlists simultaneously | +| **Duplicate Prevention** | SQLite database tracks all downloaded videos to prevent re-downloading | +| **Individual Destinations** | Assign a different destination folder for each playlist | +| **Cron-Compatible** | Run automatically on a schedule with proper logging | +| **Download Sessions** | Detailed statistics on each download run (downloaded, skipped, errors) | +| **Database Statistics** | View total videos downloaded, storage used, and per-playlist breakdowns | +| **Configurable Format** | Specify video format, download delays, and audio-only mode via config | +| **Automatic Cleanup** | Removes temporary download folders after successful transfers | +| **Colored Output** | Terminal-friendly logging (disabled in cron mode) | + +--- + +## Prerequisites + +Before using this script, ensure you have the following tools installed: + +### Required + +- **bash** (version 4.0 or higher) +- **yt-dlp** – YouTube video downloader ([install](https://github.com/yt-dlp/yt-dlp#installation)) +- **sqlite3** – Database management +- **jq** – JSON parser for config file handling + +### Installation + +**Ubuntu/Debian:** +```bash +sudo apt-get update +sudo apt-get install yt-dlp sqlite3 jq +``` + +**macOS:** +```bash +brew install yt-dlp sqlite3 jq +``` + +**Fedora/RHEL:** +```bash +sudo dnf install yt-dlp sqlite3 jq +``` + +--- + +## Installation + +### 1. Clone or Download the Script + +```bash +mkdir -p ~/.youtube_downloader +cd ~/.youtube_downloader +# Download or clone the script here +chmod +x youtube_downloader.sh +``` + +### 2. Create Configuration File + +Create the config file at `~/.config/youtube_downloader/config.json`: + +```bash +mkdir -p ~/.config/youtube_downloader +``` + +Then create `~/.config/youtube_downloader/config.json` with the following structure: + +```json +{ + "general": { + "db_dir": "$HOME/.youtube_downloader", + "temp_download_dir": "/tmp/youtube_downloads", + "video_format": "best[ext=mp4]", + "output_template": "%(title)s.%(ext)s", + "audio_only": false, + "download_delay": 2, + "debug": false + }, + "playlists": [ + { + "name": "My Music Playlist", + "url": "https://www.youtube.com/playlist?list=PLAYLIST_ID_1", + "destination": "/home/user/Music/YouTubeDownloads", + "enabled": true + }, + { + "name": "Tutorial Playlist", + "url": "https://www.youtube.com/playlist?list=PLAYLIST_ID_2", + "destination": "/home/user/Videos/Tutorials", + "enabled": true + }, + { + "name": "Disabled Playlist", + "url": "https://www.youtube.com/playlist?list=PLAYLIST_ID_3", + "destination": "/home/user/Videos/Archive", + "enabled": false + } + ] +} +``` + +### 3. Find Your Playlist IDs + +To get a playlist ID: +1. Go to a YouTube playlist +2. Look at the URL: `https://www.youtube.com/playlist?list=PLAYLIST_ID_HERE` +3. Copy the part after `list=` + +--- + +## Configuration Guide + +### General Settings + +| Setting | Description | Example | +|---------|-------------|---------| +| `db_dir` | Directory for database and logs | `$HOME/.youtube_downloader` | +| `temp_download_dir` | Temporary folder for downloads (cleaned up after) | `/tmp/youtube_downloads` | +| `video_format` | yt-dlp format specification | `best[ext=mp4]` or `worst` | +| `output_template` | Filename pattern (yt-dlp variables) | `%(title)s.%(ext)s` | +| `audio_only` | Download audio only (ignores video_format) | `true` or `false` | +| `download_delay` | Seconds between downloads (avoids rate-limiting) | `2` | +| `debug` | Enable debug logging | `true` or `false` | + +### Video Format Examples + +```json +"best[ext=mp4]" // Best quality MP4 +"worst" // Lowest quality (smallest file) +"bestvideo+bestaudio" // Best video + best audio (merged) +"bestvideo[height<=720]" // Max 720p +``` + +### Playlist Settings + +| Setting | Description | Example | +|---------|-------------|---------| +| `name` | Display name for the playlist | `"My Music Playlist"` | +| `url` | Full YouTube playlist URL | `https://www.youtube.com/playlist?list=...` | +| `destination` | Where videos are saved | `/home/user/Music/YouTubeDownloads` | +| `enabled` | Enable/disable this playlist | `true` or `false` | + +--- + +## Usage + +### Interactive Usage + +```bash +# Download all enabled playlists +./youtube_downloader.sh run + +# Download a specific playlist +./youtube_downloader.sh run "My Music Playlist" + +# View statistics +./youtube_downloader.sh stats + +# List last 20 downloaded videos +./youtube_downloader.sh list 20 + +# Show all configured playlists +./youtube_downloader.sh playlists + +# View help +./youtube_downloader.sh help + +# Reset database (WARNING: deletes all download history) +./youtube_downloader.sh reset +``` + +### Cron Job Usage + +Run downloads automatically on a schedule. Use `--cron` flag to disable colors and log only to file. + +#### Example Cron Jobs + +**Download all playlists daily at 2 AM:** +```bash +0 2 * * * /home/user/.youtube_downloader/youtube_downloader.sh --cron run >> /home/user/.youtube_downloader/cron.log 2>&1 +``` + +**Download specific playlist every 6 hours:** +```bash +0 */6 * * * /home/user/.youtube_downloader/youtube_downloader.sh --cron run "My Music Playlist" +``` + +**Download all playlists every Sunday at midnight:** +```bash +0 0 * * 0 /home/user/.youtube_downloader/youtube_downloader.sh --cron run +``` + +#### Setting Up Cron + +1. **Open cron editor:** + ```bash + crontab -e + ``` + +2. **Add a cron job** (example: daily at 2 AM): + ```bash + 0 2 * * * /home/user/.youtube_downloader/youtube_downloader.sh --cron run + ``` + +3. **Save and exit** (in nano: `Ctrl+X`, then `Y`, then `Enter`) + +4. **Verify cron job:** + ```bash + crontab -l + ``` + +#### Important Notes for Cron + +- **Use absolute paths** – cron doesn't have the same environment as your shell +- **Specify full path** to the script: `/home/user/.youtube_downloader/youtube_downloader.sh` +- **Colors are disabled** automatically in cron mode (detected via `--cron` flag) +- **All output logged** to `$HOME/.youtube_downloader/downloader.log` +- **Ensure permissions** – make the script executable: `chmod +x youtube_downloader.sh` + +--- + +## Database + +The script maintains a **SQLite database** at `~/.youtube_downloader/downloads.db` with the following tables: + +### `downloaded_videos` + +Tracks all downloaded videos to prevent duplicates. + +| Column | Type | Purpose | +|--------|------|---------| +| `id` | INTEGER | Unique record ID | +| `video_id` | TEXT | YouTube video ID (unique) | +| `title` | TEXT | Video title | +| `url` | TEXT | Full YouTube URL | +| `playlist_name` | TEXT | Which playlist it came from | +| `download_date` | TIMESTAMP | When it was downloaded | +| `file_path` | TEXT | Full path to saved file | +| `file_size` | INTEGER | File size in bytes | +| `status` | TEXT | Status (always `'completed'` for valid downloads) | + +### `download_sessions` + +Logs each download run with statistics. + +| Column | Type | Purpose | +|--------|------|---------| +| `id` | INTEGER | Session ID | +| `playlist_name` | TEXT | Which playlist was downloaded | +| `playlist_url` | TEXT | Playlist URL | +| `session_start` | TIMESTAMP | When the session started | +| `session_end` | TIMESTAMP | When the session finished | +| `videos_downloaded` | INTEGER | Count of newly downloaded videos | +| `videos_skipped` | INTEGER | Count of already-downloaded videos | +| `errors` | INTEGER | Count of download failures | + +--- + +## Logs + +All activity is logged to **`~/.youtube_downloader/downloader.log`** + +### Log Levels + +- **[INFO]** – General information (downloads, moves, session start/end) +- **[ERROR]** – Critical issues (failed downloads, missing directories) +- **[WARNING]** – Non-critical issues (missing destinations, disabled playlists) +- **[DEBUG]** – Detailed diagnostic info (only if `debug: true` in config) + +### Viewing Logs + +```bash +# View entire log +cat ~/.youtube_downloader/downloader.log + +# View last 50 lines +tail -50 ~/.youtube_downloader/downloader.log + +# Follow logs in real-time (while script is running) +tail -f ~/.youtube_downloader/downloader.log + +# View errors only +grep "ERROR" ~/.youtube_downloader/downloader.log +``` + +--- + +## Common Tasks + +### Add a New Playlist + +1. Get the playlist URL from YouTube +2. Edit `~/.config/youtube_downloader/config.json` +3. Add a new object to the `playlists` array: + +```json +{ + "name": "New Playlist Name", + "url": "https://www.youtube.com/playlist?list=PLAYLIST_ID", + "destination": "/path/to/save/videos", + "enabled": true +} +``` + +4. Save and run: `./youtube_downloader.sh run "New Playlist Name"` + +### Disable a Playlist Temporarily + +Set `"enabled": false` for that playlist in `config.json`. The playlist will be skipped during automatic runs but can still be downloaded manually by name. + +### Change Download Location + +Edit the `destination` field for a playlist in `config.json`. New videos will go to the new location; old videos remain where they were. + +### Download Audio Only + +In `config.json`, set: +```json +"audio_only": true, +"video_format": "best" +``` + +The script will extract audio only and save as MP3/M4A. + +### Reduce File Sizes + +Change `video_format` to download lower quality: +```json +"video_format": "worst[ext=mp4]" +``` + +Or cap at 720p: +```json +"video_format": "bestvideo[height<=720]+bestaudio" +``` + +### Clear Download History + +**WARNING: This is irreversible!** + +```bash +./youtube_downloader.sh reset +``` + +Then confirm with `yes`. + +--- + +## Troubleshooting + +### Issue: "Config file not found" + +**Solution:** Ensure `~/.config/youtube_downloader/config.json` exists and is readable: + +```bash +ls -la ~/.config/youtube_downloader/config.json +``` + +Create it if missing (see [Configuration Guide](#configuration-guide)). + +--- + +### Issue: "jq is not installed" + +**Solution:** Install jq: + +```bash +# Ubuntu/Debian +sudo apt-get install jq + +# macOS +brew install jq + +# Fedora +sudo dnf install jq +``` + +--- + +### Issue: "yt-dlp is not installed" + +**Solution:** Install yt-dlp: + +```bash +# Ubuntu/Debian +sudo apt-get install yt-dlp + +# macOS +brew install yt-dlp + +# Or via pip (any OS) +pip install yt-dlp +``` + +--- + +### Issue: "Failed to fetch playlist" + +**Possible causes:** +- **Invalid playlist URL** – Double-check the URL format +- **Private/restricted playlist** – YouTube playlists must be public +- **Network issue** – Check internet connection +- **Rate-limited** – Try again later + +**Check logs:** +```bash +tail -20 ~/.youtube_downloader/downloader.log +``` + +--- + +### Issue: Cron job not running + +**Debug steps:** + +1. **Check cron is enabled:** + ```bash + sudo systemctl status cron + ``` + +2. **Verify cron job exists:** + ```bash + crontab -l + ``` + +3. **Check system mail for errors:** + ```bash + mail + ``` + +4. **Test the script manually:** + ```bash + /home/user/.youtube_downloader/youtube_downloader.sh --cron run + ``` + +5. **Add logging to cron job:** + ```bash + 0 2 * * * /home/user/.youtube_downloader/youtube_downloader.sh --cron run >> /home/user/.youtube_downloader/cron.log 2>&1 + ``` + +6. **Check permissions:** + ```bash + chmod +x /home/user/.youtube_downloader/youtube_downloader.sh + ``` + +--- + +### Issue: "Permission denied" when running script + +**Solution:** Make the script executable: + +```bash +chmod +x ~/.youtube_downloader/youtube_downloader.sh +``` + +--- + +### Issue: Videos not moving to destination folder + +**Possible causes:** +- **Destination folder doesn't exist** – Script will create it automatically +- **Insufficient disk space** – Check available space +- **File permissions** – Ensure write access to destination folder + +**Check permissions:** +```bash +ls -la /path/to/destination/ +``` + +--- + +## Performance Tips + +1. **Increase download delay** if hitting YouTube rate limits: + ```json + "download_delay": 5 + ``` + +2. **Lower video quality** to save bandwidth: + ```json + "video_format": "worst[ext=mp4]" + ``` + +3. **Use audio-only mode** if you only need audio: + ```json + "audio_only": true + ``` + +4. **Schedule cron jobs during off-peak hours** to avoid network congestion + +5. **Monitor database size** – it grows with each download: + ```bash + du -sh ~/.youtube_downloader/downloads.db + ``` + +--- + +## File Structure + +``` +~/Podcast-Pownloader/ +├── podcast-downloader.sh # Main script +├── downloads.db # SQLite database +├── downloader.log # Activity log +├── cron.log # Cron job output (optional) +└── podcasts.json # Configuration file +``` + +--- + +## License + +This project is provided as-is for personal use. Please respect YouTube's Terms of Service and copyright laws when downloading content. + +--- + +## Contributing + +Found a bug or have a feature request? Feel free to submit issues or improvements! + +--- + +## Disclaimer + +**This tool is for personal use only.** Users are responsible for: +- Complying with YouTube's Terms of Service +- Respecting copyright and intellectual property rights +- Not using the tool to circumvent any protections or restrictions +- Ensuring all downloads comply with local laws + +The authors assume no liability for misuse of this tool. diff --git a/podcast-downloader.sh b/podcast-downloader.sh new file mode 100644 index 0000000..f3055d8 --- /dev/null +++ b/podcast-downloader.sh @@ -0,0 +1,426 @@ +#!/bin/bash + +# YouTube Playlist Downloader with yt-dlp and Database +# Downloads videos from multiple playlists, maintains a DB of downloaded videos, +# and automatically skips already-downloaded content + +# Color output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +# Configuration file path +CONFIG_FILE="${XDG_CONFIG_HOME:-$HOME/.config}/youtube_downloader/config.json" +DEFAULT_CONFIG_DIR="$HOME/.youtube_downloader" + +# Function to log messages +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" | tee -a "$LOG_FILE" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" | tee -a "$LOG_FILE" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" | tee -a "$LOG_FILE" +} + +log_debug() { + echo -e "${BLUE}[DEBUG]${NC} $1" | tee -a "$LOG_FILE" +} + +# Load configuration from JSON file +load_config() { + if [ ! -f "$CONFIG_FILE" ]; then + log_error "Config file not found: $CONFIG_FILE" + echo "Please create the config file first. You can copy the example from the script comments." + exit 1 + fi + + # Check if jq is installed + if ! command -v jq &> /dev/null; then + log_error "jq is not installed. Please install it first." + echo "On Ubuntu/Debian: sudo apt-get install jq" + echo "On macOS: brew install jq" + exit 1 + fi + + # Parse configuration + DB_DIR=$(jq -r '.general.db_dir' "$CONFIG_FILE" | sed "s|\$HOME|$HOME|g") + DOWNLOAD_DIR=$(jq -r '.general.temp_download_dir' "$CONFIG_FILE" | sed "s|\$HOME|$HOME|g") + VIDEO_FORMAT=$(jq -r '.general.video_format' "$CONFIG_FILE") + OUTPUT_TEMPLATE=$(jq -r '.general.output_template' "$CONFIG_FILE") + AUDIO_ONLY=$(jq -r '.general.audio_only' "$CONFIG_FILE") + DOWNLOAD_DELAY=$(jq -r '.general.download_delay' "$CONFIG_FILE") + + # Set derived paths + DB_FILE="$DB_DIR/downloads.db" + LOG_FILE="$DB_DIR/downloader.log" + + log_debug "Configuration loaded from: $CONFIG_FILE" +} + +# Get playlist count +get_playlist_count() { + jq -r '.playlists | length' "$CONFIG_FILE" +} + +# Get specific playlist data +get_playlist_data() { + local index="$1" + local field="$2" + jq -r ".playlists[$index].$field" "$CONFIG_FILE" +} + +# Get all enabled playlists +get_enabled_playlists() { + jq -r '.playlists[] | select(.enabled == true) | [.name, .url, .destination] | @tsv' "$CONFIG_FILE" +} + +# Initialize database +init_database() { + # Create directory if it doesn't exist + mkdir -p "$DB_DIR" || { + log_error "Failed to create database directory: $DB_DIR" + exit 1 + } + + # Check if sqlite3 is installed + if ! command -v sqlite3 &> /dev/null; then + log_error "sqlite3 is not installed. Please install it first." + echo "On Ubuntu/Debian: sudo apt-get install sqlite3" + echo "On macOS: brew install sqlite3" + exit 1 + fi + + # Create database and tables if they don't exist + sqlite3 "$DB_FILE" </dev/null +} + +# Start a new download session +start_session() { + local playlist_name="$1" + local playlist_url="$2" + + local session_id=$(sqlite3 "$DB_FILE" \ + "INSERT INTO download_sessions (playlist_name, playlist_url) VALUES ('$playlist_name', '$playlist_url'); SELECT last_insert_rowid();") + echo "$session_id" +} + +# Update session with results +end_session() { + local session_id="$1" + local downloaded="$2" + local skipped="$3" + local errors="$4" + + sqlite3 "$DB_FILE" </dev/null || echo "$total_size bytes")" + echo "Last download: $last_download" + echo "Total download sessions: $total_sessions" + echo "" + + # Show statistics per playlist + log_info "===== Per-Playlist Statistics =====" + sqlite3 -header -column "$DB_FILE" \ + "SELECT playlist_name, COUNT(*) as videos, ROUND(SUM(file_size)/1024/1024, 2) as size_mb FROM downloaded_videos WHERE status='completed' GROUP BY playlist_name;" +} + +# List recently downloaded videos +list_recent() { + local limit="${1:-10}" + log_info "===== Last $limit Downloads =====" + + sqlite3 -header -column "$DB_FILE" \ + "SELECT playlist_name, title, download_date FROM downloaded_videos WHERE status='completed' ORDER BY download_date DESC LIMIT $limit;" +} + +# Validate inputs +validate_setup() { + local destination_dir="$1" + + if [ ! -d "$destination_dir" ]; then + log_warning "Destination directory does not exist. Creating: $destination_dir" + mkdir -p "$destination_dir" || { + log_error "Failed to create destination directory: $destination_dir" + return 1 + } + fi + + mkdir -p "$DOWNLOAD_DIR" || { + log_error "Failed to create download directory: $DOWNLOAD_DIR" + return 1 + } + + return 0 +} + +# Main download function for a single playlist +download_playlist() { + local playlist_name="$1" + local playlist_url="$2" + local destination_dir="$3" + + log_info "==========================================" + log_info "Starting download for: $playlist_name" + log_info "Playlist URL: $playlist_url" + log_info "Destination: $destination_dir" + log_info "==========================================" + + # Validate setup + if ! validate_setup "$destination_dir"; then + return 1 + fi + + local session_id=$(start_session "$playlist_name" "$playlist_url") + log_debug "Session ID: $session_id" + + local download_count=0 + local skip_count=0 + local error_count=0 + + # Get all videos in the playlist + local playlist_data=$(get_playlist_videos "$playlist_url") + + if [ -z "$playlist_data" ]; then + log_error "Failed to fetch playlist. Check URL and internet connection." + end_session "$session_id" 0 0 1 + return 1 + fi + + # Process each video + while IFS='|' read -r video_id title video_url; do + [ -z "$video_id" ] && continue + + if is_video_downloaded "$video_id"; then + log_info "Skipping (already downloaded): $title" + skip_count=$((skip_count + 1)) + continue + fi + + log_info "Downloading: $title (ID: $video_id)" + + # Download the video + local temp_file="$DOWNLOAD_DIR/${title}.%(ext)s" + + if yt-dlp \ + -f "$VIDEO_FORMAT" \ + -o "$temp_file" \ + --quiet \ + --no-warnings \ + "$video_url" 2>/dev/null; then + + # Find the actual downloaded file + local downloaded_file=$(find "$DOWNLOAD_DIR" -name "${title}.*" -type f -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2-) + + if [ -f "$downloaded_file" ]; then + local file_size=$(stat -f%z "$downloaded_file" 2>/dev/null || stat -c%s "$downloaded_file" 2>/dev/null) + local file_name=$(basename "$downloaded_file") + + # Move to destination + if mv "$downloaded_file" "$destination_dir/" 2>/dev/null; then + log_info "Moved to destination: $file_name" + + # Add to database + add_to_database "$video_id" "$title" "$video_url" "$playlist_name" "$destination_dir/$file_name" "$file_size" + + download_count=$((download_count + 1)) + else + log_error "Failed to move: $file_name" + error_count=$((error_count + 1)) + fi + else + log_error "Downloaded file not found" + error_count=$((error_count + 1)) + fi + else + log_error "Download failed: $title" + error_count=$((error_count + 1)) + fi + + # Small delay between downloads to avoid rate limiting + sleep "$DOWNLOAD_DELAY" + + done <<< "$playlist_data" + + # Cleanup + rm -rf "$DOWNLOAD_DIR" 2>/dev/null + + # End session + end_session "$session_id" "$download_count" "$skip_count" "$error_count" + + # Summary + echo "" + log_info "===== Download Summary for: $playlist_name =====" + echo "New videos downloaded: $download_count" + echo "Videos skipped (already have): $skip_count" + echo "Errors: $error_count" + echo "" +} + +# Download from all enabled playlists +download_all_playlists() { + log_info "Starting downloads from all enabled playlists..." + + local total_downloaded=0 + local total_skipped=0 + local total_errors=0 + + get_enabled_playlists | while IFS=$'\t' read -r playlist_name playlist_url destination_dir; do + if download_playlist "$playlist_name" "$playlist_url" "$destination_dir"; then + # Note: We can't increment variables in subshells, so we'll just log per-playlist + : + fi + done + + log_info "All playlist downloads completed!" +} + +# Download from a specific playlist +download_specific_playlist() { + local playlist_name="$1" + + local count=$(get_playlist_count) + local found=false + + for ((i=0; i