pve-backup-watcher/main.go

// proxmox-backup-watch
//
// Waits for the vzdump task of the given VMID(s) to finish, then restarts the
// guest (QEMU VM or LXC container). By default it restarts regardless of the
// backup result; use -only-failed to restart only when the backup did not
// finish OK.
//
// Usage: proxmox-backup-watch [options] VMID [VMID...]
package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

// Set at build time with -ldflags (see Makefile).
var (
	version = "dev"
	commit  = "none"
	date    = "unknown"
)

type task struct {
	ID        string `json:"id"`
	StartTime int64  `json:"starttime"`
	EndTime   *int64 `json:"endtime"`
	Status    string `json:"status"`
	UPID      string `json:"upid"`
}

type config struct {
	node       string
	dryRun     bool
	timeout    time.Duration
	interval   time.Duration
	onlyFailed bool
	warnOK     bool
	startedAt  int64
}

func main() {
	os.Exit(run())
}

func run() int {
	cfg := config{startedAt: time.Now().Unix()}

	fs := flag.NewFlagSet("proxmox-backup-watch", flag.ContinueOnError)
	fs.BoolVar(&cfg.dryRun, "dry-run", false, "show what would be done, don't restart anything")
	fs.BoolVar(&cfg.dryRun, "n", false, "shorthand for -dry-run")
	fs.DurationVar(&cfg.timeout, "timeout", 6*time.Hour, "max time to wait per VMID (0 = forever)")
	fs.DurationVar(&cfg.timeout, "t", 6*time.Hour, "shorthand for -timeout")
	fs.DurationVar(&cfg.interval, "interval", 30*time.Second, "poll interval")
	fs.DurationVar(&cfg.interval, "i", 30*time.Second, "shorthand for -interval")
	fs.BoolVar(&cfg.onlyFailed, "only-failed", false, "restart only if the backup did NOT finish OK")
	fs.BoolVar(&cfg.onlyFailed, "f", false, "shorthand for -only-failed")
	fs.BoolVar(&cfg.warnOK, "warn-ok", false, `with -only-failed, treat "WARNINGS: n" as success`)
	fs.BoolVar(&cfg.warnOK, "w", false, "shorthand for -warn-ok")
	fs.StringVar(&cfg.node, "node", "", "Proxmox node name (default: short hostname)")
	var showVersion bool
	fs.BoolVar(&showVersion, "version", false, "print version and exit")
	fs.BoolVar(&showVersion, "v", false, "shorthand for -version")
	fs.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: proxmox-backup-watch [options] VMID [VMID...]")
		fs.PrintDefaults()
	}

	// Allow flags before, between and after VMIDs.
	var vmids []string
	args := os.Args[1:]
	for {
		if err := fs.Parse(args); err != nil {
			if errors.Is(err, flag.ErrHelp) {
				return 0
			}
			return 2
		}
		rest := fs.Args()
		if len(rest) == 0 {
			break
		}
		vmids = append(vmids, rest[0])
		args = rest[1:]
	}

	if showVersion {
		fmt.Printf("proxmox-backup-watch %s (commit %s, built %s)\n", version, commit, date)
		return 0
	}

	if len(vmids) == 0 {
		fmt.Fprintln(os.Stderr, "Error: at least one VMID is required")
		fs.Usage()
		return 2
	}
	for _, id := range vmids {
		if !isNumeric(id) {
			fmt.Fprintf(os.Stderr, "Error: invalid VMID %q\n", id)
			return 2
		}
	}

	if cfg.node == "" {
		h, err := os.Hostname()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error: cannot determine hostname: %v\n", err)
			return 1
		}
		cfg.node, _, _ = strings.Cut(h, ".")
	}

	fmt.Printf("proxmox-backup-watch %s\n", version)

	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()

	failed := false
	for _, vmid := range vmids {
		if err := handle(ctx, &cfg, vmid); err != nil {
			fmt.Fprintf(os.Stderr, "VM %s: %v\n", vmid, err)
			failed = true
			if ctx.Err() != nil {
				break
			}
		}
	}
	if failed {
		return 1
	}
	return 0
}

func handle(ctx context.Context, cfg *config, vmid string) error {
	fmt.Printf("VM %s: waiting for backup task to finish...\n", vmid)

	t, err := waitForTask(ctx, cfg, vmid)
	if err != nil {
		return err
	}

	if cfg.onlyFailed {
		ok := t.Status == "OK" || (cfg.warnOK && strings.HasPrefix(t.Status, "WARNINGS"))
		if ok {
			fmt.Printf("VM %s: backup finished (%s), no restart needed\n", vmid, t.Status)
			return nil
		}
	}

	fmt.Printf("VM %s: backup finished (%s) -> restarting\n", vmid, t.Status)
	return restartGuest(ctx, cfg, vmid)
}

// waitForTask polls until the newest vzdump task for vmid (running now, or
// finished after this program started) has an end time.
func waitForTask(ctx context.Context, cfg *config, vmid string) (*task, error) {
	var deadline time.Time
	if cfg.timeout > 0 {
		deadline = time.Now().Add(cfg.timeout)
	}

	for {
		t, err := latestTask(ctx, cfg, vmid)
		if err != nil {
			// Transient pvesh/JSON errors shouldn't abort a long wait.
			fmt.Fprintf(os.Stderr, "VM %s: warning: %v\n", vmid, err)
		} else if t != nil && t.EndTime != nil {
			return t, nil
		}

		if !deadline.IsZero() && time.Now().After(deadline) {
			return nil, fmt.Errorf("timed out after %s, no restart", cfg.timeout)
		}

		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case <-time.After(cfg.interval):
		}
	}
}

func latestTask(ctx context.Context, cfg *config, vmid string) (*task, error) {
	out, err := output(ctx, "pvesh", "get", "/nodes/"+cfg.node+"/tasks",
		"--typefilter", "vzdump", "--limit", "500", "--output-format", "json")
	if err != nil {
		return nil, err
	}

	var tasks []task
	if err := json.Unmarshal(out, &tasks); err != nil {
		return nil, fmt.Errorf("parsing pvesh output: %w", err)
	}

	var latest *task
	for i := range tasks {
		t := &tasks[i]
		if t.ID != vmid {
			continue
		}
		// Running (no endtime) or finished after we started.
		if t.EndTime != nil && *t.EndTime < cfg.startedAt {
			continue
		}
		if latest == nil || t.StartTime > latest.StartTime {
			latest = t
		}
	}
	return latest, nil
}

func restartGuest(ctx context.Context, cfg *config, vmid string) error {
	var tool string
	switch {
	case exec.CommandContext(ctx, "qm", "status", vmid).Run() == nil:
		tool = "qm"
	case exec.CommandContext(ctx, "pct", "status", vmid).Run() == nil:
		tool = "pct"
	default:
		return errors.New("not found on this node")
	}

	// Clear a leftover 'backup' lock; failure here is not fatal.
	_ = runCmd(ctx, cfg, tool, "unlock", vmid)

	if err := runCmd(ctx, cfg, tool, "shutdown", vmid, "--timeout", "120", "--forceStop", "1"); err != nil {
		return fmt.Errorf("shutdown failed: %w", err)
	}
	if err := runCmd(ctx, cfg, tool, "start", vmid); err != nil {
		return fmt.Errorf("start failed: %w", err)
	}
	return nil
}

func runCmd(ctx context.Context, cfg *config, name string, args ...string) error {
	if cfg.dryRun {
		fmt.Printf("[dry-run] %s %s\n", name, strings.Join(args, " "))
		return nil
	}
	cmd := exec.CommandContext(ctx, name, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func output(ctx context.Context, name string, args ...string) ([]byte, error) {
	var stdout, stderr bytes.Buffer
	cmd := exec.CommandContext(ctx, name, args...)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		return nil, fmt.Errorf("%s: %w: %s", name, err, strings.TrimSpace(stderr.String()))
	}
	return stdout.Bytes(), nil
}

func isNumeric(s string) bool {
	if s == "" {
		return false
	}
	for _, r := range s {
		if r < '0' || r > '9' {
			return false
		}
	}
	return true
}