package main import ( "bufio" "bytes" "context" "encoding/json" "errors" "fmt" "os" "os/exec" "os/signal" "path/filepath" "regexp" "sort" "strconv" "strings" "time" ) const ( version = "V0.4.0" logPath = "/var/log/proxmox-toolkit.log" defaultTimeout = 2 * time.Minute actionTimeout = 10 * time.Minute ) var ( tmpDir string nodeName string // the node this program runs on sigCh = make(chan os.Signal, 1) // PVE guest IDs are 100..999999999. numeric = regexp.MustCompile(`^[1-9][0-9]{2,8}$`) // Snapshot names: start with a letter, 2-40 chars, letters/digits/_/-. snapName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{1,39}$`) // Storage IDs must not start with '-' so they can't be parsed as options. storageName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*$`) nodeRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9-]*$`) upidRe = regexp.MustCompile(`^UPID:[A-Za-z0-9:@!._-]+$`) ) type guestKind struct { typ string // API type: qemu / lxc label string // VM / container title string } var ( kindVM = guestKind{"qemu", "VM", "Virtual Machines (QEMU)"} kindCT = guestKind{"lxc", "container", "Containers (LXC)"} ) func kindOf(typ string) guestKind { if typ == "lxc" { return kindCT } return kindVM } type guest struct { ID int Name string Status string Node string Type string // qemu / lxc Pool string Template bool } func (g guest) path(suffix string) string { return fmt.Sprintf("/nodes/%s/%s/%d%s", g.Node, g.Type, g.ID, suffix) } func (g guest) short() string { tag := "VM" if g.Type == "lxc" { tag = "CT" } return fmt.Sprintf("%d %s [%s] on %s", g.ID, g.Name, tag, g.Node) } // ---------------------------------------------------------------- main func main() { if os.Geteuid() != 0 { fmt.Fprintf(os.Stderr, "Run this program as root, for example: sudo %s\n", os.Args[0]) os.Exit(1) } if _, err := exec.LookPath("whiptail"); err != nil { fmt.Fprintln(os.Stderr, "whiptail is required. Install it with: apt install whiptail") os.Exit(1) } var err error tmpDir, err = os.MkdirTemp("", "proxmox-toolkit-*") if err != nil { fmt.Fprintf(os.Stderr, "Could not create temporary directory: %v\n", err) os.Exit(1) } defer os.RemoveAll(tmpDir) // Swallow Ctrl-C in the parent so cleanup still runs; followTask uses it // to stop following a task. signal.Notify(sigCh, os.Interrupt) nodeName = detectNode() logAction("toolkit %s started on node %s", version, nodeName) message( "Development software - do not use in production", "This toolkit is still in development and has not been validated for production use.\n\n"+ "Use only in a test environment. Actions may affect VMs, containers, storage, or cluster state "+ "on ANY node in the cluster.\n\n"+ "Actions are logged to "+logPath+".", ) mainMenu() logAction("toolkit exited") } func detectNode() string { h, err := os.Hostname() if err != nil || h == "" { return "localhost" } if i := strings.IndexByte(h, '.'); i >= 0 { h = h[:i] } return h } func logAction(format string, a ...any) { f, err := os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) if err != nil { return } defer f.Close() fmt.Fprintf(f, "%s %s\n", time.Now().Format(time.RFC3339), fmt.Sprintf(format, a...)) } // ---------------------------------------------------------------- command helpers func run(timeout time.Duration, name string, args ...string) (string, int, error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() out, err := exec.CommandContext(ctx, name, args...).CombinedOutput() status := 0 if err != nil { status = -1 var exitErr *exec.ExitError if errors.As(err, &exitErr) { status = exitErr.ExitCode() } if ctx.Err() == context.DeadlineExceeded { err = fmt.Errorf("timed out after %s", timeout) status = -1 } } return string(out), status, err } // runStdout returns only stdout so stderr warnings can't corrupt JSON. func runStdout(timeout time.Duration, name string, args ...string) ([]byte, error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return exec.CommandContext(ctx, name, args...).Output() } func errText(err error) string { var exitErr *exec.ExitError if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 { return strings.TrimSpace(string(exitErr.Stderr)) } return err.Error() } func requireCommand(name string) bool { if _, err := exec.LookPath(name); err != nil { message( "Command unavailable", fmt.Sprintf("The command '%s' is not installed or not available on this node.", name), ) return false } return true } func waitEnter() { fmt.Print("Press Enter to return to the menu...") _, _ = bufio.NewReader(os.Stdin).ReadString('\n') } // ---------------------------------------------------------------- cluster API helpers // pveshGet runs "pvesh get " and unmarshals the JSON result into v. func pveshGet(v any, path string, params ...string) error { args := append([]string{"get", path}, params...) args = append(args, "--output-format", "json") out, err := runStdout(60*time.Second, "pvesh", args...) if err != nil { return errors.New(errText(err)) } return json.Unmarshal(out, v) } // apiCall runs a state-changing pvesh call (create/set/delete) and returns // its raw output; for async operations this is the task UPID. func apiCall(method, path string, params ...string) (string, error) { args := append([]string{method, path}, params...) args = append(args, "--output-format", "json") logAction("api: pvesh %s", strings.Join(args, " ")) out, err := runStdout(defaultTimeout, "pvesh", args...) if err != nil { logAction("api failed: %s", errText(err)) return "", errors.New(errText(err)) } return strings.Trim(strings.TrimSpace(string(out)), `"`), nil } type taskStatus struct { Status string `json:"status"` ExitStatus string `json:"exitstatus"` } func upidNode(upid string) string { parts := strings.Split(upid, ":") if len(parts) >= 2 && nodeRe.MatchString(parts[1]) { return parts[1] } return nodeName } func taskState(upid string) (taskStatus, error) { var st taskStatus err := pveshGet(&st, "/nodes/"+upidNode(upid)+"/tasks/"+upid+"/status") return st, err } func taskLog(upid string, start int) []string { var raw []struct { T string `json:"t"` } if err := pveshGet(&raw, "/nodes/"+upidNode(upid)+"/tasks/"+upid+"/log", "--start", strconv.Itoa(start), "--limit", "1000"); err != nil { return nil } lines := make([]string, 0, len(raw)) for _, r := range raw { lines = append(lines, r.T) } return lines } // waitTask polls until the task stops (or the timeout passes). func waitTask(upid string, timeout time.Duration) (taskStatus, string, error) { deadline := time.Now().Add(timeout) for { st, err := taskState(upid) if err == nil && st.Status == "stopped" { return st, strings.Join(taskLog(upid, 0), "\n"), nil } if time.Now().After(deadline) { return st, strings.Join(taskLog(upid, 0), "\n"), fmt.Errorf("task still running after %s (see Cluster > Recent tasks)", timeout) } time.Sleep(time.Second) } } // runAPITask performs an API call and shows the resulting task log in a dialog. func runAPITask(title, method, path string, params ...string) { out, err := apiCall(method, path, params...) if err != nil { displayText(title, "Request failed:\n"+err.Error()+"\n") return } if !upidRe.MatchString(out) { text := "Completed (no task returned).\n" if out != "" && out != "null" { text += out + "\n" } displayText(title, text) return } st, log, werr := waitTask(out, actionTimeout) text := log + "\n" switch { case werr != nil: text += "\n" + werr.Error() + "\n" case st.ExitStatus == "OK": text += "\nResult: OK\n" default: text += "\nResult: " + st.ExitStatus + "\n" } displayText(title, text) } // followTask streams a long-running task's log to the terminal. func followTask(title, upid string) { select { // drop any stale Ctrl-C case <-sigCh: default: } fmt.Printf("\033[2J\033[H=== %s ===\nTask: %s\nPress Ctrl-C to stop following (the task keeps running on its node).\n\n", title, upid) start := 0 for { lines := taskLog(upid, start) for _, l := range lines { fmt.Println(l) } start += len(lines) if st, err := taskState(upid); err == nil && st.Status == "stopped" { for _, l := range taskLog(upid, start) { fmt.Println(l) } fmt.Printf("\nTask finished: %s\n", st.ExitStatus) break } select { case <-sigCh: fmt.Println("\nStopped following. The task is still running.") waitEnter() return case <-time.After(2 * time.Second): } } waitEnter() } // startTask issues a call and follows the task it returns. func startTask(title, method, path string, params ...string) { out, err := apiCall(method, path, params...) if err != nil { displayText(title, "Request failed:\n"+err.Error()+"\n") return } if !upidRe.MatchString(out) { displayText(title, "Unexpected response (no task ID):\n"+out+"\n") return } followTask(title, out) } // ---------------------------------------------------------------- resource discovery type resource struct { Type string `json:"type"` Node string `json:"node"` Status string `json:"status"` Name string `json:"name"` VMID json.Number `json:"vmid"` Pool string `json:"pool"` Storage string `json:"storage"` CPU float64 `json:"cpu"` Mem float64 `json:"mem"` MaxMem float64 `json:"maxmem"` Disk float64 `json:"disk"` MaxDisk float64 `json:"maxdisk"` Template json.Number `json:"template"` } func fetchResources() ([]resource, error) { var res []resource if err := pveshGet(&res, "/cluster/resources"); err != nil { return nil, err } return res, nil } func guestsFrom(res []resource, typ string) []guest { var gs []guest for _, r := range res { if r.Type != "qemu" && r.Type != "lxc" { continue } if typ != "" && r.Type != typ { continue } id, err := strconv.Atoi(r.VMID.String()) if err != nil || !nodeRe.MatchString(r.Node) { continue } g := guest{ID: id, Name: r.Name, Status: r.Status, Node: r.Node, Type: r.Type, Pool: r.Pool} if r.Template.String() == "1" { g.Template = true g.Status = "template" } gs = append(gs, g) } sort.Slice(gs, func(i, j int) bool { return gs[i].ID < gs[j].ID }) return gs } // listGuests returns guests from the whole cluster; typ "" means VMs and containers. func listGuests(typ string) ([]guest, error) { res, err := fetchResources() if err != nil { return nil, err } return guestsFrom(res, typ), nil } func findGuest(id string) (guest, bool) { gs, err := listGuests("") if err != nil { return guest{}, false } for _, g := range gs { if strconv.Itoa(g.ID) == id { return g, true } } return guest{}, false } func guestTable(gs []guest) string { var b strings.Builder fmt.Fprintf(&b, "%-8s %-6s %-26s %-10s %-16s %s\n", "ID", "TYPE", "NAME", "STATUS", "NODE", "POOL") for _, g := range gs { fmt.Fprintf(&b, "%-8d %-6s %-26.26s %-10s %-16.16s %s\n", g.ID, g.Type, g.Name, g.Status, g.Node, g.Pool) } if len(gs) == 0 { b.WriteString("(none)\n") } return b.String() } func describeGuests(gs []guest, max int) string { var lines []string for i, g := range gs { if i >= max { lines = append(lines, fmt.Sprintf("...and %d more", len(gs)-max)) break } lines = append(lines, " "+g.short()) } return strings.Join(lines, "\n") } func pickGuest(typ, prompt string) (guest, bool) { if !requireCommand("pvesh") { return guest{}, false } guests, err := listGuests(typ) if err != nil { message("Error", "Could not list guests:\n"+err.Error()) return guest{}, false } if len(guests) == 0 { message("Nothing found", "No matching guests were found in the cluster.") return guest{}, false } var items []string for _, g := range guests { tag := "VM" if g.Type == "lxc" { tag = "CT" } label := fmt.Sprintf("[%s] %s (%s) on %s", tag, g.Name, g.Status, g.Node) if g.Pool != "" { label += " {" + g.Pool + "}" } items = append(items, strconv.Itoa(g.ID), label) } choice := menu("Select guest", prompt, items...) for _, g := range guests { if strconv.Itoa(g.ID) == choice { return g, true } } return guest{}, false } func listNodes(onlineOnly bool) ([]string, error) { var raw []struct { Node string `json:"node"` Status string `json:"status"` } if err := pveshGet(&raw, "/nodes"); err != nil { return nil, err } var nodes []string for _, n := range raw { if !nodeRe.MatchString(n.Node) || (onlineOnly && n.Status != "online") { continue } nodes = append(nodes, n.Node) } sort.Strings(nodes) return nodes, nil } // pickNode lets the user choose a node; a single candidate is selected automatically. func pickNode(prompt, exclude string, onlineOnly bool) (string, bool) { nodes, err := listNodes(onlineOnly) if err != nil { message("Error", "Could not list nodes:\n"+err.Error()) return "", false } var items []string for _, n := range nodes { if n != exclude { items = append(items, n, "node") } } if len(items) == 0 { message("No nodes", "No suitable nodes were found.") return "", false } if len(items) == 2 { return items[0], true } choice := menu("Select node", prompt, items...) return choice, choice != "" } func nextID() string { out, err := runStdout(15*time.Second, "pvesh", "get", "/cluster/nextid") if err != nil { return "" } return strings.Trim(strings.TrimSpace(string(out)), `"`) } func capitalize(s string) string { if s == "" { return s } return strings.ToUpper(s[:1]) + s[1:] } // ---------------------------------------------------------------- whiptail wrappers func boxHeight(text string, width int) int { lines := 0 for _, l := range strings.Split(text, "\n") { lines += 1 + len(l)/(width-6) } return lines + 6 } func termSize() (int, int) { cmd := exec.Command("stty", "size") cmd.Stdin = os.Stdin out, err := cmd.Output() if err != nil { return 24, 80 } parts := strings.Fields(string(out)) if len(parts) != 2 { return 24, 80 } rows, e1 := strconv.Atoi(parts[0]) cols, e2 := strconv.Atoi(parts[1]) if e1 != nil || e2 != nil || rows <= 0 || cols <= 0 { return 24, 80 } return rows, cols } func whiptailOK(args ...string) (string, bool) { readEnd, writeEnd, err := os.Pipe() if err != nil { return "", false } defer readEnd.Close() args = append([]string{"--output-fd", "3"}, args...) cmd := exec.Command("whiptail", args...) cmd.Stdin = os.Stdin cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.ExtraFiles = []*os.File{writeEnd} err = cmd.Run() _ = writeEnd.Close() var result bytes.Buffer _, _ = result.ReadFrom(readEnd) if err != nil { return "", false } return strings.TrimSpace(result.String()), true } func whiptail(args ...string) string { out, _ := whiptailOK(args...) return out } func menu(title, prompt string, items ...string) string { n := len(items) / 2 menuHeight := n if menuHeight > 14 { menuHeight = 14 } height := menuHeight + strings.Count(prompt, "\n") + 8 _, cols := termSize() width := cols - 4 if width > 110 { width = 110 } if width < 60 { width = 60 } args := []string{ "--title", title, "--menu", prompt, strconv.Itoa(height), strconv.Itoa(width), strconv.Itoa(menuHeight), } args = append(args, items...) return whiptail(args...) } func message(title, text string) { whiptail("--title", title, "--msgbox", text, strconv.Itoa(boxHeight(text, 70)), "70") } func confirmAction(text string, destructive bool) bool { args := []string{"--title", "Confirm"} if destructive { args = append(args, "--defaultno") } args = append(args, "--yesno", text, strconv.Itoa(boxHeight(text, 70)), "70") cmd := exec.Command("whiptail", args...) cmd.Stdin = os.Stdin cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr return cmd.Run() == nil } func inputValue(title, prompt string) (string, bool) { return whiptailOK("--title", title, "--inputbox", prompt, strconv.Itoa(boxHeight(prompt, 72)+2), "72") } func askID(prompt string) (string, bool) { value, ok := inputValue("Enter ID", prompt) if !ok { return "", false } if !numeric.MatchString(value) { message("Invalid ID", "Enter a numeric VM or container ID between 100 and 999999999.") return "", false } return value, true } func askSnapshotName() (string, bool) { value, ok := inputValue("Snapshot name", "Enter a snapshot name (must start with a letter; letters, digits, _ or - only; max 40 chars):") if !ok { return "", false } if !snapName.MatchString(value) { message("Invalid snapshot name", "Names must start with a letter and contain only letters, digits, underscores and hyphens (2-40 characters).") return "", false } return value, true } // ---------------------------------------------------------------- output display func displayText(title, text string) { outputPath := filepath.Join(tmpDir, "output.txt") if err := os.WriteFile(outputPath, []byte(text), 0600); err != nil { message("Error", "Could not write command output to the temporary file.") return } rows, cols := termSize() height := rows - 4 width := cols - 4 if height < 12 { height = 12 } if width < 50 { width = 50 } whiptail("--scrolltext", "--title", title, "--textbox", outputPath, strconv.Itoa(height), strconv.Itoa(width)) } func showOutput(title, name string, args ...string) { out, status, err := run(defaultTimeout, name, args...) text := out if strings.TrimSpace(out) == "" && err == nil { text = "Command completed successfully; no output.\n" } if err != nil { if status < 0 { text += "\n" + err.Error() + "\n" } else { text += fmt.Sprintf("\nCommand exited with status %d.\n", status) } } displayText(title, text) } func showJSON(title, name string, args ...string) { out, err := runStdout(defaultTimeout, name, args...) if err != nil { displayText(title, errText(err)+"\n") return } var formatted bytes.Buffer if err := json.Indent(&formatted, out, "", " "); err != nil { displayText(title, "Could not format JSON:\n"+err.Error()+"\n\n"+string(out)) return } displayText(title, formatted.String()+"\n") } func showSettings(g guest) { var cfg map[string]any title := g.short() + " CPU/memory" if err := pveshGet(&cfg, g.path("/config")); err != nil { displayText(title, err.Error()+"\n") return } keys := []string{"memory", "balloon", "swap", "cores", "sockets", "vcpus", "cpu", "cpulimit", "cpuunits"} var lines []string for _, k := range keys { if v, ok := cfg[k]; ok { lines = append(lines, fmt.Sprintf("%-10s %v", k+":", v)) } } if len(lines) == 0 { lines = []string{"No matching settings found."} } displayText(title, strings.Join(lines, "\n")+"\n") } // ---------------------------------------------------------------- guest menus func guestMenu(k guestKind) { for { choice := menu( k.title+" - whole cluster", "Choose an action:", "list", "List all (every node)", "status", "Show status", "config", "Show full configuration", "resources", "Show CPU / memory settings", "start", "Start", "shutdown", "Gracefully shut down (60s timeout)", "reboot", "Reboot", "stop", "Force stop (like pulling the power)", "migrate", "Migrate to another node", "snapshots", "Snapshots", "unlock", "Unlock", "back", "Back", ) switch choice { case "back", "": return case "list": if !requireCommand("pvesh") { continue } gs, err := listGuests(k.typ) if err != nil { message("Error", "Could not list guests:\n"+err.Error()) continue } displayText(k.title, guestTable(gs)) default: g, ok := pickGuest(k.typ, "Select a "+k.label+":") if ok { guestAction(g, choice) } } } } func guestAction(g guest, action string) { k := kindOf(g.Type) name := capitalize(k.label) + " " + strconv.Itoa(g.ID) switch action { case "status": showJSON(name+" status", "pvesh", "get", g.path("/status/current"), "--output-format", "json") case "config": showJSON(name+" configuration", "pvesh", "get", g.path("/config"), "--output-format", "json") case "resources": showSettings(g) case "unlock": if confirmAction("Unlock "+g.short()+"?\n\nOnly do this after confirming no task is still running for it.", true) { runAPITask("Unlock "+name, "set", g.path("/config"), "--delete", "lock") } case "start", "shutdown", "reboot", "stop": runGuestAction(g, action) case "migrate": migrateGuest(g) case "snapshots": snapshotMenu(g) } } func runGuestAction(g guest, action string) { k := kindOf(g.Type) prompt := fmt.Sprintf("%s %s?", capitalize(action), g.short()) destructive := false params := []string{} switch action { case "shutdown": prompt += "\n\nThis requests a graceful shutdown and waits up to 60 seconds." params = []string{"--timeout", "60"} case "stop": prompt += "\n\nThis immediately powers off the guest without shutting down the OS. Unsaved data may be lost." destructive = true case "reboot": destructive = true } if !confirmAction(prompt, destructive) { return } runAPITask(capitalize(action)+" "+k.label+" "+strconv.Itoa(g.ID), "create", g.path("/status/"+action), params...) } func migrateGuest(g guest) { k := kindOf(g.Type) target, ok := pickNode("Migrate "+g.short()+" to:", g.Node, true) if !ok { return } params := []string{"--target", target} note := "" if g.Status == "running" { if g.Type == "qemu" { params = append(params, "--online", "1") note = "\n\nThe VM is running and will be live-migrated." } else { params = append(params, "--restart", "1") note = "\n\nThe container is running and will be restarted on the target (containers cannot live-migrate)." } } if !confirmAction("Migrate "+g.short()+" to node "+target+"?"+note+"\n\nLocal disks or devices may block migration.", true) { return } startTask("Migrate "+k.label+" "+strconv.Itoa(g.ID), "create", g.path("/migrate"), params...) } func pickSnapshot(g guest, action string) (string, bool) { var raw []struct { Name string `json:"name"` Description string `json:"description"` } if err := pveshGet(&raw, g.path("/snapshot")); err != nil { message("Error", "Could not list snapshots:\n"+err.Error()) return "", false } var items []string for _, s := range raw { if s.Name == "current" || !snapName.MatchString(s.Name) { continue } desc := strings.ReplaceAll(s.Description, "\n", " ") if desc == "" { desc = "(no description)" } items = append(items, s.Name, desc) } if len(items) == 0 { message("No snapshots", "This guest has no snapshots.") return "", false } name := menu("Select snapshot", "Snapshot to "+action+":", items...) return name, name != "" } func showSnapshots(g guest) { var raw []struct { Name string `json:"name"` SnapTime float64 `json:"snaptime"` Description string `json:"description"` } title := g.short() + " snapshots" if err := pveshGet(&raw, g.path("/snapshot")); err != nil { displayText(title, err.Error()+"\n") return } var b strings.Builder fmt.Fprintf(&b, "%-24s %-17s %s\n", "NAME", "CREATED", "DESCRIPTION") for _, s := range raw { created := "-" if s.SnapTime > 0 { created = time.Unix(int64(s.SnapTime), 0).Format("2006-01-02 15:04") } fmt.Fprintf(&b, "%-24.24s %-17s %s\n", s.Name, created, strings.ReplaceAll(s.Description, "\n", " ")) } displayText(title, b.String()) } func snapshotMenu(g guest) { for { choice := menu( g.short()+" snapshots", "Choose an action:", "list", "List snapshots", "create", "Create snapshot", "rollback", "Roll back to a snapshot", "delete", "Delete snapshot", "back", "Back", ) switch choice { case "list": showSnapshots(g) case "create": name, ok := askSnapshotName() if !ok { continue } if confirmAction("Create snapshot '"+name+"' for "+g.short()+"?\n\nSnapshots are not a substitute for backups and may consume storage.", false) { runAPITask("Create snapshot "+name, "create", g.path("/snapshot"), "--snapname", name) } case "rollback": name, ok := pickSnapshot(g, "roll back to") if !ok { continue } if confirmAction("Roll "+g.short()+" back to snapshot '"+name+"'?\n\nThe current state and any newer snapshots will be lost.", true) { runAPITask("Rollback to "+name, "create", g.path("/snapshot/"+name+"/rollback")) } case "delete": name, ok := pickSnapshot(g, "delete") if !ok { continue } if confirmAction("Permanently delete snapshot '"+name+"' from "+g.short()+"?\n\nThis cannot be undone.", true) { runAPITask("Delete snapshot "+name, "delete", g.path("/snapshot/"+name)) } case "back", "": return } } } // ---------------------------------------------------------------- resource pools type pool struct { ID string `json:"poolid"` Comment string `json:"comment"` } func listPools() ([]pool, error) { var pools []pool if err := pveshGet(&pools, "/pools"); err != nil { return nil, err } sort.Slice(pools, func(i, j int) bool { return pools[i].ID < pools[j].ID }) return pools, nil } func poolMembers(gs []guest, id string) []guest { var out []guest for _, g := range gs { if g.Pool == id { out = append(out, g) } } return out } func pickPool() (string, bool) { if !requireCommand("pvesh") { return "", false } pools, err := listPools() if err != nil { message("Error", "Could not list pools:\n"+err.Error()) return "", false } if len(pools) == 0 { message("No pools", "No resource pools are defined in this cluster.") return "", false } gs, _ := listGuests("") var items []string for _, p := range pools { label := fmt.Sprintf("%d guests", len(poolMembers(gs, p.ID))) if p.Comment != "" { label += " - " + strings.ReplaceAll(p.Comment, "\n", " ") } items = append(items, p.ID, label) } choice := menu("Select pool", "Resource pool:", items...) return choice, choice != "" } func showPools() { if !requireCommand("pvesh") { return } pools, err := listPools() if err != nil { message("Error", "Could not list pools:\n"+err.Error()) return } gs, _ := listGuests("") var b strings.Builder for _, p := range pools { fmt.Fprintf(&b, "== %s ==", p.ID) if p.Comment != "" { fmt.Fprintf(&b, " (%s)", strings.ReplaceAll(p.Comment, "\n", " ")) } b.WriteString("\n") b.WriteString(guestTable(poolMembers(gs, p.ID))) b.WriteString("\n") } if len(pools) == 0 { b.WriteString("No resource pools are defined.\n") } displayText("Resource pools", b.String()) } // bulkRun issues one API call per guest, then waits for all tasks and reports. func bulkRun(title string, targets []guest, fn func(guest) (string, error)) { fmt.Printf("\033[2J\033[H=== %s ===\n\n", title) logAction("bulk: %s (%d guests)", title, len(targets)) type pending struct { g guest upid string } var queue []pending failed := 0 for i, g := range targets { fmt.Printf("[%d/%d] %s: requesting...\n", i+1, len(targets), g.short()) out, err := fn(g) switch { case err != nil: failed++ fmt.Printf(" FAILED: %s\n", err) case upidRe.MatchString(out): queue = append(queue, pending{g, out}) default: fmt.Println(" done") } } if len(queue) > 0 { fmt.Println("\nWaiting for tasks to finish...") } for _, p := range queue { st, _, err := waitTask(p.upid, actionTimeout) switch { case err != nil: failed++ fmt.Printf(" %s: %s\n", p.g.short(), err) case st.ExitStatus != "OK": failed++ fmt.Printf(" %s: FAILED (%s)\n", p.g.short(), st.ExitStatus) default: fmt.Printf(" %s: OK\n", p.g.short()) } } fmt.Printf("\nFinished: %d succeeded, %d failed.\n", len(targets)-failed, failed) waitEnter() } func poolAction(action string) { id, ok := pickPool() if !ok { return } all, err := listGuests("") if err != nil { message("Error", "Could not list guests:\n"+err.Error()) return } var targets []guest for _, g := range poolMembers(all, id) { switch action { case "start": if g.Status == "stopped" && !g.Template { targets = append(targets, g) } case "shutdown": if g.Status == "running" { targets = append(targets, g) } case "snapshot": if !g.Template { targets = append(targets, g) } } } if len(targets) == 0 { message("Nothing to do", "No guests in pool '"+id+"' match this action.") return } var fn func(guest) (string, error) verb := "" switch action { case "start": verb = "Start" fn = func(g guest) (string, error) { return apiCall("create", g.path("/status/start")) } case "shutdown": verb = "Gracefully shut down" fn = func(g guest) (string, error) { return apiCall("create", g.path("/status/shutdown"), "--timeout", "60") } case "snapshot": name, ok := askSnapshotName() if !ok { return } verb = "Snapshot '" + name + "' on" fn = func(g guest) (string, error) { return apiCall("create", g.path("/snapshot"), "--snapname", name) } } text := fmt.Sprintf("%s these %d guests in pool '%s'?\n\n%s", verb, len(targets), id, describeGuests(targets, 12)) if !confirmAction(text, action != "start") { return } bulkRun(verb+" pool "+id, targets, fn) } func poolsMenu() { for { choice := menu( "Resource Pools", "Choose an action (applies across all nodes):", "list", "List pools and members", "start", "Start all stopped guests in a pool", "shutdown", "Gracefully shut down all running guests in a pool", "snapshot", "Snapshot all guests in a pool", "back", "Back", ) switch choice { case "list": showPools() case "start", "shutdown", "snapshot": poolAction(choice) case "back", "": return } } } // ---------------------------------------------------------------- backup / restore func pickBackupStorage(node string) (string, bool) { var raw []struct { Storage string `json:"storage"` Type string `json:"type"` Avail float64 `json:"avail"` } if err := pveshGet(&raw, "/nodes/"+node+"/storage", "--content", "backup", "--enabled", "1"); err != nil { message("Storage error", "Could not list backup storage on "+node+":\n"+err.Error()) return "", false } var items []string for _, s := range raw { if !storageName.MatchString(s.Storage) { continue } items = append(items, s.Storage, fmt.Sprintf("%s, %.1f GiB free", s.Type, s.Avail/(1<<30))) } if len(items) == 0 { message("No backup storage", "No enabled storage with content type 'backup' on node "+node+".") return "", false } choice := menu("Backup storage ("+node+")", "Select a storage:", items...) return choice, choice != "" } func backupMenu() { g, ok := pickGuest("", "Select the VM or container to back up:") if !ok { return } storage, ok := pickBackupStorage(g.Node) if !ok { return } mode := menu("Backup mode", "Choose a backup mode:", "snapshot", "Snapshot - guest keeps running (recommended)", "suspend", "Suspend - brief pause while copying", "stop", "Stop - shut down, back up, restart", ) if mode == "" { return } compress := menu("Compression", "Choose compression:", "zstd", "zstd (fast, good ratio)", "gzip", "gzip", "lzo", "lzo (fastest, larger)", "0", "None", ) if compress == "" { return } summary := fmt.Sprintf("Back up %s?\n\nStorage: %s\nMode: %s\nCompression: %s", g.short(), storage, mode, compress) if !confirmAction(summary, false) { return } startTask("Backup "+g.short(), "create", "/nodes/"+g.Node+"/vzdump", "--vmid", strconv.Itoa(g.ID), "--storage", storage, "--mode", mode, "--compress", compress) } func manualArchive() (string, bool) { archive, ok := inputValue("Backup archive", "Enter the full path to the vzdump archive:") if !ok { return "", false } info, err := os.Stat(archive) if archive == "" || !filepath.IsAbs(archive) || err != nil || !info.Mode().IsRegular() { message("Archive not found", "Enter an absolute path to an existing regular file.") return "", false } return archive, true } func pickBackup(node, storage string) (string, bool) { var raw []struct { VolID string `json:"volid"` Size float64 `json:"size"` CTime float64 `json:"ctime"` } err := pveshGet(&raw, "/nodes/"+node+"/storage/"+storage+"/content", "--content", "backup") local := node == nodeName // manual paths can only be checked on this node if err != nil || len(raw) == 0 { if local { message("No backups listed", "No backups were found on '"+storage+"'. You can enter an archive path manually.") return manualArchive() } message("No backups", "No backups were found on '"+storage+"' ("+node+").") return "", false } sort.Slice(raw, func(i, j int) bool { return raw[i].CTime > raw[j].CTime }) var items []string for i, b := range raw { base := b.VolID if idx := strings.LastIndexByte(base, '/'); idx >= 0 && !strings.Contains(base, "backup/vm/") && !strings.Contains(base, "backup/ct/") { base = base[idx+1:] } items = append(items, strconv.Itoa(i+1), fmt.Sprintf("%s %.1f GiB", base, b.Size/(1<<30))) } if local { items = append(items, "manual", "Enter an archive path manually...") } choice := menu("Select backup", "Newest first:", items...) switch choice { case "": return "", false case "manual": return manualArchive() } idx, err := strconv.Atoi(choice) if err != nil || idx < 1 || idx > len(raw) { return "", false } return raw[idx-1].VolID, true } func detectArchiveType(archive string) string { switch { case strings.Contains(archive, "vzdump-qemu-"), strings.Contains(archive, "backup/vm/"): return "qemu" case strings.Contains(archive, "vzdump-lxc-"), strings.Contains(archive, "backup/ct/"): return "lxc" } return "" } func restoreMenu() { if !requireCommand("pvesh") { return } prompt := "Enter the target VMID / container ID:" if n := nextID(); n != "" { prompt = "Enter the target VMID / container ID (next free ID: " + n + "):" } id, ok := askID(prompt) if !ok { return } // An existing guest can only be overwritten on the node it lives on. existing, exists := findGuest(id) var node string if exists { if !confirmAction("ID "+id+" already exists:\n "+existing.short()+"\n\nRestoring will DESTROY and overwrite it. Continue?", true) { return } node = existing.Node } else if node, ok = pickNode("Restore on which node?", "", true); !ok { return } storage, ok := pickBackupStorage(node) if !ok { return } archive, ok := pickBackup(node, storage) if !ok { return } typ := detectArchiveType(archive) if typ == "" { typ = menu("Archive type", "Could not detect the archive type. What is it?", "qemu", "QEMU VM", "lxc", "Container (LXC)", ) if typ == "" { return } } if exists && existing.Type != typ { message("Type mismatch", "The existing guest is a "+kindOf(existing.Type).label+ " but the archive is a "+kindOf(typ).label+". Choose a different ID.") return } target, ok := inputValue("Target storage", "Storage for the restored disks (leave blank for the default):") if !ok { return } if target != "" && !storageName.MatchString(target) { message("Invalid storage", "Storage names may only contain letters, digits, _, . and -, and must start with a letter or digit.") return } if !confirmAction("Restore '"+archive+"' as ID "+id+" on node "+node+"?", true) { return } params := []string{"--vmid", id} if typ == "qemu" { params = append(params, "--archive", archive) } else { params = append(params, "--ostemplate", archive, "--restore", "1") } if target != "" { params = append(params, "--storage", target) } if exists { params = append(params, "--force", "1") } startTask("Restore ID "+id, "create", "/nodes/"+node+"/"+typ, params...) } // ---------------------------------------------------------------- reports func clusterOverview() { if !requireCommand("pvesh") { return } res, err := fetchResources() if err != nil { message("Error", "Could not read cluster resources:\n"+err.Error()) return } gib := func(v float64) float64 { return v / (1 << 30) } var b strings.Builder b.WriteString("NODES\n") fmt.Fprintf(&b, "%-16s %-9s %-7s %-20s %s\n", "NODE", "STATUS", "CPU", "MEMORY (GiB)", "VMs/CTs running") gs := guestsFrom(res, "") for _, r := range res { if r.Type != "node" { continue } running, total := 0, 0 for _, g := range gs { if g.Node == r.Node { total++ if g.Status == "running" { running++ } } } fmt.Fprintf(&b, "%-16.16s %-9s %5.1f%% %6.1f / %-10.1f %d/%d\n", r.Node, r.Status, r.CPU*100, gib(r.Mem), gib(r.MaxMem), running, total) } b.WriteString("\nSTORAGE\n") fmt.Fprintf(&b, "%-20s %-16s %-10s %s\n", "STORAGE", "NODE", "STATUS", "USED") for _, r := range res { if r.Type != "storage" { continue } pct := 0.0 if r.MaxDisk > 0 { pct = r.Disk / r.MaxDisk * 100 } fmt.Fprintf(&b, "%-20.20s %-16.16s %-10s %5.1f%% of %.0f GiB\n", r.Storage, r.Node, r.Status, pct, gib(r.MaxDisk)) } b.WriteString("\nGUESTS\n") b.WriteString(guestTable(gs)) displayText("Cluster overview (all nodes)", b.String()) } func storageMenu() { for { choice := menu( "Storage (this node)", "Choose a report:", "status", "Proxmox storage status (pvesm status)", "pvs", "Physical volumes (pvs)", "vgs", "Volume groups (vgs)", "lvs", "Logical volumes (lvs)", "zfs", "ZFS pool status (zpool status)", "df", "Filesystem usage (df -h)", "back", "Back", ) switch choice { case "status": if requireCommand("pvesm") { showOutput("Proxmox storage status", "pvesm", "status") } case "pvs", "vgs", "lvs": if requireCommand(choice) { showOutput(choice, choice) } case "zfs": if requireCommand("zpool") { showOutput("ZFS pool status", "zpool", "status") } case "df": showOutput("Filesystem usage", "df", "-h") case "back", "": return } } } func resourcesMenu() { for { choice := menu( "Cluster Resources", "Choose a resource type to display as formatted JSON:", "vm", "Virtual machines and containers", "storage", "Storage", "node", "Nodes", "sdn", "SDN", "pool", "Pools", "back", "Back", ) switch choice { case "vm", "storage", "node", "sdn", "pool": if requireCommand("pvesh") { showJSON("Cluster "+choice+" resources", "pvesh", "get", "/cluster/resources", "--type", choice, "--output-format", "json") } case "back", "": return } } } func clusterMenu() { for { choice := menu( "Cluster and Node", "Choose a report:", "overview", "Cluster overview: nodes, storage, all guests", "status", "Cluster status (pvecm status)", "nodes", "List nodes (JSON)", "nodestatus", "Node status: CPU, memory, uptime", "tasks", "Recent cluster tasks", "back", "Back", ) switch choice { case "overview": clusterOverview() case "status": if requireCommand("pvecm") { showOutput("Cluster status", "pvecm", "status") } case "nodes": if requireCommand("pvesh") { showJSON("Cluster nodes", "pvesh", "get", "/nodes", "--output-format", "json") } case "nodestatus": if !requireCommand("pvesh") { continue } if n, ok := pickNode("Show status for:", "", false); ok { showJSON("Node "+n+" status", "pvesh", "get", "/nodes/"+n+"/status", "--output-format", "json") } case "tasks": if requireCommand("pvesh") { showJSON("Recent cluster tasks", "pvesh", "get", "/cluster/tasks", "--output-format", "json") } case "back", "": return } } } func systemMenu() { for { choice := menu( "System (this node)", "Choose a report:", "services", "Proxmox service status", "updates", "Upgradable packages (as of last apt update)", "network", "Network addresses (ip -br address)", "log", "Toolkit action log", "back", "Back", ) switch choice { case "services": showOutput("Proxmox services", "systemctl", "status", "pve-cluster", "pvedaemon", "pveproxy", "pvestatd", "--no-pager", "--lines=0") case "updates": if requireCommand("apt") { showOutput("Upgradable packages", "apt", "list", "--upgradable") } case "network": showOutput("Network addresses", "ip", "-br", "address") case "log": showOutput("Toolkit log (last 200 lines)", "tail", "-n", "200", logPath) case "back", "": return } } } // ---------------------------------------------------------------- main menu func mainMenu() { for { choice := menu( "Proxmox VE Toolkit - DEVELOPMENT ONLY - NOT FOR PRODUCTION "+version, "Select a task (running on node: "+nodeName+"):", "vms", "Virtual machines (QEMU) - whole cluster", "containers", "Containers (LXC) - whole cluster", "pools", "Resource pools - bulk actions", "backup", "Back up a VM or container (vzdump)", "restore", "Restore a VM or container", "cluster", "Cluster overview and reports", "storage", "Storage / LVM / ZFS reports (this node)", "system", "System: services, updates, network, log", "resources", "Cluster resources (formatted JSON)", "version", "Proxmox version (pveversion -v)", "perf", "Host performance test (pveperf)", "subscription", "Subscription status", "dab", "Debian Appliance Builder help", "quit", "Exit", ) switch choice { case "vms": guestMenu(kindVM) case "containers": guestMenu(kindCT) case "pools": poolsMenu() case "backup": backupMenu() case "restore": restoreMenu() case "cluster": clusterMenu() case "storage": storageMenu() case "system": systemMenu() case "resources": resourcesMenu() case "version": if requireCommand("pveversion") { showOutput("Proxmox VE version", "pveversion", "-v") } case "perf": if requireCommand("pveperf") && confirmAction("Run pveperf?\n\nThis can cause temporary disk activity.", false) { showOutput("Host performance", "pveperf") } case "subscription": if requireCommand("pvesubscription") { showOutput("Subscription status", "pvesubscription", "get") } case "dab": if requireCommand("dab") { showOutput("Debian Appliance Builder help", "dab", "help") } case "quit", "": return } } }