#!/usr/bin/env bash
#=============================================================================
# pve-upgrade.sh —— Proxmox VE 通用一键升级 (单机 / 集群自动识别)
#
#   起点: PVE 6.x / 7.x / 8.x / 9.x (Debian buster/bullseye/bookworm/trixie)
#   终点: 最新 PVE 9.x + 最新 Proxmox 内核系列,每个节点全程只重启一次
#
# 角色 (同一个文件,按参数/环境自动切换):
#   launcher    默认。检测是否多节点集群:是 -> 协调器;否 -> 单机 worker。
#   worker      在本机按阶段执行升级链 (后台 systemd 单元,SSH 断开不影响)。
#   coordinator 集群编排: 把本文件分发到全部节点,全节点【同一跳并行】
#               + 每跳栅栏 (任意时刻集群版本差 <= 一个相邻大版本),
#               链完成后【串行金丝雀重启】(本机最后),quorum 全程不丢。
#
# 用法:
#   单机:   I_UNDERSTAND=yes bash /root/pve-upgrade.sh          # 跟随日志
#   集群:   同上,在【想最后重启】的那台节点上跑 (通常是最关键/出口节点)
#   失败:   修复后重跑同一命令即断点续接 (state 文件天然幂等)
#   重启:   bash /root/pve-upgrade.sh reboot     # 单机: 本机;集群: 串行编排
#   验收:   bash /root/pve-upgrade.sh verify
#   看状态: bash /root/pve-upgrade.sh status
#
# 环境开关 (全部可选):
#   AUTO_REBOOT=1        链完成后自动重启 (单机直接重启;集群自动进入串行编排)
#   LATEST_KERNEL=1      [默认 1] 装仓库里最新的 proxmox-kernel-X.Y 系列
#                        (=0 保持 proxmox-default-kernel 指向的官方默认内核)
#   KEEP_GUESTS="100 105" 升级期间【不关闭】的 VM/CT ID (软路由/OOB 跳板等,
#                        它们是你出口网络的一部分时必填;最终重启时仍会被正常关闭)
#   STOP_GUESTS=auto     auto=起点 <9 (要跨大版本) 才停全部虚机;=1 总停;=0 总不停
#   SKIP_CHECKS=1        pveXtoY --full 出现真 FAIL 也继续 (慎用)
#   KEEP_DEBIAN_KERNEL=1 保留 Debian 原生 linux-image-* (默认清掉,PVE 只需 proxmox 内核)
#   MODE=single|cluster  强制模式 (默认 auto 检测)
#   NODES="ip1 ip2 ..."  集群: 手工指定 ssh 目标 (默认解析 corosync.conf ring0_addr)
#   STAGE_TIMEOUT=10800  集群: 每跳每节点最长等待秒数
#
# 前提与边界:
#   - 每台都有 vzdump 外部备份 + IPMI/console 兜底。全程只保 /etc + config.db。
#   - Ceph 一律拒绝 (Ceph 必须按官方序列单独升级)。
#   - 起点 <9 时按官方"逐大版本 dist-upgrade"链式执行但【不逐跳重启】——
#     这是非官方路径,旧内核一路带飞到最后一次重启。起点 =9 时是纯刷新。
#   - 已吸收的实战教训: buster PVE 源已全网下架 (退化为 Debian 归档基座 + 无
#     移除 upgrade,绕开 pve-apt-hook);PVE 6 遗留的 efiboot 孤儿钩子会连坐
#     initramfs (跳 7 前预清);Debian 底子装的 PVE 会拖着 linux-image-amd64
#     (收尾清理);grub 可移动引导路径需 force_efi_extra_removable;网卡命名
#     按 MAC 钉死;运行内核 hold 住当回退梯子;探测全部放条件位防竞态。
#=============================================================================
set -Eeuo pipefail
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export DEBIAN_FRONTEND=noninteractive
export APT_LISTCHANGES_FRONTEND=none
export UCF_FORCE_CONFFOLD=1
export NEEDRESTART_MODE=a
export LC_ALL=C LANG=C

TAG=pveup
LOG=/var/log/pve-upgrade.log
CLOG=/var/log/pve-upgrade-cluster.log
STATE=/var/lib/pve-upgrade.state
STATEDIR=/var/lib/pve-upgrade.d
BK=/root/pve-upgrade-backup
UNIT=pve-upgrade
CUNIT=pve-upgrade-cluster
SELF=$(readlink -f "$0")
SELF_BASE=$(basename "$SELF")
REMOTE_SELF="/root/$SELF_BASE"
CURRENT_STAGE=init
STAGES_SEQ=(preflight to64 to7 to8 to9 refresh finalize)

: "${AUTO_REBOOT:=0}"
: "${LATEST_KERNEL:=1}"
: "${KEEP_GUESTS:=}"
: "${STOP_GUESTS:=auto}"
: "${SKIP_CHECKS:=0}"
: "${KEEP_DEBIAN_KERNEL:=0}"
: "${I_UNDERSTAND:=}"
: "${MODE:=auto}"
: "${NODES:=}"
: "${STAGE_TIMEOUT:=10800}"
# --- 协调器 -> worker 的内部接口 (人工使用无需理会) ---
: "${CLUSTER_OK:=0}"   # 1 = 本 worker 由协调器驱动 (放行多节点集群,不自行重启)
: "${STAGES:=all}"     # all 或逗号分隔阶段名
: "${BATCH:=0}"        # 1 = 启动后台单元后立即返回,不 tail

if [ "$STAGES" != all ]; then
  UNIT="pve-upgrade-$(echo "$STAGES" | tr -c 'a-zA-Z0-9' '-' | sed 's/-*$//')"
fi
want_stage() {
  [ "$STAGES" = all ] && return 0
  case ",$STAGES," in *",$1,"*) return 0 ;; esac
  return 1
}

APT_GET=(apt-get -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold)

#=============================================================== 通用工具
ts()  { date '+%F %T'; }
log() { echo "[$(ts)] $*"; }
die() {
  echo "[$(ts)] 致命错误: $*" >&2
  echo "[$(ts)] 修复后重新运行同一命令即可从断点续接 (state: $STATE)" >&2
  exit 1
}
trap 'rc=$?; echo "[$(ts)] !! [$CURRENT_STAGE] 出错 (exit=$rc)。修复后重跑同一命令可续。" >&2' ERR

stage_done() { grep -qx "$1" "$STATE" 2>/dev/null; }
mark_done()  { mkdir -p "$(dirname "$STATE")"; echo "$1" >> "$STATE"; }
kv_set()     { mkdir -p "$STATEDIR"; printf '%s' "$2" > "$STATEDIR/$1"; }
kv_get()     { cat "$STATEDIR/$1" 2>/dev/null || true; }

pmver() { pveversion 2>/dev/null | sed -n 's|^pve-manager/\([^/]*\)/.*|\1|p'; }
major() { pveversion 2>/dev/null | sed -n 's|^pve-manager/\([0-9]*\)\..*|\1|p'; }
require_major() { local mj; mj=$(major); [ "$mj" = "$1" ] || die "期望到达 PVE $1,实际为 '$(pmver)'"; }
newest_kernel() { ls /lib/modules 2>/dev/null | sort -V | tail -1; }
is_cluster() { [ -e /etc/pve/corosync.conf ] && [ "$(ls /etc/pve/nodes 2>/dev/null | wc -l)" -gt 1 ]; }
repo_mode()  { kv_get repomode; }   # chain | native

#=============================================================== APT 源管理
disable_foreign_repos() {
  local f
  shopt -s nullglob
  for f in /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do
    mv -f "$f" "$f.off-$TAG"; log "已停用源文件: $f -> $f.off-$TAG"
  done
  shopt -u nullglob
}
write_chain_aptconf() {
  cat > /etc/apt/apt.conf.d/99-pve-upgrade-chain <<'EOF'
Acquire::Check-Valid-Until "false";
Acquire::Retries "3";
EOF
}
fetch_key() {
  local k=$1 dst="/etc/apt/trusted.gpg.d/$1"
  wget -qO "$dst" "https://enterprise.proxmox.com/debian/$k" \
    || wget -qO "$dst" "http://download.proxmox.com/debian/$k" \
    || die "下载仓库签名 key 失败: $k"
  [ "$(stat -c%s "$dst")" -gt 500 ] || die "签名 key 异常(过小): $dst"
  log "已导入仓库 key: $dst"
}
write_sources() { # $1 codename  $2 live|archive
  local cn=$1 mode=$2 comp
  case "$cn" in
    buster|bullseye) comp="main contrib non-free" ;;
    bookworm|trixie) comp="main contrib non-free non-free-firmware" ;;
    *) die "未知 codename: $cn" ;;
  esac
  {
    echo "# generated by pve-upgrade @ $(ts)  [$cn/$mode]"
    if [ "$mode" = archive ]; then
      echo "deb http://archive.debian.org/debian $cn $comp"
      if [ "$cn" = buster ]; then echo "deb http://archive.debian.org/debian-security buster/updates $comp"
      else echo "deb http://archive.debian.org/debian-security $cn-security $comp"; fi
    else
      echo "deb http://deb.debian.org/debian $cn $comp"
      echo "deb http://deb.debian.org/debian $cn-updates $comp"
      if [ "$cn" = buster ]; then echo "deb http://security.debian.org/debian-security buster/updates $comp"
      else echo "deb http://security.debian.org/debian-security $cn-security $comp"; fi
    fi
    echo "deb http://download.proxmox.com/debian/pve $cn pve-no-subscription"
  } > /etc/apt/sources.list
  log "sources.list 已切换到 $cn ($mode)"
}
apt_update() {
  if [ "$(repo_mode)" = chain ]; then disable_foreign_repos; fi
  apt-get update --allow-releaseinfo-change
}
switch_repo() { # buster 强制归档;其余先 live 后 archive
  local cn=$1
  if [ "$cn" = buster ]; then
    write_sources buster archive; apt_update || die "buster 归档源 update 失败"
  else
    write_sources "$cn" live
    if ! apt_update; then
      log "live 源失败,回退 archive.debian.org ($cn)"
      write_sources "$cn" archive; apt_update || die "$cn 源 update 失败 (live+archive 均不可用)"
    fi
  fi
}
repair_dpkg()  { dpkg --configure -a || true; "${APT_GET[@]}" -f install || true; }
dist_upgrade() { repair_dpkg; "${APT_GET[@]}" dist-upgrade; apt-get clean; }

#=============================================================== 门禁 / 虚机 / 网卡
checker_gate() { # pve6to7 / pve7to8 / pve8to9
  local tool=$1 out fails kern_fails real
  command -v "$tool" >/dev/null 2>&1 || { log "未找到 $tool,跳过检查门禁"; return 0; }
  log "运行 $tool --full ..."
  out=$("$tool" --full 2>&1) || true
  printf '%s\n' "$out"
  fails=$(printf '%s\n' "$out" | grep -c '^FAIL:' || true)
  kern_fails=$(printf '%s\n' "$out" | grep '^FAIL:' | grep -ciE 'running kernel|newer kernel|latest kernel' || true)
  real=$(( fails - kern_fails ))
  if [ "$real" -gt 0 ] && [ "$SKIP_CHECKS" != 1 ]; then
    die "$tool 报告 $real 个 FAIL (另 $kern_fails 个 running-kernel 误报已豁免)。逐项处理后重跑,或 SKIP_CHECKS=1 强行继续。"
  fi
  log "$tool 门禁通过: FAIL=$fails, 其中 running-kernel 误报 $kern_fails"
}

keep_guest() { case " $KEEP_GUESTS " in *" $1 "*) return 0 ;; esac; return 1; }
stop_guests() {
  local sid id
  [ -n "$KEEP_GUESTS" ] && log "KEEP_GUESTS: 保持运行 -> $KEEP_GUESTS (软路由/跳板类,最终重启时才关)"
  if command -v ha-manager >/dev/null 2>&1; then
    for sid in $(ha-manager config 2>/dev/null | grep -oE '^(vm|ct):[0-9]+' || true); do
      keep_guest "${sid#*:}" && continue
      log "HA 资源 $sid -> disabled"; ha-manager set "$sid" --state disabled || true
    done
  fi
  if command -v qm >/dev/null 2>&1; then
    for id in $(qm list 2>/dev/null | awk 'NR>1 && $3=="running"{print $1}'); do
      keep_guest "$id" && { log "保留 VM $id 运行"; continue; }
      log "关闭 VM $id ..."; qm shutdown "$id" --timeout 180 --forceStop 1 || qm stop "$id" || true
    done
  fi
  if command -v pct >/dev/null 2>&1; then
    for id in $(pct list 2>/dev/null | awk 'NR>1 && $2=="running"{print $1}'); do
      keep_guest "$id" && { log "保留 CT $id 运行"; continue; }
      log "关闭 CT $id ..."; pct shutdown "$id" --timeout 120 --forceStop 1 || pct stop "$id" || true
    done
  fi
  sleep 2
  log "虚机/容器处理完毕 (onboot 的会在最终重启后自动拉起)"
}

pin_nic_names() {
  mkdir -p /etc/systemd/network
  local p n mac f
  for p in /sys/class/net/*; do
    n=$(basename "$p")
    [ -e "$p/device" ] || continue
    [[ $n =~ ^eth[0-9]+$ ]] && { log "网卡 $n 为内核命名风格,跳过钉名"; continue; }
    mac=$(cat "$p/address" 2>/dev/null) || continue
    { [ -n "$mac" ] && [ "$mac" != "00:00:00:00:00:00" ]; } || continue
    f="/etc/systemd/network/50-$TAG-pin-$n.link"
    [ -e "$f" ] && continue
    printf '[Match]\nMACAddress=%s\n\n[Link]\nName=%s\n' "$mac" "$n" > "$f"
    log "已钉死网卡命名: $n <- $mac"
  done
}

purge_pve6_relics() {
  # PVE 6 时代 pve-kernel-helper 的 efiboot 钩子,升到 7 后调已改名的工具而报错,
  # 连坐所有内核包的 initramfs 生成。7 之前预清,幂等。
  local f
  for f in /etc/initramfs/post-update.d/pve-efiboot-sync \
           /etc/kernel/postinst.d/zz-pve-efiboot /etc/kernel/postrm.d/zz-pve-efiboot; do
    [ -e "$f" ] && { rm -f "$f"; log "已清理 PVE 6 遗留钩子: $f"; }
  done
  return 0
}

#=============================================================== 各阶段
stage_preflight() {
  [ "$(id -u)" = 0 ] || die "需要 root"
  command -v pveversion >/dev/null 2>&1 || die "未找到 pveversion,这不是 PVE 系统?"
  local mj; mj=$(major)
  [ -n "$mj" ] || die "无法解析 pve-manager 版本"
  { [ "$mj" -ge 6 ] && [ "$mj" -le 9 ]; } || die "不支持的起始版本: $(pmver)"
  [ "$(dpkg --print-architecture)" = amd64 ] || die "仅支持 amd64"
  log "起始版本: pve-manager $(pmver), 运行内核 $(uname -r)"

  if is_cluster; then
    if [ "$CLUSTER_OK" = 1 ]; then log "集群编排模式: $(ls /etc/pve/nodes | wc -l) 节点,由协调器负责栅栏与串行重启"
    else die "检测到多节点集群。请直接运行 'bash $SELF' 由 launcher 自动进入协调器模式,勿用 MODE=single。"; fi
  fi
  if [ -e /etc/pve/ceph.conf ] || dpkg -s ceph-mon >/dev/null 2>&1 || dpkg -s ceph-osd >/dev/null 2>&1; then
    die "检测到 Ceph。Ceph 必须按官方序列单独升级,本脚本不处理。"
  fi

  # 源策略: 起点 <9 走受管链 (统一主 sources.list);起点 =9 尊重现有源配置
  if [ "$mj" -lt 9 ]; then kv_set repomode chain; else kv_set repomode native; fi
  log "源模式: $(repo_mode)  (chain=接管为官方无订阅源链;native=沿用你现有的源配置)"

  # 磁盘: 每跨一个大版本预留 2GB,基座 2GB
  local need free_root free_boot
  need=$(( 2048 + (9 - mj) * 2048 ))
  free_root=$(df --output=avail -m / | tail -1 | tr -d ' ')
  if [ "$free_root" -lt "$need" ]; then
    log "根分区可用 ${free_root}MB,需要 ${need}MB。最大目录:"
    du -xh --max-depth=1 / 2>/dev/null | sort -h | tail -6 || true
    die "/ 空间不足,清理或挪走大文件后重跑"
  fi
  if mountpoint -q /boot; then
    free_boot=$(df --output=avail -m /boot | tail -1 | tr -d ' ')
    [ "$free_boot" -ge 300 ] || die "/boot 独立分区仅剩 ${free_boot}MB,先清理旧内核"
  fi

  if command -v dkms >/dev/null 2>&1 && [ -n "$(dkms status 2>/dev/null)" ]; then
    log "警告: 检测到 DKMS 模块,重启进新内核后需能重编译:"; dkms status 2>/dev/null || true
  fi
  wget -q --spider http://download.proxmox.com/debian/pve/dists/trixie/Release \
    || die "无法访问 download.proxmox.com (trixie),检查 DNS/默认网关/出口。若网关是本集群上的 VM,用 KEEP_GUESTS 保住它。"

  mkdir -p "$BK"
  tar czf "$BK/etc-$(date +%Y%m%d%H%M%S).tgz" /etc 2>/dev/null || true
  cp -a /var/lib/pve-cluster/config.db "$BK/config.db.$(date +%s)" 2>/dev/null || true
  { pveversion -v; echo; ip -o addr; echo; dpkg -l; } > "$BK/state-before.txt" 2>&1 || true
  log "已备份 /etc 与 config.db 到 $BK (这不是虚机备份,vzdump 请自行确保!)"

  [ "$(repo_mode)" = chain ] && write_chain_aptconf
  mkdir -p /etc/needrestart/conf.d
  cat > /etc/needrestart/conf.d/99-pve-upgrade.conf <<'EOF'
$nrconf{restart} = 'a';
$nrconf{override_rc}{qr(^networking)} = 0;
$nrconf{override_rc}{qr(^ifup@)} = 0;
1;
EOF

  local do_stop=0
  case "$STOP_GUESTS" in
    1) do_stop=1 ;; 0) do_stop=0 ;;
    *) [ "$mj" -lt 9 ] && do_stop=1 ;;
  esac
  if [ "$do_stop" = 1 ]; then stop_guests
  else log "起点已是 PVE 9 (纯刷新),不关闭虚机;最终重启时由 systemd 正常关停"; fi
  pin_nic_names

  # hold 正在运行的内核包 (任何代际都适用),当回退梯子
  local rk
  rk=$(dpkg -S "/boot/vmlinuz-$(uname -r)" 2>/dev/null | head -1 | cut -d: -f1 || true)
  if [ -n "$rk" ]; then
    apt-mark hold "$rk" >/dev/null && kv_set heldkernel "$rk"
    log "已 hold 运行内核包 $rk (重启验证无误后 unhold + autoremove)"
  fi
}

stage_to64() {
  local mj; mj=$(major)
  [ "$mj" -ge 7 ] && { log "当前已是 PVE $mj,跳过 6.4 步"; return 0; }
  # buster PVE 源已从 download/enterprise 两站下架且无官方归档:
  # 先试;404 则只刷 Debian buster 归档基座,且只做【不移除包】的 upgrade
  # (纯 Debian 源下 dist-upgrade 会试图移除 proxmox-ve,被 pve-apt-hook 拦下)。
  write_sources buster archive
  if apt_update; then dist_upgrade
  else
    log "PVE buster 源已下架 (404),退化为纯 Debian buster 归档基座"
    sed -i '/download\.proxmox\.com/d' /etc/apt/sources.list
    apt_update || die "Debian buster 归档源也不可用"
    repair_dpkg
    "${APT_GET[@]}" upgrade || log "警告: buster 基座更新未完全成功,不阻塞"
    apt-get clean
  fi
  log "==> 6.x 段完成 (当前 $(pmver)),下一跳直取 PVE 7"
}

stage_to7() {
  local mj; mj=$(major)
  [ "$mj" -ge 7 ] && { log "当前已是 PVE $mj,跳过 7 跳"; return 0; }
  purge_pve6_relics
  checker_gate pve6to7
  fetch_key proxmox-release-bullseye.gpg
  switch_repo bullseye
  dist_upgrade
  "${APT_GET[@]}" install ifupdown2 chrony || log "警告: ifupdown2/chrony 安装未完全成功"
  require_major 7
  log "==> 已到 PVE $(pmver) (bullseye)"
}

stage_to8() {
  local mj; mj=$(major)
  [ "$mj" -ge 8 ] && { log "当前已是 PVE $mj,跳过 8 跳"; return 0; }
  checker_gate pve7to8
  fetch_key proxmox-release-bookworm.gpg
  switch_repo bookworm
  dist_upgrade
  require_major 8
  log "==> 已到 PVE $(pmver) (bookworm)"
}

stage_to9() {
  local mj; mj=$(major)
  [ "$mj" -ge 9 ] && { log "当前已是 PVE $mj,跳过 9 跳"; return 0; }
  if dpkg -s systemd-boot >/dev/null 2>&1; then
    "${APT_GET[@]}" remove systemd-boot || log "警告: systemd-boot 元包移除失败"
  fi
  checker_gate pve8to9
  fetch_key proxmox-release-trixie.gpg
  switch_repo trixie
  dist_upgrade
  dpkg -s proxmox-ve >/dev/null 2>&1 || "${APT_GET[@]}" install proxmox-ve
  require_major 9
  log "==> 已到 PVE $(pmver) (trixie)"
}

stage_refresh() {
  # 无论起点几,这里都把 PVE 9 刷到最新点版本 + 选定内核系列
  require_major 9
  if [ "$(repo_mode)" = chain ]; then switch_repo trixie
  else apt_update || die "现有源 apt update 失败 (企业源无订阅?),修好源或改用无订阅源后重跑"; fi
  dist_upgrade
  local lk
  if [ "$LATEST_KERNEL" = 1 ]; then
    lk=$(apt-cache search --names-only '^proxmox-kernel-[0-9]+\.[0-9]+$' 2>/dev/null | awk '{print $1}' | sort -V | tail -1)
    [ -n "$lk" ] && { log "LATEST_KERNEL=1 -> 安装最新内核系列 $lk"; "${APT_GET[@]}" install "$lk" || log "警告: $lk 安装失败"; }
  else
    dpkg -s proxmox-default-kernel >/dev/null 2>&1 || "${APT_GET[@]}" install proxmox-default-kernel || true
  fi
  log "==> PVE $(pmver) 已刷到最新;最新内核: $(newest_kernel) (运行中: $(uname -r))"
}

stage_finalize() {
  purge_pve6_relics
  #--- 内核瘦身: 保留 运行中的系列 + 最新两个 proxmox 系列;Debian 原生内核默认清掉
  local pkg ser runser keep1 keep2 purge=()
  runser=$(uname -r | grep -oE '^[0-9]+\.[0-9]+')
  mapfile -t kpkgs < <(dpkg-query -W -f='${Package} ${db:Status-Abbrev}\n' 'pve-kernel-[0-9]*' 'proxmox-kernel-[0-9]*' 2>/dev/null | awk '$2 ~ /^i/{print $1}')
  mapfile -t allser < <(printf '%s\n' "${kpkgs[@]}" | grep -oE '[0-9]+\.[0-9]+' | sort -uV)
  keep1=""; keep2=""
  [ "${#allser[@]}" -ge 1 ] && keep1="${allser[-1]}"
  [ "${#allser[@]}" -ge 2 ] && keep2="${allser[-2]}"
  for pkg in "${kpkgs[@]}"; do
    ser=$(echo "$pkg" | grep -oE '[0-9]+\.[0-9]+' | head -1); [ -z "$ser" ] && continue
    [ "$ser" = "$runser" ] && continue; [ "$ser" = "$keep1" ] && continue; [ "$ser" = "$keep2" ] && continue
    purge+=("$pkg")
  done
  if [ "$KEEP_DEBIAN_KERNEL" != 1 ] && [ "${#kpkgs[@]}" -gt 0 ]; then
    mapfile -t dpkgs < <(dpkg-query -W -f='${Package} ${db:Status-Abbrev}\n' 'linux-image-*' 2>/dev/null | awk '$2 ~ /^i/{print $1}')
    for pkg in "${dpkgs[@]}"; do
      [ "$pkg" = "linux-image-$(uname -r)" ] && continue   # 正在跑的留到重启后
      purge+=("$pkg")
    done
  fi
  if [ "${#purge[@]}" -gt 0 ]; then
    log "清理内核: ${purge[*]}"
    "${APT_GET[@]}" purge "${purge[@]}" || log "警告: 内核清理未完全成功,重启后可手动处理"
  fi

  #--- 刷引导
  if [ -e /etc/kernel/proxmox-boot-uuids ]; then
    if [ -d /sys/firmware/efi ] && ! command -v bootctl >/dev/null 2>&1; then
      "${APT_GET[@]}" install systemd-boot-tools systemd-boot-efi || true
    fi
    proxmox-boot-tool refresh || die "proxmox-boot-tool refresh 失败,重启前必须解决"
  else
    if [ -d /sys/firmware/efi ]; then
      dpkg -s grub-efi-amd64 >/dev/null 2>&1 || "${APT_GET[@]}" install grub-efi-amd64 || log "警告: grub-efi-amd64 安装失败"
      if [ -e /boot/efi/EFI/BOOT/BOOTX64.efi ]; then
        # 固件走可移动回退路径引导的机器: 不设此标志,grub 更新只刷 EFI/debian 不刷它
        echo 'grub-efi-amd64 grub2/force_efi_extra_removable boolean true' | debconf-set-selections
        "${APT_GET[@]}" install --reinstall grub-efi-amd64 || die "grub-efi-amd64 重装失败,重启前必须解决"
      fi
    fi
    update-grub || die "update-grub 失败,重启前必须解决"
  fi

  #--- 源收尾 (仅受管链模式改写;native 模式不动你的源)
  rm -f /etc/apt/apt.conf.d/99-pve-upgrade-chain
  if [ "$(repo_mode)" = chain ]; then
    disable_foreign_repos
    local sb=/etc/apt/trusted.gpg.d/proxmox-release-trixie.gpg
    "${APT_GET[@]}" install proxmox-archive-keyring || true
    [ -e /usr/share/keyrings/proxmox-archive-keyring.gpg ] && sb=/usr/share/keyrings/proxmox-archive-keyring.gpg
    cat > /etc/apt/sources.list.d/debian.sources <<'EOF'
Types: deb
URIs: http://deb.debian.org/debian
Suites: trixie trixie-updates
Components: main contrib non-free non-free-firmware
Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg

Types: deb
URIs: http://security.debian.org/debian-security
Suites: trixie-security
Components: main contrib non-free non-free-firmware
Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
EOF
    cat > /etc/apt/sources.list.d/proxmox.sources <<EOF
Types: deb
URIs: http://download.proxmox.com/debian/pve
Suites: trixie
Components: pve-no-subscription
Signed-By: $sb
EOF
    echo "# pve-upgrade: 源已迁移至 /etc/apt/sources.list.d/*.sources" > /etc/apt/sources.list
    apt-get update || log "警告: 最终 apt update 未通过,重启后请检查源"
  fi

  #--- 总结
  local nk; nk=$(newest_kernel)
  kv_set newestkernel "$nk"
  log "==================== 升级完成 ===================="
  pveversion -v 2>/dev/null | head -n 6 || true
  if [ "$nk" = "$(uname -r)" ]; then
    log "运行内核已是最新 ($nk):可不重启;若中途更新了 QEMU/系统库,仍建议择机重启一次。"
    kv_set rebootneeded 0
  else
    log "当前运行内核 $(uname -r) -> 重启后进入 $nk (引导菜单保留当前内核作回退)"
    kv_set rebootneeded 1
  fi
  local hk; hk=$(kv_get heldkernel)
  log "重启验证后清理: apt-mark unhold ${hk:-<运行内核包>} && apt autoremove --purge -y"
  [ "$(repo_mode)" = chain ] && log "第三方源已改名 *.off-$TAG,按需改回并适配 trixie"
  if [ "$CLUSTER_OK" = 1 ]; then
    log "集群编排模式: 本节点不自行重启,由协调器串行编排"
  elif [ "$(kv_get rebootneeded)" = 1 ]; then
    if [ "$AUTO_REBOOT" = 1 ]; then log "AUTO_REBOOT=1 -> 20 秒后重启..."; sleep 20; systemctl reboot
    else log ">>> 请执行: bash $SELF reboot   (或直接 reboot) <<<"; fi
  fi
}

#=============================================================== worker
run_stage() {
  local name=$1 fn=$2
  CURRENT_STAGE=$name
  stage_done "$name" && { log "[skip] 阶段 $name 已完成"; return 0; }
  log ">>>>>> 进入阶段: $name"
  "$fn"
  mark_done "$name"
  log "<<<<<< 阶段 $name 完成"
}
worker_main() {
  mkdir -p "$(dirname "$STATE")" "$STATEDIR"
  exec >>"$LOG" 2>&1
  log "############ pve-upgrade worker 启动 (pve-manager $(pmver), kernel $(uname -r), stages=$STAGES) ############"
  want_stage preflight && run_stage preflight stage_preflight
  want_stage to64      && run_stage to64      stage_to64
  want_stage to7       && run_stage to7       stage_to7
  want_stage to8       && run_stage to8       stage_to8
  want_stage to9       && run_stage to9       stage_to9
  want_stage refresh   && run_stage refresh   stage_refresh
  want_stage finalize  && run_stage finalize  stage_finalize
  log "############ 本次点名阶段执行完毕 (stages=$STAGES) ############"
}

#=============================================================== 协调器 (集群)
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
rsh() { local n=$1; shift; ssh "${SSH_OPTS[@]}" "root@$n" "$@"; }
declare -a ALL_NODES=(); declare -A NODE_NAME=(); SELF_TARGET=""

discover_nodes() {
  local myname; myname=$(hostname)
  if [ -n "$NODES" ]; then
    local t nm
    for t in $NODES; do
      nm=$(rsh "$t" hostname 2>/dev/null) || die "无法 ssh 到 $t"
      ALL_NODES+=("$t"); NODE_NAME[$t]=$nm; [ "$nm" = "$myname" ] && SELF_TARGET=$t
    done
  else
    [ -r /etc/pve/corosync.conf ] || die "读不到 /etc/pve/corosync.conf"
    local line nm="" addr=""
    while IFS= read -r line; do
      case "$line" in
        *"node {"*) nm=""; addr="" ;;
        *" name: "*|*$'\t'"name: "*) nm=$(echo "$line" | awk '{print $2}') ;;
        *"ring0_addr: "*)            addr=$(echo "$line" | awk '{print $2}') ;;
        *"}"*) if [ -n "$nm" ] && [ -n "$addr" ]; then
                 ALL_NODES+=("$addr"); NODE_NAME[$addr]=$nm; [ "$nm" = "$myname" ] && SELF_TARGET=$addr; nm=""; addr=""
               fi ;;
      esac
    done < /etc/pve/corosync.conf
  fi
  [ "${#ALL_NODES[@]}" -ge 2 ] || die "只发现 ${#ALL_NODES[@]} 个节点"
  [ -n "$SELF_TARGET" ] || die "本机 ($myname) 不在节点列表里"
  local n; declare -a others=()
  for n in "${ALL_NODES[@]}"; do [ "$n" != "$SELF_TARGET" ] && others+=("$n"); done
  ALL_NODES=("${others[@]}" "$SELF_TARGET")
  log "节点编队: $(for n in "${ALL_NODES[@]}"; do printf '%s(%s) ' "${NODE_NAME[$n]}" "$n"; done) [最后=协调器本机]"
}
for_each() { local n; for n in "${ALL_NODES[@]}"; do "$@" "$n"; done; }
_stop_svc() { local svc=$1 n=$2; rsh "$n" "systemctl disable --now $svc >/dev/null 2>&1 || true"; }

cluster_preflight() {
  local n v
  for n in "${ALL_NODES[@]}"; do
    v=$(rsh "$n" "pveversion 2>/dev/null" | sed -n 's|^pve-manager/\([^/]*\)/.*|\1|p') || true
    [ -n "$v" ] || die "节点 ${NODE_NAME[$n]} ($n) ssh 不通或不是 PVE"
    log "节点 ${NODE_NAME[$n]}: pve-manager $v, kernel $(rsh "$n" uname -r)"
    if rsh "$n" "[ -e /etc/pve/ceph.conf ] || dpkg -s ceph-mon >/dev/null 2>&1 || dpkg -s ceph-osd >/dev/null 2>&1"; then
      die "节点 ${NODE_NAME[$n]} 检测到 Ceph,本方案不适用"
    fi
  done
  pvecm status 2>/dev/null | grep -q 'Quorate:.*Yes' || die "集群当前不 quorate,先恢复健康"
  grep -q ring1_addr /etc/pve/corosync.conf 2>/dev/null || log "提示: corosync 单链路,升级窗口内不要动网络配置"
  if [ -s /etc/pve/vzdump.cron ] && grep -qE '^[0-9]' /etc/pve/vzdump.cron 2>/dev/null; then
    log "警告: 存在定时备份任务,确认升级窗口不会撞上"
  fi
  log "全集群停用 HA (lrm -> crm) 与 pvesr.timer ..."
  for_each _stop_svc pve-ha-lrm; for_each _stop_svc pve-ha-crm; for_each _stop_svc pvesr.timer
  for n in "${ALL_NODES[@]}"; do
    scp "${SSH_OPTS[@]}" -q "$SELF" "root@$n:$REMOTE_SELF"; rsh "$n" "chmod +x $REMOTE_SELF"
  done
  log "脚本已分发到全部节点: $REMOTE_SELF"
}

node_stage_done()   { rsh "$1" "grep -qx '$2' $STATE 2>/dev/null"; }
node_stage_failed() { rsh "$1" "systemctl is-failed --quiet 'pve-upgrade-$2' 2>/dev/null"; }
launch_stage() { # $1 stage  $2 node (for_each 把节点追加在最后)
  local st=$1 n=$2
  node_stage_done "$n" "$st" && { log "  [${NODE_NAME[$n]}] 阶段 $st 已完成 (续跑跳过)"; return 0; }
  rsh "$n" "I_UNDERSTAND=yes BATCH=1 CLUSTER_OK=1 AUTO_REBOOT=0 STAGES=$st \
            LATEST_KERNEL=$LATEST_KERNEL SKIP_CHECKS=$SKIP_CHECKS STOP_GUESTS=$STOP_GUESTS \
            KEEP_DEBIAN_KERNEL=$KEEP_DEBIAN_KERNEL KEEP_GUESTS='$KEEP_GUESTS' \
            bash $REMOTE_SELF" >/dev/null \
    || die "在 ${NODE_NAME[$n]} ($n) 启动阶段 $st 失败"
  log "  [${NODE_NAME[$n]}] 阶段 $st 已启动"
}
barrier_wait() {
  local st=$1 n t0 elapsed pending
  t0=$(date +%s)
  while :; do
    pending=""
    for n in "${ALL_NODES[@]}"; do
      if node_stage_done "$n" "$st"; then continue
      elif node_stage_failed "$n" "$st"; then
        die "节点 ${NODE_NAME[$n]} 阶段 $st 失败。上去看 $LOG,修复后重跑协调器即可续接。"
      else pending+=" ${NODE_NAME[$n]}"; fi
    done
    [ -z "$pending" ] && break
    elapsed=$(( $(date +%s) - t0 ))
    [ "$elapsed" -le "$STAGE_TIMEOUT" ] || die "阶段 $st 超时 (${STAGE_TIMEOUT}s),未完成:$pending"
    log "  栅栏等待 [$st] ${elapsed}s,未完成:$pending"
    sleep 20
  done
  log "== 栅栏通过: 全部节点完成阶段 [$st] =="
}
version_matrix() {
  local n
  for n in "${ALL_NODES[@]}"; do
    log "  ${NODE_NAME[$n]}: $(rsh "$n" "pveversion 2>/dev/null" || echo '?')  kernel=$(rsh "$n" uname -r 2>/dev/null || echo '?')"
  done
}
orchestrate_chain() {
  local st
  for st in "${STAGES_SEQ[@]}"; do
    log "########## 跳: $st (全节点并行) ##########"
    for_each launch_stage "$st"; barrier_wait "$st"; version_matrix
  done
  log "########## 链全部完成,全部节点已是最新 PVE 9 ##########"
}

# node_on_newest: 仅当 ssh 通【且】运行内核 == 最新已装内核时返回 0 (ssh 不通不会被误判为就绪)
node_on_newest()    { rsh "$1" '[ "$(uname -r)" = "$(ls /lib/modules | sort -V | tail -1)" ]'; }
node_needs_reboot() { ! node_on_newest "$1"; }
reboot_one() {
  local n=$1 t0 elapsed mj kr q i
  if node_on_newest "$n"; then
    log "[${NODE_NAME[$n]}] 已在最新内核 ($(rsh "$n" uname -r 2>/dev/null || echo '?')),跳过"; return 0
  fi
  node_stage_done "$n" finalize || die "[${NODE_NAME[$n]}] finalize 未完成,不能重启"
  log "[${NODE_NAME[$n]}] 重启中 ..."
  rsh "$n" "systemctl reboot" || true
  sleep 20; t0=$(date +%s)
  while :; do
    # 全部探测放在条件位: 早期引导阶段的瞬时失败只会让条件为假,不会触发 set -e
    if mj=$(rsh "$n" "pveversion 2>/dev/null" 2>/dev/null | sed -n 's|^pve-manager/\([0-9]*\)\..*|\1|p') \
       && kr=$(rsh "$n" uname -r 2>/dev/null) && [ "$mj" = 9 ] \
       && node_on_newest "$n" \
       && rsh "$n" "systemctl is-active --quiet pve-cluster"; then
      log "[${NODE_NAME[$n]}] 回来了: PVE $mj, kernel $kr, pve-cluster 在跑;等待重新入群 ..."
      q=0
      for i in $(seq 1 30); do
        if pvecm status 2>/dev/null | grep -q 'Quorate:.*Yes'; then q=1; break; fi
        [ $((i % 6)) = 0 ] && log "  仍在等待 quorate ... $((i*5))s"
        sleep 5
      done
      [ "$q" = 1 ] || die "[${NODE_NAME[$n]}] 起来了但集群未恢复 quorate,人工检查后再继续"
      log "[${NODE_NAME[$n]}] 集群 quorate 确认,下一台"; return 0
    fi
    elapsed=$(( $(date +%s) - t0 ))
    [ "$elapsed" -le 900 ] || die "[${NODE_NAME[$n]}] 重启 ${elapsed}s 未恢复。上 IPMI/console 看引导 (引导或网卡命名问题,.link 钉名在 /etc/systemd/network/)"
    log "  等待 [${NODE_NAME[$n]}] 回归 ... ${elapsed}s"; sleep 15
  done
}
orchestrate_reboots() {
  local n first=1
  for n in "${ALL_NODES[@]}"; do
    [ "$n" = "$SELF_TARGET" ] && continue
    [ "$first" = 1 ] && { log ">>>>> 金丝雀节点: ${NODE_NAME[$n]} 先行,验证通过才继续 <<<<<"; first=0; }
    reboot_one "$n"
  done
  if ! node_on_newest "$SELF_TARGET"; then
    node_stage_done "$SELF_TARGET" finalize || die "本机 finalize 未完成,不能重启"
    log "其余节点全部就位。协调器本机 20 秒后自重启,回来后执行: bash $SELF verify"
    sleep 20; systemctl reboot
  else
    log "本机已在最新内核,无需重启。执行 bash $SELF verify 收尾。"
  fi
}
coordinator_main() {
  exec >>"$CLOG" 2>&1
  log "############ pve-upgrade 集群协调器启动 ############"
  discover_nodes; cluster_preflight; orchestrate_chain
  if [ "$AUTO_REBOOT" = 1 ]; then log "AUTO_REBOOT=1 -> 进入串行重启编排"; orchestrate_reboots
  else log ">>> 链已全部完成。人工确认后执行: bash $SELF reboot <<<"; fi
}

#=============================================================== 子命令
cmd_verify() {
  if is_cluster; then
    discover_nodes
    log "=================== 集群升级验收 ==================="; version_matrix
    pvecm status 2>/dev/null | grep -q 'Quorate:.*Yes' && log "集群 Quorate: Yes" || log "!! 集群未 quorate"
  else
    log "=================== 单机升级验收 ==================="
    log "  $(pveversion 2>/dev/null)  最新内核=$(newest_kernel)"
  fi
  log "---------------- 收尾清单 (人工执行,每台) ----------------"
  local hk; hk=$(kv_get heldkernel)
  log "1) 释放回退梯子:  apt-mark unhold ${hk:-<运行内核包>} && apt autoremove --purge -y"
  log "2) 恢复 HA (如在用): systemctl enable --now pve-ha-crm pve-ha-lrm;  恢复复制: systemctl enable --now pvesr.timer"
  log "3) 第三方源已改名 *.off-$TAG (chain 模式),按需改回并适配 trixie"
  log "4) ZFS: 一切正常后才 zpool upgrade (不可逆);老 CT (cgroup v1 时代) 需重建"
  log "5) 网卡钉名文件 /etc/systemd/network/50-$TAG-pin-*.link 是正式契约,别删"
}
cmd_status() {
  local n
  if is_cluster; then
    discover_nodes
    for n in "${ALL_NODES[@]}"; do
      echo "== ${NODE_NAME[$n]} ($n): $(rsh "$n" "pveversion 2>/dev/null; echo kernel=\$(uname -r) newest=\$(ls /lib/modules|sort -V|tail -1); echo done=\$(tr '\n' ' ' < $STATE 2>/dev/null); systemctl list-units --no-legend 'pve-upgrade*' 2>/dev/null" 2>&1 | tr '\n' ' ')"
    done
  else
    echo "$(pveversion 2>/dev/null)  kernel=$(uname -r) newest=$(newest_kernel)"
    echo "done stages: $(tr '\n' ' ' < "$STATE" 2>/dev/null)"
    systemctl list-units --no-legend 'pve-upgrade*' 2>/dev/null || true
  fi
}
cmd_reboot() {
  [ "$(id -u)" = 0 ] || { echo 请用root; exit 1; }
  if is_cluster && [ "$MODE" != single ]; then discover_nodes; orchestrate_reboots
  else
    stage_done finalize || die "finalize 未完成,不能重启"
    [ "$(uname -r)" = "$(newest_kernel)" ] && { echo "已在最新内核,无需重启"; exit 0; }
    echo "20 秒后重启本机 (Ctrl-C 取消) ..."; sleep 20; systemctl reboot
  fi
}

#=============================================================== launcher
confirm() {
  [ "$I_UNDERSTAND" = yes ] && return 0
  if [ -t 0 ]; then
    read -rp "输入 UPGRADE 确认执行: " ans; [ "$ans" = "UPGRADE" ] || { echo "已取消"; exit 1; }
  else echo "非交互环境请加 I_UNDERSTAND=yes"; exit 1; fi
}
launcher() {
  [ "$(id -u)" = 0 ] || { echo "请用 root 运行"; exit 1; }
  local role=single
  if [ "$CLUSTER_OK" != 1 ]; then
    case "$MODE" in
      cluster) role=cluster ;;
      single)  role=single ;;
      *) is_cluster && role=cluster ;;
    esac
  fi
  if [ "$role" = cluster ]; then
    touch "$CLOG"
    if systemctl is-active --quiet "$CUNIT" 2>/dev/null; then echo "协调器已在后台运行,接入日志:"; exec tail -n 50 -f "$CLOG"; fi
    systemctl reset-failed "$CUNIT" 2>/dev/null || true
    echo "=============================================================="
    echo " PVE 集群升级协调器: 并行链 + 每跳栅栏 + 串行金丝雀重启 (本机最后)"
    echo " 目标: 最新 PVE 9 + $([ "$LATEST_KERNEL" = 1 ] && echo 最新内核系列 || echo 官方默认内核)"
    echo " 开关: AUTO_REBOOT=$AUTO_REBOOT LATEST_KERNEL=$LATEST_KERNEL STOP_GUESTS=$STOP_GUESTS KEEP_GUESTS='$KEEP_GUESTS'"
    echo " 前提: 全部节点 vzdump 备份 + IPMI/console 兜底;无 Ceph。日志: $CLOG"
    echo "=============================================================="
    confirm
    systemd-run --unit "$CUNIT" --description "PVE cluster chained upgrade coordinator" \
      --property=StandardOutput=append:"$CLOG" --property=StandardError=append:"$CLOG" \
      --setenv=LATEST_KERNEL="$LATEST_KERNEL" --setenv=SKIP_CHECKS="$SKIP_CHECKS" \
      --setenv=AUTO_REBOOT="$AUTO_REBOOT" --setenv=NODES="$NODES" --setenv=STAGE_TIMEOUT="$STAGE_TIMEOUT" \
      --setenv=STOP_GUESTS="$STOP_GUESTS" --setenv=KEEP_GUESTS="$KEEP_GUESTS" \
      --setenv=KEEP_DEBIAN_KERNEL="$KEEP_DEBIAN_KERNEL" \
      /bin/bash "$SELF" --coordinator
    echo "协调器已在后台单元 [$CUNIT] 启动 (SSH 断开不影响)。跟随日志:"
    exec tail -n +1 -f "$CLOG"
  fi

  # ---- 单机 / 协调器驱动的 worker ----
  touch "$LOG"
  if systemctl is-active --quiet "$UNIT" 2>/dev/null; then
    [ "$BATCH" = 1 ] && { echo "单元 [$UNIT] 已在运行"; exit 0; }
    echo "升级任务已在后台运行,接入日志:"; exec tail -n 50 -f "$LOG"
  fi
  systemctl reset-failed "$UNIT" 2>/dev/null || true
  if [ "$BATCH" != 1 ]; then
    echo "=============================================================="
    echo " PVE 单机一次重启链式升级 -> 最新 PVE 9 + $([ "$LATEST_KERNEL" = 1 ] && echo 最新内核系列 || echo 官方默认内核)"
    echo " 当前: $(pmver 2>/dev/null || echo '?')  内核: $(uname -r)"
    echo " 开关: AUTO_REBOOT=$AUTO_REBOOT LATEST_KERNEL=$LATEST_KERNEL STOP_GUESTS=$STOP_GUESTS KEEP_GUESTS='$KEEP_GUESTS'"
    echo " 前提: vzdump 外部备份 + IPMI/console 兜底;无 Ceph。日志: $LOG"
    echo "=============================================================="
    confirm
  fi
  systemd-run --unit "$UNIT" --description "PVE chained one-reboot upgrade (stages=$STAGES)" \
    --property=StandardOutput=append:"$LOG" --property=StandardError=append:"$LOG" \
    --setenv=AUTO_REBOOT="$AUTO_REBOOT" --setenv=LATEST_KERNEL="$LATEST_KERNEL" \
    --setenv=SKIP_CHECKS="$SKIP_CHECKS" --setenv=CLUSTER_OK="$CLUSTER_OK" --setenv=STAGES="$STAGES" \
    --setenv=STOP_GUESTS="$STOP_GUESTS" --setenv=KEEP_GUESTS="$KEEP_GUESTS" \
    --setenv=KEEP_DEBIAN_KERNEL="$KEEP_DEBIAN_KERNEL" \
    /bin/bash "$SELF" --worker
  if [ "$BATCH" = 1 ]; then echo "已启动后台单元 [$UNIT] (stages=$STAGES),日志: $LOG"; exit 0; fi
  echo "已在后台单元 [$UNIT] 中启动 (SSH 断开不影响)。跟随日志:"
  exec tail -n +1 -f "$LOG"
}

usage() { sed -n '2,/^set -Eeuo/p' "$SELF" | sed '$d'; }

case "${1:-}" in
  --worker)      worker_main ;;
  --coordinator) coordinator_main ;;
  reboot)        cmd_reboot ;;
  verify)        cmd_verify ;;
  status)        cmd_status ;;
  -h|--help)     usage ;;
  *)             launcher ;;
esac