#!/usr/bin/env bash
# ==================================================================
#  DGX Spark (GB10) 二手验机脚本  v2.0
#  dashen.wang
#
#  设计原则：按"致命度"排序。时间不够就从上往下跑，跑到哪算哪。
#  §1供电链异常是高优先级复核项；最终以NVIDIA/OEM官方诊断为准。
#
#  用法：
#    chmod +x spark-check-v2.sh
#    sudo ./spark-check-v2.sh            # 标准，约 12 分钟
#    sudo ./spark-check-v2.sh --quick    # 快速，约 3 分钟（跳过拷机）
#    sudo ./spark-check-v2.sh --full     # 完整，约 25 分钟（含长时间稳定性）
#
#  产出：
#    spark-check-<时间>.log       人读报告
#    spark-check-<时间>.json      机读摘要
#    spark-samples-<时间>.csv     逐秒遥测（机器死机也能事后读）
# ==================================================================

set -uo pipefail

EXIT_PASS=0
EXIT_FATAL=10
EXIT_RISKY=20
EXIT_INCOMPLETE=30
MODE=standard
case "${1:-}" in
  ""|--standard) MODE=standard ;;
  --quick) MODE=quick ;;
  --full) MODE=full ;;
  --help|-h) echo "用法: sudo bash test.sh [--quick|--standard|--full]"; exit 0 ;;
  *) echo "未知参数: ${1}. 用法: sudo bash test.sh [--quick|--standard|--full]" >&2; exit 2 ;;
esac

TS=$(date +%Y%m%d-%H%M%S)
OUT="${DASHEN_DGX_SPARK_OUTPUT_DIR:-$PWD}"
if [[ ! -d "$OUT" || ! -w "$OUT" ]]; then
  echo "报告目录不可写: $OUT" >&2
  exit 2
fi
OUT="$(cd "$OUT" 2>/dev/null && pwd)"
LOG="$OUT/spark-check-${TS}.log"
JSON="$OUT/spark-check-${TS}.json"
CSV="$OUT/spark-samples-${TS}.csv"
WORK=$(mktemp -d -t dashen-dgx-spark.XXXXXX)
BURN_PID=""; SAMPLER=""; SAMPLER2=""; BP=""; CPU_PIDS=()
wait_tracked(){
  local pid
  for pid in "$@"; do
    [[ -n "$pid" ]] && wait "$pid" 2>/dev/null || true
  done
}
cleanup(){
  for pid in "${CPU_PIDS[@]}" "${BURN_PID:-}" "${SAMPLER:-}" "${SAMPLER2:-}" "${BP:-}"; do
    [[ -n "$pid" ]] && kill -- "-${pid}" 2>/dev/null || true
    [[ -n "$pid" ]] && kill "$pid" 2>/dev/null || true
  done
  wait_tracked "${CPU_PIDS[@]}" "${BURN_PID:-}" "${SAMPLER:-}" "${SAMPLER2:-}" "${BP:-}"
  rm -f "$WORK/burn.cu" "$WORK/burn" "$WORK/burn.py" "$WORK/bw.c" "$WORK/bw" "$WORK/kv.bin"
  rmdir "$WORK" 2>/dev/null || true
}
trap cleanup EXIT
trap 'cleanup; trap - EXIT; exit 130' INT
trap 'cleanup; trap - EXIT; exit 143' TERM

exec > >(stdbuf -oL tee -a "$LOG") 2>&1

R=$'\e[31m'; Y=$'\e[33m'; G=$'\e[32m'; C=$'\e[36m'; B=$'\e[1m'; N=$'\e[0m'
declare -a FATAL=() REDS=() WARNS=()
INCOMPLETE=0
declare -A KV=()

fatal(){ FATAL+=("$1"); echo "${R}${B}[致命]${N} $1"; }
red(){   REDS+=("$1");  echo "${R}[红旗]${N} $1"; }
warn(){  WARNS+=("$1"); echo "${Y}[注意]${N} $1"; }
ok(){    echo "${G}[正常]${N} $1"; }
info(){  echo "       $1"; }
sec(){   echo; echo "${C}${B}━━━ $1 ━━━${N}"; }
kv(){    KV["$1"]="$2"; }
have(){  command -v "$1" >/dev/null 2>&1; }

echo "${B}DGX Spark 二手验机  v2.0  |  dashen.wang${N}"
echo "模式: $MODE   时间: $(date)   日志: $LOG"
[[ $EUID -ne 0 ]] && { warn "非 root，部分检测会跳过。建议使用 sudo 重跑。"; INCOMPLETE=1; }

# ==================================================================
sec "§0  工具自检"
# ==================================================================
MISSING=()
for t in nvidia-smi nvme lspci dmidecode fwupdmgr gcc python3 setsid; do
  have "$t" || MISSING+=("$t")
done
if [[ ${#MISSING[@]} -gt 0 ]]; then
  warn "缺少工具: ${MISSING[*]}"
  INCOMPLETE=1
  info "请按DGX OS官方方式预先安装对应工具；本脚本不会自动安装或升级系统。"
else
  ok "核心工具齐备"
fi
have nvcc && ok "nvcc 可用（供电测试用真 CUDA kernel，最准）" \
          || warn "无 nvcc，供电测试降级到 PyTorch 或 Ollama"

# ==================================================================
sec "§1  供电链  ★★★ 最致命，先跑这个"
# ==================================================================
cat <<'TXT'
背景：部分批次 DGX Spark / GX10 出厂带 PD controller 缺陷，USB-C PD 3.1 与
240W 电源的协商静默失败，PMIC 把整机功耗锁死在约 30W。机器能开机、能跑模型，
就是永远上不了二档。刷固件无效，NVIDIA 的结论是硬件缺陷需 RMA。
二手交易的保修资格可能受限，异常结果必须结合官方诊断和当地销售条款复核。

社区/作者现场筛查参考：满载时 SM 频率能否上到 2000 MHz 以上。
这不是 NVIDIA Field Diagnostics 官方判据，受适配器、室温、驱动与负载实现影响。
      健康 ≈ 2400-2500 MHz     故障 ≈ 卡死在 640 MHz 或利用率永远接近 0
TXT
echo

# --- 电源适配器 ---
echo "-- 电源适配器 --"
PDINFO=$(grep -H . /sys/class/typec/port*/power_operation_mode 2>/dev/null)
[[ -n "$PDINFO" ]] && echo "$PDINFO"
for f in /sys/class/power_supply/*/; do
  [[ -f "$f/voltage_now" ]] && echo "  $(basename "$f"): $(cat "$f/voltage_now" 2>/dev/null) uV"
done
info "现场必须确认卖家给的是原装 240W 适配器。NVIDIA 明确写：用别的适配器会降性能、"
info "无法开机或意外关机。缺少原装适配器或线材应视为配件不完整，并影响测试可信度。"

# --- dmesg 供电签名 ---
echo
echo "-- dmesg 供电签名 --"
PWRSIG=$(dmesg 2>/dev/null | grep -iE "insufficient power|power on the PCIe slot|PD .*negotiat|under-?voltage" || true)
if [[ -n "$PWRSIG" ]]; then
  echo "$PWRSIG" | tail -10
  fatal "dmesg 出现供电不足签名 —— 这是 PD 缺陷的直接证据"
  kv power_dmesg fail
else
  ok "本次开机无供电不足报错"
  kv power_dmesg clean
  info "注意：只覆盖本次开机。卖家刚重启过的话看 journalctl -k -b -1 | grep -i 'insufficient power'"
fi
journalctl -k -b -1 2>/dev/null | grep -i "insufficient power" | tail -3

# --- 构造 GPU 负载 ---
echo
echo "-- 满载采样（60 秒）--"
BURN_PID=""
BURN_METHOD=none

if have nvcc; then
  cat > "$WORK/burn.cu" <<'CU'
#include <cstdio>
#include <cstdlib>
#include <ctime>
__global__ void k(float* o, int iters){
    float a = threadIdx.x * 1e-4f, b = 1.0000001f;
    #pragma unroll 8
    for (int i = 0; i < iters; ++i) a = fmaf(a, b, 1e-7f);
    if (a == 1234.5678f) o[blockIdx.x] = a;   // 防优化掉
}
int main(int argc, char** argv){
    int secs = (argc > 1) ? atoi(argv[1]) : 60;
    float* d; if (cudaMalloc(&d, 4096*sizeof(float)) != cudaSuccess) return 2;
    time_t t0 = time(NULL);
    while (time(NULL) - t0 < secs) {
        k<<<4096, 256>>>(d, 200000);
        cudaDeviceSynchronize();
    }
    return 0;
}
CU
  for ARCH in sm_121a sm_121 sm_120 native; do
    if nvcc -O3 -arch=$ARCH -o "$WORK/burn" "$WORK/burn.cu" 2>/dev/null; then
      ok "CUDA burn kernel 编译成功 (arch=$ARCH)"; BURN_METHOD="nvcc/$ARCH"; break
    fi
  done
fi

if [[ "$BURN_METHOD" == none ]] && have python3 && python3 -c "import torch" 2>/dev/null; then
  cat > "$WORK/burn.py" <<'PY'
import torch, time, sys
s = int(sys.argv[1]) if len(sys.argv) > 1 else 60
d = torch.device("cuda")
a = torch.randn(8192, 8192, device=d, dtype=torch.bfloat16)
b = torch.randn(8192, 8192, device=d, dtype=torch.bfloat16)
t = time.time()
while time.time() - t < s:
    c = a @ b
torch.cuda.synchronize()
PY
  BURN_METHOD=torch
  ok "使用 PyTorch bf16 矩阵乘作为负载"
fi

if [[ "$BURN_METHOD" == none ]]; then
  red "无法构造 GPU 负载（无 nvcc / 无 torch）"
  INCOMPLETE=1
  info "退路：让卖家当场用 ollama 跑一个模型，你在旁边看下面的采样。"
  info "  ollama run gpt-oss:20b --verbose  然后随便问一句"
fi

# --- 遥测采样 ---
echo "timestamp,clocks_sm_mhz,power_w,util_pct,temp_c,throttle" > "$CSV"
(
  for i in $(seq 1 75); do
    L=$(nvidia-smi --query-gpu=timestamp,clocks.sm,power.draw,utilization.gpu,temperature.gpu \
        --format=csv,noheader,nounits 2>/dev/null | head -1)
    T=$(nvidia-smi -q -d PERFORMANCE 2>/dev/null \
        | grep -A20 -iE "clocks (event|throttle) reasons" \
        | grep -i "Active" | head -3 | tr -d ' \n' )
    echo "${L},${T:-none}" >> "$CSV"; sync
    sleep 1
  done
) & SAMPLER=$!

sleep 3
case "$BURN_METHOD" in
  nvcc/*) setsid "$WORK/burn" 60 >/dev/null 2>&1 & BURN_PID=$! ;;
  torch)  setsid python3 "$WORK/burn.py" 60 >/dev/null 2>&1 & BURN_PID=$! ;;
esac

# CPU 也压上，才是真实整机功耗场景
for i in $(seq 1 "$(nproc)"); do setsid timeout 60 bash -c 'while :; do :; done' & CPU_PIDS+=("$!"); done

echo "采样中（60s）……满载曲线："
printf "  %-6s %-10s %-9s %-7s %-7s\n" "秒" "SM频率" "功耗" "利用率" "温度"
for t in 10 20 30 40 50 60; do
  sleep 10
  L=$(tail -1 "$CSV")
  printf "  %-6s %-10s %-9s %-7s %-7s\n" "$t" \
    "$(echo "$L" | cut -d, -f2)MHz" "$(echo "$L" | cut -d, -f3)W" \
    "$(echo "$L" | cut -d, -f4)%" "$(echo "$L" | cut -d, -f5)C"
done

[[ -n "$BURN_PID" ]] && kill -- "-$BURN_PID" 2>/dev/null || true
[[ -n "$BURN_PID" ]] && kill "$BURN_PID" 2>/dev/null || true
kill "$SAMPLER" 2>/dev/null || true
wait_tracked "$BURN_PID" "$SAMPLER" "${CPU_PIDS[@]}"
BURN_PID=""; SAMPLER=""; CPU_PIDS=()

# --- 判读 ---
MAXCLK=$(awk -F, 'NR>1 && $2+0>0 {if($2+0>m) m=$2+0} END{print int(m)}' "$CSV")
MAXPWR=$(awk -F, 'NR>1 && $3+0>0 {if($3+0>m) m=$3+0} END{printf "%.1f", m}' "$CSV")
MAXUTL=$(awk -F, 'NR>1 && $4+0>0 {if($4+0>m) m=$4+0} END{print int(m)}' "$CSV")
MAXTMP=$(awk -F, 'NR>1 && $5+0>0 {if($5+0>m) m=$5+0} END{print int(m)}' "$CSV")
kv max_sm_clock_mhz "${MAXCLK:-0}"; kv max_power_w "${MAXPWR:-0}"
kv max_util_pct "${MAXUTL:-0}"; kv max_temp_c "${MAXTMP:-0}"

echo
echo "峰值：SM ${MAXCLK:-?} MHz | 功耗 ${MAXPWR:-?} W | 利用率 ${MAXUTL:-?}% | 温度 ${MAXTMP:-?}C"

if [[ "$BURN_METHOD" != none ]]; then
  if [[ "${MAXCLK:-0}" -ge 2000 && "${MAXUTL:-0}" -ge 85 ]]; then
    ok "供电链健康：满载能上高频"
    kv power_verdict healthy
  elif [[ "${MAXCLK:-0}" -gt 0 && "${MAXCLK:-0}" -lt 1000 ]]; then
    fatal "SM 频率满载只有 ${MAXCLK} MHz（健康值 2400+）→ PD controller 30W 缺陷特征"
    fatal "→ 停止交易判断，先用NVIDIA Field Diagnostics或OEM官方支持复核。"
    kv power_verdict pd_defect
  elif [[ "${MAXUTL:-0}" -lt 50 ]]; then
    red "利用率始终上不去（峰值 ${MAXUTL}%）"
    info "先看驱动版本：550.x + CUDA 12.4 有已知的 5W 卡死 bug，升到 580.x + CUDA 13 可修。"
    info "如果驱动已经是 580.x 还这样，按 PD 硬件缺陷处理。"
    kv power_verdict low_util
  else
    warn "介于健康与故障之间，SM ${MAXCLK} MHz。多跑几次，并检查散热。"
    kv power_verdict marginal
  fi
fi

echo
echo "-- 降频原因 --"
nvidia-smi -q -d PERFORMANCE 2>/dev/null | grep -A16 -iE "clocks (event|throttle) reasons" || info "不可用"
THR=$(grep -c "Active" "$CSV" 2>/dev/null || true); THR=${THR:-0}
[[ "${THR:-0}" -gt 20 ]] && warn "60 秒里有 ${THR} 个采样点在降频"
info "另一个已知问题：满载功耗被压在 ~100W（额定 240W 的不到一半），属散热降频，"
info "环境相关，改善进风可缓解，不是致命缺陷但要砍价。"

# ==================================================================
sec "§2  固件版本"
# ==================================================================
if have fwupdmgr; then
  fwupdmgr get-devices 2>/dev/null | grep -iE "device|version|embedded controller|system firmware|pd |soc|tpm" | head -40
  echo
  info "脚本作者记录的2026年3月参考值（非通用官方判定，OEM可能不同）："
  info "  BIOS v0103 / SOC 0x305 / EC 2.78.18.3 / PD 0x507"
  info "固件明显落后的话，成交后第一件事："
  info "请通过DGX Dashboard按NVIDIA官方指南检查更新；验机脚本不会执行更新。"
  echo
  echo "-- 刷机历史（能看出前任有没有折腾过）--"
  fwupdmgr get-history 2>/dev/null | head -20 || info "无记录"
else
  warn "无 fwupdmgr，固件版本未知"
fi
DRV=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -1)
echo "驱动: ${DRV:-未知}"
kv driver "${DRV:-unknown}"
[[ "$DRV" == 550.* ]] && warn "驱动 550.x 在 GB10 上有已知低功耗卡死 bug，别据此判死刑，先升级"

# ==================================================================
sec "§3  机器身份与配置真伪"
# ==================================================================
echo "架构: $(uname -m)   内核: $(uname -r)"
[[ "$(uname -m)" == "aarch64" ]] || fatal "非 aarch64，这不是 GB10 机器"
[[ -f /etc/dgx-release ]] && cat /etc/dgx-release || warn "无 /etc/dgx-release：重装过或非原厂 DGX OS"
grep PRETTY /etc/os-release 2>/dev/null

if have dmidecode; then
  MF=$(dmidecode -s system-manufacturer 2>/dev/null)
  PN=$(dmidecode -s system-product-name 2>/dev/null)
  SN=$(dmidecode -s system-serial-number 2>/dev/null | tr -d ' ')
  echo "厂商: ${MF}"; echo "型号: ${PN}"; echo "序列号: ${B}${SN}${N}"
  kv manufacturer "$MF"; kv product "$PN"; kv serial "$SN"
  if [[ ! "$MF" =~ [Nn][Vv][Ii][Dd] ]]; then
    warn "非 NVIDIA 公版 → OEM 版（华硕 GX10 / 戴尔 / 微星 / 技嘉）"
    warn "OEM 版走 OEM 保修，且已有用户报告 OEM 渠道机保修查不到。定价必须低于公版。"
  fi
fi

echo
echo "-- CPU（应为 20 核：10x Cortex-X925 + 10x Cortex-A725）--"
NC=$(nproc); echo "核心数: $NC"
kv cpu_cores "$NC"
[[ "$NC" -ne 20 ]] && red "核心数 $NC ≠ 20，配置对不上"
lscpu 2>/dev/null | grep -iE "model name|cpu\(s\)|part number" | head -5

echo
echo "-- GPU --"
nvidia-smi --query-gpu=name,uuid,driver_version --format=csv 2>/dev/null
if have nvidia-smi; then
  NAME=$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1)
  [[ "$NAME" == *GB10* ]] && ok "GB10 识别正常" || red "GPU 名称异常: $NAME"
fi
info "预期规格：48 SM / 6144 CUDA 核 / 最高 2.42 GHz / 计算能力 12.1 (sm_121)"

# ==================================================================
sec "§4  内存"
# ==================================================================
echo "-- 容量 --"
free -g | head -2
MEMG=$(free -g | awk '/^Mem:/{print $2}')
kv mem_gib "${MEMG:-0}"
echo "可见 ${MEMG} GiB"
info "128GB 机器 deviceQuery 报约 122,570 MB，free 报约 119-121 GiB，都正常。"
[[ "${MEMG:-0}" -lt 110 ]] && fatal "内存显著低于 128GB → 有 die 被屏蔽，或根本不是这台机器"
[[ "${MEMG:-0}" -ge 110 ]] && ok "容量正常"

echo
info "提醒：nvidia-smi 在 GB10 上显示 Memory-Usage: Not Supported 是官方预期行为"
info "（iGPU 无独立显存），不是故障。真实可用内存看 free 的 available 列。"

echo
echo "-- 内存报错 --"
MCE=$(dmesg 2>/dev/null | grep -Eci "machine check|hardware error|EDAC|uncorrectable" || true); MCE=${MCE:-0}
if [[ "${MCE:-0}" -gt 0 ]]; then
  fatal "内核报告内存/总线硬件错误 ${MCE} 条 —— LPDDR5X 焊死在 GB10 封装上，无维修路径"
  dmesg | grep -Ei "machine check|hardware error|EDAC|uncorrectable" | tail -5
else
  ok "本次开机无内存硬件报错"
fi

echo
echo "-- 带宽（理论 273 GB/s，16 通道 256-bit LPDDR5X 8533）--"
if have gcc; then
  cat > "$WORK/bw.c" <<'C'
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#define N (300L*1024*1024)
int main(){
  double *a=aligned_alloc(64,N*sizeof(double));
  double *b=aligned_alloc(64,N*sizeof(double));
  double *c=aligned_alloc(64,N*sizeof(double));
  if(!a||!b||!c){printf("  分配失败\n");return 1;}
  #pragma omp parallel for
  for(long i=0;i<N;i++){a[i]=1.0;b[i]=2.0;c[i]=0.0;}
  struct timespec t0,t1; double best=1e30;
  for(int r=0;r<5;r++){
    clock_gettime(CLOCK_MONOTONIC,&t0);
    #pragma omp parallel for
    for(long i=0;i<N;i++) c[i]=a[i]+3.0*b[i];
    clock_gettime(CLOCK_MONOTONIC,&t1);
    double s=(t1.tv_sec-t0.tv_sec)+(t1.tv_nsec-t0.tv_nsec)/1e9;
    if(s<best) best=s;
  }
  printf("%.1f\n", 3.0*N*sizeof(double)/best/1e9);
  return 0;
}
C
  gcc -O3 -march=native -fopenmp -o "$WORK/bw" "$WORK/bw.c" 2>/dev/null || \
  gcc -O3 -fopenmp -o "$WORK/bw" "$WORK/bw.c" 2>/dev/null || \
  gcc -O3 -o "$WORK/bw" "$WORK/bw.c" 2>/dev/null
  if [[ -x "$WORK/bw" ]]; then
    BW=$("$WORK/bw")
    echo "STREAM Triad: ${BW} GB/s"
    kv stream_gbs "$BW"
    awk -v b="$BW" 'BEGIN{ if(b<100) exit 1; else if(b<160) exit 2; else exit 0 }'
    case $? in
      0) ok "CPU 侧带宽正常" ;;
      2) warn "带宽偏低（${BW} GB/s），可能在降频或内存配置异常" ;;
      1) red "带宽严重偏低（${BW} GB/s），追问原因" ;;
    esac
  fi
else
  warn "无 gcc，带宽测试跳过"
fi

if [[ "$MODE" == full ]]; then
  if have stress-ng; then
    echo
    echo "-- 内存压力（8GB × 2 分钟）--"
    stress-ng --vm 4 --vm-bytes 2G --vm-method all --timeout 120s --metrics-brief 2>&1 | tail -8
  else
    warn "Full模式缺少stress-ng，内存压力项目未执行"
    INCOMPLETE=1
  fi
fi

# ==================================================================
sec "§5  硬盘履历  ★★ 唯一撒不了谎的里程表"
# ==================================================================
NVME=$(ls /dev/nvme?n1 2>/dev/null | head -1)
if [[ -n "${NVME:-}" ]] && have nvme; then
  nvme id-ctrl "$NVME" 2>/dev/null | grep -E "^(mn|sn|fr|tnvmcap|oacs)"
  echo
  S=$(nvme smart-log "$NVME" 2>/dev/null)
  echo "$S" | grep -Ei "critical_warning|percentage_used|power_on_hours|data_units_written|media_errors|unsafe_shutdowns|power_cycles|temperature"

  POH=$(echo "$S" | awk -F: '/power_on_hours/{gsub(/[ ,]/,"",$2);print $2+0}')
  PU=$(echo "$S"  | awk -F: '/percentage_used/{gsub(/[ %,]/,"",$2);print $2+0}')
  DUW=$(echo "$S" | awk -F: '/data_units_written/{gsub(/[ ,]/,"",$2);print $2+0}')
  ME=$(echo "$S"  | awk -F: '/media_errors/{gsub(/[ ,]/,"",$2);print $2+0}')
  CW=$(echo "$S"  | awk -F: '/critical_warning/{gsub(/[ ,]/,"",$2);print $2+0}')
  US=$(echo "$S"  | awk -F: '/unsafe_shutdowns/{gsub(/[ ,]/,"",$2);print $2+0}')
  kv nvme_power_on_hours "${POH:-0}"; kv nvme_pct_used "${PU:-0}"
  kv nvme_unsafe_shutdowns "${US:-0}"

  echo
  [[ -n "${POH:-}" ]] && echo "→ 通电 ${B}${POH}${N} 小时 ≈ 每天 8 小时用了 $((POH/8)) 天"
  [[ -n "${DUW:-}" ]] && echo "→ 累计写入 ≈ $((DUW/2/1024/1024)) TB"
  [[ "${ME:-0}" -gt 0 ]] && red "media_errors=${ME}，颗粒已出错"
  [[ "${CW:-0}" -gt 0 ]] && red "critical_warning=${CW}"
  [[ "${PU:-0}" -gt 5 ]] && warn "寿命已用 ${PU}%"
  [[ "${POH:-0}" -gt 2000 ]] && warn "通电超 2000 小时，这是台干过活的机器，不是'偶尔玩玩'"
  if [[ "${US:-0}" -gt 20 ]]; then
    red "非正常关机 ${US} 次 —— 与'满载死机/自动重启'的已知故障高度吻合，重点追问"
  fi

  CAP=$(lsblk -bdno SIZE "$NVME" 2>/dev/null)
  if [[ -n "$CAP" ]]; then
    CAPG=$((CAP/1000/1000/1000))
    echo "→ 容量 ${CAPG} GB"
    kv nvme_gb "$CAPG"
    [[ "$CAPG" -lt 3500 ]] && warn "1TB 版本。官方配置是 1TB 或 4TB 两种，1TB 必须比 4TB 便宜一大截。"
  fi

  echo
  echo "-- 自加密状态（SED / OPAL）--"
  nvme id-ctrl "$NVME" 2>/dev/null | grep -i "oacs"
  have sedutil-cli && sedutil-cli --scan 2>/dev/null | head -5
  info "官方 SSD 带自加密。若前任启用过 OPAL 锁而不解锁，你重装会撞墙。"
  info "成交前让他解锁 / 恢复出厂，你当场重装一次系统验证。"
elif have smartctl; then
  smartctl -a "${NVME:-/dev/nvme0}" 2>/dev/null | \
    grep -Ei "Model|Serial|Power On Hours|Percentage Used|Data Units Written|Media and Data|Unsafe"
else
  red "无 nvme-cli/smartmontools —— 这是最重要的一节，务必想办法装上再验机"
fi

echo
echo "-- 抹盘取证：文件系统创建时间 vs 通电小时 --"
ROOTDEV=$(findmnt -no SOURCE / 2>/dev/null)
if have tune2fs && [[ -n "$ROOTDEV" ]]; then
  tune2fs -l "$ROOTDEV" 2>/dev/null | grep -iE "created|mount count|last mounted"
fi
info "文件系统是最近几天建的、而通电小时数四位数 = 卖前刚抹盘。"
info "抹盘本身不违法，但'几乎没用过'这句话作废，可以据此砍价。"

echo
echo "-- 开机历史 --"
if have journalctl; then
  NB=$(journalctl --list-boots 2>/dev/null | wc -l)
  echo "journal 记录开机 ${NB} 次"
  journalctl --list-boots 2>/dev/null | head -3
  echo "  ..."
  journalctl --list-boots 2>/dev/null | tail -3
fi

# ==================================================================
sec "§6  网络接口"
# ==================================================================
echo "-- ConnectX-7（公版两个 QSFP）--"
CX=$(lspci 2>/dev/null | grep -iE "mellanox|connectx")
if [[ -n "$CX" ]]; then echo "$CX"; ok "ConnectX-7 已识别"
else red "未识别到 ConnectX-7"; fi
have ibdev2netdev && ibdev2netdev 2>/dev/null
have mlxfwmanager && mlxfwmanager --query 2>/dev/null | grep -Ei "device type|part number|psid|^ *FW" | head -8
have mlxlink && info "有 mlxlink：可用 mlxlink -d <dev> 看链路误码"

echo
echo "-- 全部网口 --"
ip -br link 2>/dev/null
info "应该看到：1 个 10GbE RJ45 + 2 个 QSFP (ConnectX-7) + Wi-Fi 7 + 蓝牙 5.4"
echo
echo "-- 无线 --"
have rfkill && rfkill list 2>/dev/null | head -8
lspci 2>/dev/null | grep -iE "network|wireless" | head -3
have bluetoothctl && (echo "list"; sleep 1) | bluetoothctl 2>/dev/null | head -3

# ==================================================================
sec "§7  持续稳定性  ★★ 已知的高频故障"
# ==================================================================
cat <<'TXT'
论坛上反复出现的三类崩溃：
  · 持续推理几分钟后整机硬死（PowerStress 报温度超限或热传感器损坏）
  · 每 20-30 分钟自动重启（UEFI watchdog 触发）
  · 开机黑屏 + 机身发烫，只能长按电源键
TXT
echo
if [[ "$MODE" == quick ]]; then
  warn "quick 模式跳过。这一节能抓到最贵的坑，时间允许务必跑。"
else
  DUR=300; [[ "$MODE" == full ]] && DUR=1200
  echo "持续拷机 $((DUR/60)) 分钟。遥测逐秒写入 ${CSV}（每次调用sync请求落盘），"
  echo "机器中途死机的话，重启后读这个 CSV 就知道死在什么温度和功耗上。"
  echo

  ( for i in $(seq 1 $((DUR+30))); do
      L=$(nvidia-smi --query-gpu=timestamp,clocks.sm,power.draw,utilization.gpu,temperature.gpu \
          --format=csv,noheader,nounits 2>/dev/null | head -1)
      echo "${L},stability" >> "$CSV"; sync
      sleep 1
    done ) & SAMPLER2=$!

  case "$BURN_METHOD" in
    nvcc/*) setsid "$WORK/burn" "$DUR" >/dev/null 2>&1 & BP=$! ;;
    torch)  setsid python3 "$WORK/burn.py" "$DUR" >/dev/null 2>&1 & BP=$! ;;
    *)      BP="" ;;
  esac
  for i in $(seq 1 "$(nproc)"); do setsid timeout "$DUR" bash -c 'while :; do :; done' & CPU_PIDS+=("$!"); done

  STEP=$((DUR/10))
  printf "  %-7s %-9s %-8s %-7s\n" "分钟" "SM频率" "功耗" "温度"
  for i in $(seq 1 10); do
    sleep "$STEP"
    L=$(tail -1 "$CSV")
    printf "  %-7s %-9s %-8s %-7s\n" \
      "$(awk -v a=$i -v s=$STEP 'BEGIN{printf "%.1f", a*s/60}')" \
      "$(echo "$L" | cut -d, -f2)MHz" "$(echo "$L" | cut -d, -f3)W" "$(echo "$L" | cut -d, -f5)C"
  done
  [[ -n "${BP:-}" ]] && kill -- "-$BP" 2>/dev/null || true
  [[ -n "${BP:-}" ]] && kill "$BP" 2>/dev/null || true
  kill "$SAMPLER2" 2>/dev/null || true
  wait_tracked "$BP" "$SAMPLER2" "${CPU_PIDS[@]}"
  BP=""; SAMPLER2=""; CPU_PIDS=()

  ok "本轮持续负载结束；这只代表本次测试期间未中断"

  ETMP=$(awk -F, '$6=="stability" && $5+0>0 {if($5+0>m) m=$5+0} END{print int(m)}' "$CSV")
  ECLK=$(awk -F, '$6=="stability" && $2+0>0 {if(n==0||$2+0<mn) mn=$2+0; n++} END{print int(mn)}' "$CSV")
  echo "长时间峰值温度: ${ETMP}C   最低 SM 频率: ${ECLK} MHz"
  kv sustained_max_temp "${ETMP:-0}"; kv sustained_min_clock "${ECLK:-0}"
  [[ "${ETMP:-0}" -gt 88 ]] && warn "持续温度 ${ETMP}C 偏高。官方理想环境温度是 5-30C，室温也要一起看。"
  [[ -n "${MAXCLK:-}" && "${ECLK:-0}" -lt $((MAXCLK*60/100)) ]] && \
    warn "长时间后频率掉到峰值的 $((ECLK*100/MAXCLK))%，散热衰减明显"

  echo
  echo "-- 拷机后再查内核 --"
  dmesg 2>/dev/null | tail -30 | grep -iE "error|fail|thermal|throttl|xid|reset" | tail -10 || info "干净"
fi

# ==================================================================
sec "§8  真实推理吞吐"
# ==================================================================
cat <<'TXT'
社区实测基线（健康机，CES 2026 软件更新后）：
  gpt-oss-120b  MXFP4  llama.cpp   tg ≈ 59 tok/s   pp2048 ≈ 1900 tok/s
  gpt-oss-120b  MXFP4  SGLang      ≈ 52 tok/s
  gpt-oss-120b  MXFP4  Ollama      ≈ 38-41 tok/s
  gpt-oss-20b   MXFP4  llama.cpp   ≈ 61 tok/s
  Gemma 4 26B-A4B FP8  vLLM        ≈ 52 tok/s

跑出来只有基线的三到五成，且 §1 显示频率上不去 → 供电缺陷。
TXT
echo
if [[ "$MODE" == quick ]]; then
  info "quick 模式跳过 Ollama 真实推理；需要时运行 Standard 或 Full。"
elif have ollama && ollama list 2>/dev/null | grep -q .; then
  echo "卖家机器上已有的模型："
  ollama list 2>/dev/null | head -10
  M=$(ollama list 2>/dev/null | awk 'NR==2{print $1}')
  if [[ -n "$M" ]]; then
    echo
    echo "用 $M 实测（--verbose 会打印 eval rate）："
    echo "写一段 300 字的说明文。" | timeout 300 ollama run "$M" --verbose 2>&1 | tail -12
  fi
else
  warn "无 ollama 或无已下载模型"
  info "对策：验机前自己在 U 盘上准备一个 gpt-oss-20b 的 GGUF（约 12GB），"
  info "现场用 llama-bench 跑，或者要求卖家提前把模型下好、当着你的面跑一次。"
  info "  llama-bench -m model.gguf -fa 1 -p 2048 -n 32 -ub 2048"
fi

# ==================================================================
sec "§9  前任痕迹与交割"
# ==================================================================
echo "-- 用户账号 --"
awk -F: '$3>=1000 && $3<65534 {print "  "$1"  home="$6}' /etc/passwd

echo
echo "-- 未清理的配置 --"
FOUND=0
for p in /home/*/.ssh /home/*/.aws /home/*/.ngc /home/*/.config/nvidia \
         /home/*/.cache/huggingface /home/*/.docker /home/*/.ollama /etc/nvsm; do
  [[ -e $p ]] && { echo "  存在: $p"; FOUND=1; }
done
[[ $FOUND -eq 1 ]] && warn "机器没恢复出厂，前任数据还在" || ok "较干净"

have docker && { echo; echo "-- Docker --"; docker images 2>/dev/null | head -6; }

echo
cat <<'TXT'
交割清单（脚本管不了，人来管）：
  □ 原装 240W 电源适配器 + 原装 USB-C 线    ← 官方明确要求，缺一件就是缺性能
  □ 让卖家当场退出 NVIDIA Sync / NGC 账号
  □ 让卖家当场执行 DGX OS 恢复出厂
  □ 你自己重装一遍系统，验证 SSD 没被 OPAL 锁
  □ 拿序列号上 nvidia.com 查激活日期与地区，对照卖家说法
  □ 外壳螺丝有无花、有无拆机撬痕
  □ 走闲鱼担保或验货宝，不接受直接转账
TXT

# ==================================================================
sec "结论"
# ==================================================================
if [[ ${#FATAL[@]} -gt 0 ]]; then
  echo "${R}${B}致命问题 ${#FATAL[@]} 项 —— 建议放弃这台机器：${N}"
  for f in "${FATAL[@]}"; do echo "  ✗ $f"; done
  VERDICT=abort
elif [[ ${#REDS[@]} -gt 0 ]]; then
  echo "${R}红旗 ${#REDS[@]} 项 —— 需要卖家给出解释，否则大幅压价：${N}"
  for f in "${REDS[@]}"; do echo "  ! $f"; done
  VERDICT=risky
elif [[ $INCOMPLETE -gt 0 ]]; then
  echo "${Y}${B}检测不完整。不能把跳过项目当成通过。${N}"
  VERDICT=incomplete
else
  echo "${G}${B}未发现致命问题。${N}"
  VERDICT=pass
fi
if [[ ${#WARNS[@]} -gt 0 ]]; then
  echo
  echo "${Y}注意 ${#WARNS[@]} 项（议价筹码）：${N}"
  for f in "${WARNS[@]}"; do echo "  · $f"; done
fi

cat <<'TXT'

最后提醒一句，跟检测结果无关：
NVIDIA 美国 Founders Edition 官方有限保修写明仅原始所有者、不可转让；
OEM 版和其他地区政策可能不同。成交前必须拿序列号向当地销售方或 OEM 书面确认，
不要把卖家的口头承诺当成保修凭证。
TXT

# --- JSON与退出码 ---
if [[ ${#FATAL[@]} -gt 0 ]]; then EXIT_CODE=$EXIT_FATAL
elif [[ ${#REDS[@]} -gt 0 ]]; then EXIT_CODE=$EXIT_RISKY
elif [[ $INCOMPLETE -gt 0 ]]; then EXIT_CODE=$EXIT_INCOMPLETE
else EXIT_CODE=$EXIT_PASS
fi
for k in "${!KV[@]}"; do printf '%s\0%s\0' "$k" "${KV[$k]}"; done > "$WORK/kv.bin"
python3 - "$JSON" "$WORK/kv.bin" "$(date -Iseconds)" "$VERDICT" "$EXIT_CODE" \
  "${#FATAL[@]}" "${#REDS[@]}" "${#WARNS[@]}" "$LOG" <<'PY'
import json,sys
out,kv_path,timestamp,verdict,exit_code,fatal_count,red_count,warn_count,log=sys.argv[1:]
raw=open(kv_path,'rb').read().split(b'\0')
if raw and raw[-1]==b'': raw.pop()
kv={raw[i].decode('utf-8','replace'):raw[i+1].decode('utf-8','replace') for i in range(0,len(raw),2)}
data={'timestamp':timestamp,'verdict':verdict,'exit_code':int(exit_code),'fatal_count':int(fatal_count),'red_count':int(red_count),'warn_count':int(warn_count),**kv,'log':log}
with open(out,'w',encoding='utf-8') as f: json.dump(data,f,ensure_ascii=False,indent=2); f.write('\n')
PY

echo
echo "报告: $LOG"
echo "摘要: $JSON"
echo "遥测: $CSV"
echo "退出码: $EXIT_CODE（0通过 / 10致命 / 20红旗 / 30检测不完整）"
exit "$EXIT_CODE"