Initial commit: CMP 170HX per-card check tooling
Add check_cards.py CLI and cxlib.py host helpers for preparing, unlocking, and PCIe-checking NVIDIA CMP 170HX cards, plus operator README, agent docs, and bundled nvflash tooling.
This commit is contained in:
@@ -0,0 +1,884 @@
|
||||
#!/usr/bin/env python3
|
||||
"""CMP 170HX setup, unlock, PCIe checks, and optional diagnostics."""
|
||||
import datetime
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
HARNESS_DIR = SCRIPT_DIR / "harness"
|
||||
CARDS_DIR = SCRIPT_DIR / "cards"
|
||||
UNIDENTIFIED_DIR = SCRIPT_DIR / "unidentified"
|
||||
|
||||
UNLOCKER_COMMIT = "f51da030ef4260656288edf07e95ae45f339aea8" # upstream v0.3
|
||||
UNLOCKER_ZIP_SHA256 = "a10b226ed4667fb2ae4c9c5ad1575938a85ceb707f8c40ea146887778930ce21"
|
||||
REPOS = {"cmpunlocker": ("commits", UNLOCKER_COMMIT)}
|
||||
VERIFY_SH = HARNESS_DIR / "cmpunlocker" / "verify.sh"
|
||||
INSTALL_SH = HARNESS_DIR / "cmpunlocker" / "install.sh"
|
||||
PENDING_FILE = HARNESS_DIR / "unlock_pending.json"
|
||||
GPU_IDS = ("20c2", "2082")
|
||||
|
||||
SPEED_TO_GEN = {"2.5GT/s": 1, "5GT/s": 2, "8GT/s": 3, "16GT/s": 4, "32GT/s": 5, "64GT/s": 6}
|
||||
|
||||
USE_COLOR = sys.stdout.isatty()
|
||||
|
||||
|
||||
class C:
|
||||
RESET = "\033[0m" if USE_COLOR else ""
|
||||
BOLD = "\033[1m" if USE_COLOR else ""
|
||||
DIM = "\033[2m" if USE_COLOR else ""
|
||||
GREEN = "\033[32m" if USE_COLOR else ""
|
||||
YELLOW = "\033[33m" if USE_COLOR else ""
|
||||
RED = "\033[31m" if USE_COLOR else ""
|
||||
CYAN = "\033[36m" if USE_COLOR else ""
|
||||
|
||||
|
||||
def info(msg):
|
||||
print(f"{C.CYAN}[*]{C.RESET} {msg}", flush=True)
|
||||
|
||||
|
||||
def ok(msg):
|
||||
print(f"{C.GREEN}[+]{C.RESET} {msg}", flush=True)
|
||||
|
||||
|
||||
def warn(msg):
|
||||
print(f"{C.YELLOW}[!]{C.RESET} {msg}", flush=True)
|
||||
|
||||
|
||||
def err(msg):
|
||||
print(f"{C.RED}[ERROR] {msg}{C.RESET}", flush=True)
|
||||
|
||||
|
||||
def step(msg):
|
||||
print(f"\n{C.BOLD}== {msg} =={C.RESET}", flush=True)
|
||||
|
||||
|
||||
def ask(prompt, default=""):
|
||||
suffix = f" [{default}]" if default else ""
|
||||
try:
|
||||
val = input(f"{C.BOLD}?{C.RESET} {prompt}{suffix}: ").strip()
|
||||
except EOFError:
|
||||
return default
|
||||
return val if val else default
|
||||
|
||||
|
||||
def now_iso():
|
||||
return datetime.datetime.now().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def sanitize(name):
|
||||
return re.sub(r"[^A-Za-z0-9._-]+", "_", name).strip("._-") or "UNKNOWN"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- subprocess
|
||||
|
||||
def run(cmd, timeout=60, env=None, cwd=None):
|
||||
try:
|
||||
p = subprocess.run(cmd, capture_output=True, text=True, errors="replace",
|
||||
timeout=timeout, env=env, cwd=cwd)
|
||||
return {"rc": p.returncode, "out": p.stdout, "err": p.stderr, "cmd": cmd}
|
||||
except FileNotFoundError:
|
||||
return {"rc": 127, "out": "", "err": f"not found: {cmd[0]}", "cmd": cmd}
|
||||
except subprocess.TimeoutExpired as e:
|
||||
out = e.stdout.decode(errors="replace") if isinstance(e.stdout, bytes) else (e.stdout or "")
|
||||
return {"rc": 124, "out": out, "err": "timeout", "cmd": cmd}
|
||||
|
||||
|
||||
def as_root(cmd, timeout=60, cwd=None):
|
||||
if os.geteuid() == 0:
|
||||
return run(cmd, timeout=timeout, cwd=cwd)
|
||||
if shutil.which("sudo"):
|
||||
r = run(["sudo", "-n"] + cmd, timeout=timeout, cwd=cwd)
|
||||
if r["rc"] != 0 and "password is required" in (r["err"] + r["out"]).lower():
|
||||
r["err"] = "sudo requires a password (sudo -n failed)"
|
||||
return r
|
||||
return {"rc": 1, "out": "", "err": "no root: neither uid 0 nor sudo available", "cmd": cmd}
|
||||
|
||||
|
||||
def run_logged_as_root(cmd, log_path, timeout, cwd=None):
|
||||
"""Show a long-running command live while keeping its full output on disk."""
|
||||
actual = cmd if os.geteuid() == 0 else ["sudo", "-n"] + cmd
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tail = bytearray()
|
||||
try:
|
||||
with log_path.open("wb") as log:
|
||||
process = subprocess.Popen(actual, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, cwd=cwd,
|
||||
start_new_session=True)
|
||||
|
||||
def copy_output():
|
||||
show_console = True
|
||||
while True:
|
||||
try:
|
||||
chunk = os.read(process.stdout.fileno(), 4096)
|
||||
except OSError:
|
||||
break
|
||||
if not chunk:
|
||||
break
|
||||
log.write(chunk)
|
||||
log.flush()
|
||||
if show_console:
|
||||
try:
|
||||
if hasattr(sys.stdout, "buffer"):
|
||||
sys.stdout.buffer.write(chunk)
|
||||
else:
|
||||
sys.stdout.write(chunk.decode(errors="replace"))
|
||||
sys.stdout.flush()
|
||||
except BrokenPipeError:
|
||||
show_console = False
|
||||
tail.extend(chunk)
|
||||
if len(tail) > 4096:
|
||||
del tail[:-4096]
|
||||
|
||||
def stop_group(sig):
|
||||
try:
|
||||
os.killpg(process.pid, sig)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
reader = threading.Thread(target=copy_output, daemon=True)
|
||||
reader.start()
|
||||
try:
|
||||
rc = process.wait(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
stop_group(signal.SIGTERM)
|
||||
try:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
stop_group(signal.SIGKILL)
|
||||
process.wait()
|
||||
rc = 124
|
||||
except KeyboardInterrupt:
|
||||
stop_group(signal.SIGTERM)
|
||||
try:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
stop_group(signal.SIGKILL)
|
||||
process.wait()
|
||||
raise
|
||||
finally:
|
||||
reader.join(timeout=10)
|
||||
if reader.is_alive():
|
||||
stop_group(signal.SIGTERM)
|
||||
reader.join(timeout=5)
|
||||
if reader.is_alive():
|
||||
stop_group(signal.SIGKILL)
|
||||
reader.join(timeout=5)
|
||||
process.stdout.close()
|
||||
reader.join(timeout=2)
|
||||
except OSError as exc:
|
||||
return {"rc": 127, "out": "", "err": str(exc), "cmd": actual,
|
||||
"log": str(log_path)}
|
||||
return {"rc": rc, "out": tail.decode(errors="replace"), "err": "",
|
||||
"cmd": actual, "log": str(log_path)}
|
||||
|
||||
|
||||
def which(name):
|
||||
return shutil.which(name)
|
||||
|
||||
|
||||
def find_nvflash():
|
||||
"""nvflash binary: repo-bundled copy first (no system install needed), then PATH."""
|
||||
import platform
|
||||
arch_map = {"x86_64": "x64", "amd64": "x64", "aarch64": "aarch64",
|
||||
"ppc64le": "ppc64", "ppc64": "ppc64", "i686": "x86", "i386": "x86"}
|
||||
arch = arch_map.get(platform.machine(), "x64")
|
||||
candidates = [
|
||||
*sorted(SCRIPT_DIR.glob(f"nvflash_*_linux/{arch}/nvflash")),
|
||||
*sorted(SCRIPT_DIR.glob("nvflash_*_linux/x64/nvflash")),
|
||||
SCRIPT_DIR / "nvflash",
|
||||
]
|
||||
for p in candidates:
|
||||
if p.is_file() and os.access(p, os.X_OK):
|
||||
return str(p)
|
||||
return which("nvflash")
|
||||
|
||||
|
||||
def have_root():
|
||||
return os.geteuid() == 0 or bool(shutil.which("sudo"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- preflight
|
||||
|
||||
def ensure_repos():
|
||||
"""Fetch upstream source archives without changing any Git remotes."""
|
||||
HARNESS_DIR.mkdir(exist_ok=True)
|
||||
complete = True
|
||||
for name, (kind, ref) in REPOS.items():
|
||||
dest = HARNESS_DIR / name
|
||||
if (dest / ("install.sh" if name == "cmpunlocker" else "README.md")).exists():
|
||||
continue
|
||||
suffix = f"archive/{ref}.zip" if kind == "commits" else f"archive/refs/{kind}/{ref}.zip"
|
||||
url = f"https://github.com/amoghmunikote/{name}/{suffix}"
|
||||
info(f"downloading {name} ({ref}) ...")
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=60) as response:
|
||||
archive = response.read()
|
||||
if name == "cmpunlocker" and hashlib.sha256(archive).hexdigest() != UNLOCKER_ZIP_SHA256:
|
||||
raise ValueError("cmpunlocker archive checksum differs from pinned v0.3 source")
|
||||
with tempfile.TemporaryDirectory(dir=HARNESS_DIR) as temp:
|
||||
staging = Path(temp) / name
|
||||
staging.mkdir()
|
||||
archive_root = f"{name}-{ref[1:] if kind == 'tags' and ref.startswith('v') else ref}"
|
||||
with zipfile.ZipFile(io.BytesIO(archive)) as zf:
|
||||
for member in zf.infolist():
|
||||
parts = Path(member.filename).parts
|
||||
if parts == (archive_root,):
|
||||
continue
|
||||
if len(parts) < 2 or ".." in parts or parts[0] != archive_root:
|
||||
raise ValueError(f"unsafe archive path: {member.filename}")
|
||||
if (member.external_attr >> 16) & 0o170000 == 0o120000:
|
||||
raise ValueError(f"archive symlink: {member.filename}")
|
||||
path = staging.joinpath(*parts[1:])
|
||||
if member.is_dir():
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(member) as source, path.open("wb") as output:
|
||||
shutil.copyfileobj(source, output)
|
||||
path.chmod(((member.external_attr >> 16) & 0o777) or 0o644)
|
||||
if dest.exists():
|
||||
raise ValueError(f"{dest} exists but lacks required source files")
|
||||
staging.rename(dest)
|
||||
ok(f"downloaded {name}")
|
||||
except (OSError, ValueError, zipfile.BadZipFile, urllib.error.URLError) as exc:
|
||||
err(f"could not download {name}: {exc}")
|
||||
complete = False
|
||||
return complete
|
||||
|
||||
|
||||
TOOL_CHECKS = {
|
||||
"lspci": ("critical", "PCIe checks (package: pciutils)"),
|
||||
"setpci": ("critical", "link generation probe (package: pciutils)"),
|
||||
"nvidia-smi": ("critical", "GPU state / cmpunlocker verify"),
|
||||
"nvflash": ("optional", "VBIOS backup + serial (bundled in repo, or on PATH)"),
|
||||
"dmesg": ("optional", "kernel log scrape"),
|
||||
"journalctl": ("optional", "kernel log scrape fallback"),
|
||||
"vulkaninfo": ("optional", "Vulkan capability probe (package: vulkan-tools)"),
|
||||
}
|
||||
def os_release():
|
||||
data = read_file("/etc/os-release") or ""
|
||||
out = {}
|
||||
for line in data.splitlines():
|
||||
if "=" in line:
|
||||
k, v = line.split("=", 1)
|
||||
out[k] = v.strip().strip('"')
|
||||
return out
|
||||
|
||||
|
||||
def sudo_precondition(interactive=True):
|
||||
"""Check root or working sudo for host setup and privileged inspection.
|
||||
|
||||
Tries passwordless sudo first; falls back to an interactive `sudo -v` so the
|
||||
credentials get cached, after which plain sudo -n calls succeed.
|
||||
"""
|
||||
if os.geteuid() == 0:
|
||||
return True, "running as root"
|
||||
if not shutil.which("sudo"):
|
||||
return False, "sudo is not installed"
|
||||
r = run(["sudo", "-n", "true"], timeout=15)
|
||||
if r["rc"] == 0:
|
||||
return True, "sudo available (credentials cached)"
|
||||
if interactive and sys.stdin.isatty():
|
||||
info("sudo authentication required - enter your password")
|
||||
try:
|
||||
p = subprocess.run(["sudo", "-v"])
|
||||
except KeyboardInterrupt:
|
||||
return False, "interrupted"
|
||||
if p.returncode == 0:
|
||||
return True, "sudo authenticated just now"
|
||||
return False, "sudo authentication failed"
|
||||
return False, "sudo requires a password (non-interactive session)"
|
||||
|
||||
|
||||
def boot_id():
|
||||
return (read_file("/proc/sys/kernel/random/boot_id") or "").strip()
|
||||
|
||||
|
||||
def pending_state():
|
||||
try:
|
||||
return json.loads(PENDING_FILE.read_text())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def mark_pending(stage, card=None):
|
||||
HARNESS_DIR.mkdir(exist_ok=True)
|
||||
PENDING_FILE.write_text(json.dumps({"stage": stage, "boot_id": boot_id(),
|
||||
"card": card}, indent=2))
|
||||
|
||||
|
||||
def card_identity(bdf, device_id, serial=None):
|
||||
"""Identify a card across boots without treating its GPU UUID as a serial."""
|
||||
query = run(["nvidia-smi", "-i", bdf, "--query-gpu=uuid",
|
||||
"--format=csv,noheader"], timeout=30)
|
||||
uuid = query["out"].strip().splitlines()[0] if query["rc"] == 0 and query["out"].strip() else None
|
||||
if uuid and uuid.upper() in ("N/A", "UNKNOWN", "NONE"):
|
||||
uuid = None
|
||||
return {"serial": serial, "gpu_uuid": uuid, "device_id": device_id, "bdf": bdf}
|
||||
|
||||
|
||||
def same_card(previous, current):
|
||||
"""Return None when no stable identity is available for comparison."""
|
||||
if not isinstance(previous, dict):
|
||||
return None
|
||||
for key in ("serial", "gpu_uuid"):
|
||||
if previous.get(key) and current.get(key):
|
||||
return previous[key] == current[key]
|
||||
if previous.get("device_id") and previous["device_id"] != current["device_id"]:
|
||||
return False
|
||||
return None
|
||||
|
||||
|
||||
def supported_versions():
|
||||
text = read_file(HARNESS_DIR / "cmpunlocker" / "driver" / "VERSION") or ""
|
||||
return [line.strip() for line in text.splitlines() if re.fullmatch(r"\d+\.\d+\.\d+", line.strip())]
|
||||
|
||||
|
||||
def current_driver_version():
|
||||
r = run(["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader"], timeout=30)
|
||||
if r["rc"] == 0:
|
||||
return r["out"].splitlines()[0].strip() if r["out"].strip() else None
|
||||
return None
|
||||
|
||||
|
||||
def open_driver_active():
|
||||
version_text = read_file("/proc/driver/nvidia/version") or ""
|
||||
return "Open Kernel Module" in version_text
|
||||
|
||||
|
||||
def preflight(auto_install=True, do_download=True):
|
||||
step("Preflight")
|
||||
report = {"tools": {}, "root": have_root()}
|
||||
rel = os_release()
|
||||
report["os"] = {"id": rel.get("ID", ""), "version_id": rel.get("VERSION_ID", ""),
|
||||
"pretty": rel.get("PRETTY_NAME", "unknown")}
|
||||
info(f"OS: {report['os']['pretty']}")
|
||||
if do_download:
|
||||
report["repos_ok"] = ensure_repos()
|
||||
versions = supported_versions()
|
||||
report["supported_driver_versions"] = versions
|
||||
if not INSTALL_SH.exists() or not VERIFY_SH.exists() or not versions:
|
||||
report["blocking_missing"] = ["cmpunlocker v0.3 source"]
|
||||
report["preflight_ok"] = False
|
||||
err("preflight blocked: cmpunlocker v0.3 source is missing or incomplete")
|
||||
return report
|
||||
pending = pending_state()
|
||||
if (pending and pending.get("stage") == "nvidia-driver-610-open"
|
||||
and pending.get("boot_id") == boot_id()):
|
||||
report["reboot_required"] = True
|
||||
warn(f"{pending['stage']} installed during this boot; cold shut down and power on, then rerun")
|
||||
return report
|
||||
driver_rebooted = bool(pending and pending.get("boot_id") != boot_id()
|
||||
and pending.get("stage") == "nvidia-driver-610-open")
|
||||
version = current_driver_version()
|
||||
report["driver_version"] = version
|
||||
need_driver = not version or version not in versions or not open_driver_active()
|
||||
if driver_rebooted and need_driver:
|
||||
report["blocking_missing"] = ["nvidia-driver-610-open installed but inactive after reboot; inspect Secure Boot and driver logs"]
|
||||
report["preflight_ok"] = False
|
||||
err(report["blocking_missing"][0])
|
||||
return report
|
||||
if driver_rebooted:
|
||||
PENDING_FILE.unlink(missing_ok=True)
|
||||
pkgs = ["pciutils", "build-essential", "patch", "dkms", "curl",
|
||||
f"linux-headers-{run(['uname', '-r'])['out'].strip()}"]
|
||||
build_tools = ("lspci", "setpci", "gcc", "make", "patch", "curl")
|
||||
need_tools = not all(which(t) for t in build_tools)
|
||||
need_headers = not Path("/lib/modules", run(["uname", "-r"])["out"].strip(), "build").exists()
|
||||
if auto_install and (need_tools or need_headers or need_driver) and rel.get("ID") == "ubuntu":
|
||||
allowed, reason = sudo_precondition()
|
||||
report["sudo"] = {"ok": allowed, "why": reason}
|
||||
if allowed:
|
||||
update_log = HARNESS_DIR / "logs" / f"apt_update_{datetime.datetime.now().strftime('%Y%m%dT%H%M%S%f')}.log"
|
||||
info(f"apt-get update log: {update_log}")
|
||||
update = run_logged_as_root(["apt-get", "update"], update_log, timeout=300)
|
||||
if update["rc"] != 0:
|
||||
err(f"apt-get update failed: {(update['err'] or update['out'])[-300:]}")
|
||||
else:
|
||||
to_install = pkgs + (["nvidia-driver-610-open"] if need_driver else [])
|
||||
info("installing: " + " ".join(to_install))
|
||||
install_log = HARNESS_DIR / "logs" / f"apt_install_{datetime.datetime.now().strftime('%Y%m%dT%H%M%S%f')}.log"
|
||||
info(f"apt-get install log: {install_log}")
|
||||
install = run_logged_as_root(["apt-get", "install", "-y"] + to_install,
|
||||
install_log, timeout=1800)
|
||||
report["install"] = {"rc": install["rc"], "log": str(install_log),
|
||||
"tail": (install["out"] + install["err"])[-2000:]}
|
||||
if install["rc"] != 0:
|
||||
err("package installation failed: " + report["install"]["tail"][-300:])
|
||||
elif need_driver:
|
||||
mark_pending("nvidia-driver-610-open")
|
||||
report["reboot_required"] = True
|
||||
warn("driver installed; cold shut down and power on, then rerun")
|
||||
else:
|
||||
err(f"sudo unavailable: {reason}")
|
||||
|
||||
for tool, (level, why) in TOOL_CHECKS.items():
|
||||
path = find_nvflash() if tool == "nvflash" else which(tool)
|
||||
report["tools"][tool] = {"found": bool(path), "path": path, "level": level, "why": why}
|
||||
info(f"{tool}: {path or 'missing'}")
|
||||
blocking = [t for t in build_tools if not which(t)]
|
||||
if not report.get("reboot_required") and (current_driver_version() not in versions or not open_driver_active()):
|
||||
blocking.append("supported active nvidia-open 610 driver")
|
||||
if need_headers and not Path("/lib/modules", run(["uname", "-r"])["out"].strip(), "build").exists():
|
||||
blocking.append("matching kernel headers")
|
||||
report["blocking_missing"] = blocking
|
||||
report["preflight_ok"] = not blocking and not report.get("reboot_required")
|
||||
if blocking and not report.get("reboot_required"):
|
||||
err("preflight blocked: " + ", ".join(blocking))
|
||||
elif report["preflight_ok"]:
|
||||
ok("preflight complete")
|
||||
return report
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- GPU detect
|
||||
|
||||
def detect_gpus():
|
||||
bdfs = []
|
||||
for dev_id in GPU_IDS:
|
||||
r = run(["lspci", "-D", "-d", f"10de:{dev_id}"])
|
||||
for line in r["out"].splitlines():
|
||||
parts = line.split()
|
||||
if not parts:
|
||||
continue
|
||||
bdf = parts[0]
|
||||
if re.match(r"^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-f]$", bdf):
|
||||
bdfs.append(bdf.lower())
|
||||
return sorted(set(bdfs))
|
||||
|
||||
|
||||
def rescan_bus():
|
||||
r = as_root(["sh", "-c", "echo 1 > /sys/bus/pci/rescan"], timeout=15)
|
||||
if r["rc"] == 0:
|
||||
time.sleep(2)
|
||||
return r["rc"] == 0
|
||||
|
||||
|
||||
def sysfs(bdf):
|
||||
return Path("/sys/bus/pci/devices") / bdf
|
||||
|
||||
|
||||
def read_file(path, binary=False):
|
||||
try:
|
||||
with open(path, "rb" if binary else "r", errors="replace") as f:
|
||||
return f.read()
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- PCIe checks
|
||||
|
||||
def parse_link(text):
|
||||
out = {"lnksta_speed": None, "lnksta_width": None, "lnksta_gen": None,
|
||||
"lnkcap_speed": None, "lnkcap_width": None, "lnksta_raw": None}
|
||||
for line in text.splitlines():
|
||||
s = line.strip()
|
||||
if s.startswith("LnkSta:") and not s.startswith("LnkSta2"):
|
||||
out["lnksta_raw"] = s
|
||||
m = re.search(r"Speed\s+([^,\s]+)", s)
|
||||
if m:
|
||||
out["lnksta_speed"] = m.group(1)
|
||||
out["lnksta_gen"] = SPEED_TO_GEN.get(m.group(1))
|
||||
m = re.search(r"Width\s+(x\d+)", s)
|
||||
if m:
|
||||
out["lnksta_width"] = m.group(1)
|
||||
elif s.startswith("LnkCap:"):
|
||||
m = re.search(r"Speed\s+([^,\s]+)", s)
|
||||
if m:
|
||||
out["lnkcap_speed"] = m.group(1)
|
||||
m = re.search(r"Width\s+(x\d+)", s)
|
||||
if m:
|
||||
out["lnkcap_width"] = m.group(1)
|
||||
return out
|
||||
|
||||
|
||||
def parse_aer_flags(text):
|
||||
res = {"CESta": [], "UESta": []}
|
||||
for line in text.splitlines():
|
||||
s = line.strip()
|
||||
for key in ("CESta", "UESta"):
|
||||
if s.startswith(key + ":"):
|
||||
body = s.split(":", 1)[1]
|
||||
res[key].extend(re.findall(r"([A-Za-z0-9_]+)\+", body))
|
||||
return res
|
||||
|
||||
|
||||
def parent_bdf(bdf):
|
||||
path = Path(os.path.realpath(sysfs(bdf)))
|
||||
while True:
|
||||
path = path.parent
|
||||
name = path.name
|
||||
if re.match(r"^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-f]$", name):
|
||||
return name.lower()
|
||||
if name.startswith("pci") or path.parent == path:
|
||||
return None
|
||||
|
||||
|
||||
def aer_snapshot(bdf):
|
||||
snap = {"aer_sysfs_correctable": None, "aer_sysfs_fatal": None,
|
||||
"aer_correctable_sum": 0, "aer_fatal_sum": 0,
|
||||
"ce_active": [], "ue_active": [],
|
||||
"parent_bdf": None, "parent_ce_active": []}
|
||||
base = sysfs(bdf)
|
||||
data = read_file(base / "aer_dev_correctable")
|
||||
if data is not None:
|
||||
snap["aer_sysfs_correctable"] = data.strip()
|
||||
snap["aer_correctable_sum"] = sum(int(x) for x in re.findall(r"\d+", data))
|
||||
fatal = read_file(base / "aer_dev_fatal")
|
||||
if fatal is not None:
|
||||
snap["aer_sysfs_fatal"] = fatal.strip()
|
||||
snap["aer_fatal_sum"] = sum(int(x) for x in re.findall(r"\d+", fatal))
|
||||
r = run(["lspci", "-vvv", "-D", "-s", bdf], timeout=30)
|
||||
if r["rc"] == 0:
|
||||
aer = parse_aer_flags(r["out"])
|
||||
snap["ce_active"] = aer["CESta"]
|
||||
snap["ue_active"] = aer["UESta"]
|
||||
parent = parent_bdf(bdf)
|
||||
if parent:
|
||||
snap["parent_bdf"] = parent
|
||||
rp = run(["lspci", "-vvv", "-D", "-s", parent], timeout=30)
|
||||
if rp["rc"] == 0:
|
||||
snap["parent_ce_active"] = parse_aer_flags(rp["out"])["CESta"]
|
||||
return snap
|
||||
|
||||
|
||||
def link_gen_setpci(bdf):
|
||||
r = as_root(["setpci", "-s", bdf, "CAP_EXP+12.w"], timeout=15)
|
||||
val = r["out"].strip()
|
||||
if re.match(r"^[0-9a-fA-F]{4}$", val):
|
||||
return int(val, 16) & 0x0F
|
||||
return None
|
||||
|
||||
|
||||
def check_link(bdf):
|
||||
cmd = ["lspci", "-vvv", "-D", "-s", bdf]
|
||||
r = run(cmd, timeout=30)
|
||||
link = parse_link(r["out"]) if r["rc"] == 0 else {}
|
||||
if (not link.get("lnksta_width") or not link.get("lnksta_gen")) and os.geteuid() != 0:
|
||||
allowed, _ = sudo_precondition()
|
||||
if allowed:
|
||||
privileged = as_root(cmd, timeout=30)
|
||||
privileged_link = parse_link(privileged["out"]) if privileged["rc"] == 0 else {}
|
||||
fields = ("lnksta_width", "lnksta_gen")
|
||||
if sum(bool(privileged_link.get(field)) for field in fields) > sum(
|
||||
bool(link.get(field)) for field in fields):
|
||||
r, link = privileged, privileged_link
|
||||
link["setpci_current_gen"] = link_gen_setpci(bdf)
|
||||
link["lspci_rc"] = r["rc"]
|
||||
return link, r["out"]
|
||||
|
||||
|
||||
def dump_config(bdf):
|
||||
r = run(["lspci", "-nnvvxxxx", "-D", "-s", bdf], timeout=30)
|
||||
return r["out"] if r["rc"] == 0 else f"lspci failed rc={r['rc']}: {r['err']}"
|
||||
|
||||
|
||||
def retrain_test(bdf):
|
||||
result = {"attempted": False, "ok": None, "note": "", "width_before": None, "width_after": None}
|
||||
link_before, _ = check_link(bdf)
|
||||
result["width_before"] = link_before.get("lnksta_width")
|
||||
node = sysfs(bdf) / "link" / "retrain"
|
||||
if not node.exists():
|
||||
result["note"] = "sysfs link/retrain not available on this kernel"
|
||||
return result
|
||||
result["attempted"] = True
|
||||
r = as_root(["sh", "-c", f"echo 1 > {node}"], timeout=15)
|
||||
if r["rc"] != 0:
|
||||
result["note"] = f"retrain write failed: {r['err'].strip()}"
|
||||
return result
|
||||
time.sleep(3)
|
||||
if bdf not in detect_gpus():
|
||||
result.update(ok=False, note="device vanished after retrain")
|
||||
return result
|
||||
link_after, _ = check_link(bdf)
|
||||
result["width_after"] = link_after.get("lnksta_width")
|
||||
result["ok"] = result["width_after"] == result["width_before"] and result["width_after"] is not None
|
||||
result["note"] = f"{result['width_before']} -> {result['width_after']}"
|
||||
return result
|
||||
|
||||
|
||||
def rescan_test(bdf):
|
||||
result = {"attempted": True, "ok": None, "note": ""}
|
||||
if not rescan_bus():
|
||||
result["note"] = "bus rescan not permitted (no root)"
|
||||
return result
|
||||
time.sleep(1)
|
||||
present = bdf in detect_gpus()
|
||||
result["ok"] = present
|
||||
result["note"] = "re-enumerated at same BDF" if present else "device missing after rescan"
|
||||
return result
|
||||
|
||||
|
||||
def dmesg_capture(bdf, since):
|
||||
"""Warn only for recent lines naming this endpoint; keep others separately."""
|
||||
since = since.replace("T", " ")
|
||||
text, source = "", None
|
||||
for command, label in (
|
||||
(["journalctl", "-k", "-b", "--since", since, "--no-pager", "-o", "short-precise"], "journalctl"),
|
||||
(["dmesg", "--ctime", "--since", since], "dmesg")):
|
||||
r = as_root(command, timeout=25)
|
||||
if r["rc"] == 0:
|
||||
text, source = r["out"], label
|
||||
break
|
||||
if source is None:
|
||||
return {"source": None, "related": [], "errors": [], "other_pcie": [],
|
||||
"note": "recent kernel log unavailable (root or journal access needed)"}
|
||||
tokens = (bdf.lower(), bdf[:-2].lower())
|
||||
related, errors, other = [], [], []
|
||||
for line in text.splitlines():
|
||||
low = line.lower()
|
||||
if any(token in low for token in tokens):
|
||||
related.append(line)
|
||||
if any(k in low for k in ("xid", "error", "aer:", "fail", "corrupted", "fallen off")):
|
||||
errors.append(line)
|
||||
elif any(k in low for k in ("aer:", "pcieport", "nvrm", "xid")):
|
||||
other.append(line)
|
||||
return {"source": source, "related": related[-200:], "errors": errors[-50:],
|
||||
"other_pcie": other[-200:], "note": f"{len(related)} card lines, {len(other)} other PCIe/GPU lines since {since}"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- BIOS / serial
|
||||
|
||||
def nvflash_bin(args):
|
||||
if getattr(args, "nvflash", None):
|
||||
return args.nvflash
|
||||
return find_nvflash()
|
||||
|
||||
|
||||
def usable_serial(value):
|
||||
return bool(value and sanitize(value) == value and
|
||||
value.upper() not in ("N/A", "UNKNOWN", "NONE") and
|
||||
not re.fullmatch(r"0+", value))
|
||||
|
||||
|
||||
def get_serial(nf, bdf):
|
||||
"""Read the physical board serial; never substitute a GPU UUID or PCI address."""
|
||||
smi = run(["nvidia-smi", "-i", bdf, "--query-gpu=serial",
|
||||
"--format=csv,noheader"], timeout=30)
|
||||
if smi["rc"] == 0:
|
||||
serial = smi["out"].strip().splitlines()[0] if smi["out"].strip() else ""
|
||||
if usable_serial(serial):
|
||||
info(f"serial auto-read from nvidia-smi: {serial}")
|
||||
return serial, "nvidia-smi"
|
||||
|
||||
detail = run(["nvidia-smi", "-q", "-i", bdf], timeout=30)
|
||||
if detail["rc"] == 0:
|
||||
match = re.search(r"^\s*Serial Number\s*:\s*(\S+)", detail["out"], re.I | re.M)
|
||||
if match and usable_serial(match.group(1)):
|
||||
info(f"serial auto-read from nvidia-smi details: {match.group(1)}")
|
||||
return match.group(1), "nvidia-smi-detail"
|
||||
|
||||
vpd_cmd = ["lspci", "-vvv", "-D", "-s", bdf]
|
||||
for vpd in (run(vpd_cmd, timeout=30), as_root(vpd_cmd, timeout=30)):
|
||||
if vpd["rc"] == 0:
|
||||
match = re.search(r"\[SN\]\s+Serial number:\s*(\S+)", vpd["out"], re.I)
|
||||
if match and usable_serial(match.group(1)):
|
||||
info(f"serial auto-read from PCI VPD: {match.group(1)}")
|
||||
return match.group(1), "pci-vpd"
|
||||
|
||||
if nf:
|
||||
q = as_root([nf, "--pcisegbus", bdf, "--query"], timeout=40)
|
||||
serials = {m.group(1) for m in re.finditer(
|
||||
r"Serial(?:\s*Number)?\s*[:=]\s*(\S+)", q["out"], re.I)
|
||||
if q["rc"] == 0 and usable_serial(m.group(1))}
|
||||
if q["rc"] == 0 and len(serials) == 1:
|
||||
serial = serials.pop()
|
||||
info(f"serial auto-read from card-scoped nvflash query: {serial}")
|
||||
return serial, "nvflash-pcisegbus-query"
|
||||
return None, "unavailable"
|
||||
|
||||
|
||||
def normalize_bdf(value):
|
||||
match = re.search(r"([0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7])$", value)
|
||||
return match.group(1).lower() if match else None
|
||||
|
||||
|
||||
def memory_state(bdf, device_id):
|
||||
"""Identify unlock from this card's reported memory, independent of Gen2 status."""
|
||||
r = run(["nvidia-smi", "--query-gpu=pci.bus_id,memory.total",
|
||||
"--format=csv,noheader,nounits"], timeout=30)
|
||||
if r["rc"] != 0:
|
||||
return {"status": "unavailable", "note": (r["err"] or r["out"])[-300:]}
|
||||
for line in r["out"].splitlines():
|
||||
fields = [field.strip() for field in line.split(",")]
|
||||
if len(fields) != 2 or normalize_bdf(fields[0]) != bdf:
|
||||
continue
|
||||
try:
|
||||
mib = int(fields[1])
|
||||
except ValueError:
|
||||
return {"status": "unavailable", "note": f"invalid memory total: {fields[1]}"}
|
||||
if device_id == "10de:20c2":
|
||||
status = "unlocked" if mib >= 60000 else "stock" if 7680 <= mib <= 8704 else "unexpected"
|
||||
elif device_id == "10de:2082":
|
||||
status = "unlocked" if 35000 <= mib < 60000 else "stock" if 9728 <= mib <= 10752 else "unexpected"
|
||||
else:
|
||||
status = "unsupported"
|
||||
return {"status": status, "memory_mib": mib}
|
||||
return {"status": "unavailable", "note": f"{bdf} absent from nvidia-smi"}
|
||||
|
||||
|
||||
def ensure_unlocked(bdf, device_id, folder, card=None, allow_install=True):
|
||||
before = memory_state(bdf, device_id)
|
||||
result = {"before": before, "status": before["status"], "install_rc": None}
|
||||
pending = pending_state()
|
||||
if pending and pending.get("stage") == "cmpunlocker":
|
||||
if pending.get("boot_id") == boot_id():
|
||||
result.update(status="reboot-required", note="cold shut down and power on after cmpunlocker install")
|
||||
return result
|
||||
match = same_card(pending.get("card"), card or card_identity(bdf, device_id))
|
||||
if match is False:
|
||||
warn("previous unlock marker belongs to a different card; checking this card anew")
|
||||
PENDING_FILE.unlink(missing_ok=True)
|
||||
elif match is None and before["status"] != "unlocked":
|
||||
result.update(status="failed", note="cannot tie the previous unlock to this card; inspect the pending marker before retrying")
|
||||
return result
|
||||
elif before["status"] != "unlocked":
|
||||
result.update(status="failed", note="cmpunlocker installed but card is not unlocked after reboot")
|
||||
return result
|
||||
else:
|
||||
PENDING_FILE.unlink(missing_ok=True)
|
||||
if before["status"] == "unlocked":
|
||||
result.update(status="unlocked", note="unlocked memory is active")
|
||||
return result
|
||||
if before["status"] != "stock":
|
||||
result.update(status="failed", note="card memory/PCI ID is unavailable or unexpected; refusing to select an unlock profile")
|
||||
return result
|
||||
if not allow_install:
|
||||
result.update(status="failed", note="unlock inactive and --no-install was requested")
|
||||
return result
|
||||
if not INSTALL_SH.exists():
|
||||
result.update(status="failed", note="cmpunlocker install.sh is missing")
|
||||
return result
|
||||
allowed, reason = sudo_precondition()
|
||||
if not allowed:
|
||||
result.update(status="failed", note=f"root access required: {reason}")
|
||||
return result
|
||||
info("installing cmpunlocker for the detected card (this can take several minutes)")
|
||||
log_path = (folder / "extras" /
|
||||
f"cmpunlocker_install_{datetime.datetime.now().strftime('%Y%m%dT%H%M%S%f')}.log")
|
||||
info(f"cmpunlocker install log: {log_path}")
|
||||
install = run_logged_as_root(["bash", str(INSTALL_SH)], log_path, timeout=3600,
|
||||
cwd=INSTALL_SH.parent)
|
||||
result["install_rc"] = install["rc"]
|
||||
result["install_log"] = str(log_path)
|
||||
if install["rc"] != 0:
|
||||
result.update(status="failed", note=f"cmpunlocker installer failed (rc={install['rc']}); see {log_path}")
|
||||
return result
|
||||
mark_pending("cmpunlocker", card or card_identity(bdf, device_id))
|
||||
result.update(status="reboot-required", note="cmpunlocker installed; cold shut down and power on, then rerun")
|
||||
return result
|
||||
|
||||
|
||||
def save_bios(nf, out_path, bdf):
|
||||
result = {"method": None, "path": None, "sha256": None, "size": 0, "ok": False, "note": ""}
|
||||
if out_path.exists():
|
||||
data = out_path.read_bytes()
|
||||
if len(data) > 512 and data.strip(b"\x00"):
|
||||
result.update(method="existing", path=str(out_path), size=len(data),
|
||||
sha256=hashlib.sha256(data).hexdigest(), ok=True,
|
||||
note="existing backup preserved")
|
||||
else:
|
||||
result["note"] = "existing backup is empty or invalid; move it before retrying"
|
||||
return result
|
||||
if nf:
|
||||
r = as_root([nf, "--pcisegbus", bdf, "--save", str(out_path)], timeout=90)
|
||||
if r["rc"] == 0 and out_path.exists() and out_path.stat().st_size > 512:
|
||||
data = out_path.read_bytes()
|
||||
if data.strip(b"\x00") == b"":
|
||||
result["note"] = "nvflash wrote an all-zero image"
|
||||
else:
|
||||
result.update(method="nvflash", path=str(out_path), size=len(data),
|
||||
sha256=hashlib.sha256(data).hexdigest(), ok=True)
|
||||
return result
|
||||
else:
|
||||
result["note"] = f"card-scoped nvflash --save failed or image too small (rc={r['rc']}): {(r['err'] or r['out'])[-200:]}"
|
||||
out_path.unlink(missing_ok=True)
|
||||
else:
|
||||
result["note"] = "nvflash unavailable"
|
||||
|
||||
rom = sysfs(bdf) / "rom"
|
||||
if as_root(["sh", "-c", f"echo 1 > {rom}"], timeout=15)["rc"] == 0:
|
||||
command = ["cat", str(rom)] if os.geteuid() == 0 else ["sudo", "-n", "cat", str(rom)]
|
||||
try:
|
||||
read = subprocess.run(command, capture_output=True, timeout=30)
|
||||
data = read.stdout if read.returncode == 0 else None
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
data = None
|
||||
finally:
|
||||
as_root(["sh", "-c", f"echo 0 > {rom}"], timeout=15)
|
||||
if data and len(data) > 512 and data.strip(b"\x00") != b"":
|
||||
out_path.write_bytes(data)
|
||||
result.update(method="sysfs-rom", path=str(out_path), size=len(data),
|
||||
sha256=hashlib.sha256(data).hexdigest(), ok=True)
|
||||
result["note"] = (result["note"] + " | saved via sysfs rom").strip(" |")
|
||||
return result
|
||||
result["note"] += " | sysfs rom empty/all-zero"
|
||||
else:
|
||||
result["note"] += " | sysfs rom enable failed"
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- nvidia / verify
|
||||
|
||||
def nvidia_smi_info():
|
||||
if not which("nvidia-smi"):
|
||||
return {"available": False}
|
||||
q = run(["nvidia-smi", "--query-gpu=name,uuid,memory.total,pcie.link.gen.current,"
|
||||
"pcie.link.gen.max,pcie.link.width.current,temperature.gpu,driver_version",
|
||||
"--format=csv,noheader"], timeout=30)
|
||||
if q["rc"] != 0:
|
||||
return {"available": False, "error": (q["err"] or q["out"])[-300:]}
|
||||
rows = [[f.strip() for f in line.split(",")] for line in q["out"].strip().splitlines()]
|
||||
return {"available": True, "gpus": rows}
|
||||
|
||||
|
||||
def run_verify_unlocker(bdf, device_id):
|
||||
if not VERIFY_SH.exists():
|
||||
return {"ran": False, "note": "harness/cmpunlocker source missing"}
|
||||
if not which("nvidia-smi"):
|
||||
return {"ran": False, "note": "nvidia-smi missing - unlock verify skipped"}
|
||||
kernel = run(["uname", "-r"])["out"].strip()
|
||||
inventory = Path("/lib/modules") / kernel / "updates" / "cmpunlocker" / "gpu_inventory"
|
||||
contents = read_file(inventory)
|
||||
if contents:
|
||||
entries = [line.split() for line in contents.splitlines()
|
||||
if line.strip() and not line.lstrip().startswith("#")]
|
||||
if len(entries) != 1 or len(entries[0]) < 2 or normalize_bdf(entries[0][0]) != bdf or not entries[0][1].lower().endswith(device_id[-4:]):
|
||||
return {"ran": False, "status": "skipped-stale-inventory",
|
||||
"note": f"installed gpu_inventory does not match {bdf} ({device_id}); live per-card memory and PCIe checks remain authoritative"}
|
||||
r = as_root(["bash", str(VERIFY_SH)], timeout=180)
|
||||
lines = [l for l in (r["out"] + r["err"]).splitlines() if l.strip()]
|
||||
status = "unknown"
|
||||
if r["rc"] == 0 and any("unlocked memory" in l.lower() for l in lines):
|
||||
status = "unlocked-ok"
|
||||
elif "STOCK" in r["out"]:
|
||||
status = "stock-memory"
|
||||
elif "MISSING" in r["out"]:
|
||||
status = "gpu-missing-in-smi"
|
||||
elif r["rc"] != 0:
|
||||
status = f"verify-failed-rc{r['rc']}"
|
||||
return {"ran": True, "rc": r["rc"], "status": status, "tail": lines[-30:]}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- optional Vulkan capability probe
|
||||
|
||||
def run_vulkan_probe():
|
||||
if not which("vulkaninfo"):
|
||||
return {"requested": True, "ran": False, "note": "vulkaninfo is missing"}
|
||||
r = run(["vulkaninfo", "--summary"], timeout=30)
|
||||
return {"requested": True, "ran": True, "rc": r["rc"],
|
||||
"note": "generic Vulkan capability listing; does not validate CMP VRAM",
|
||||
"log_tail": (r["out"] + r["err"]).splitlines()[-40:]}
|
||||
Reference in New Issue
Block a user