commit b30be648d7b02e5add4a3e8b67733795a1ba0111 Author: Quentin BUISSON-DEBON Date: Sun Sep 27 21:02:27 2026 +0200 Initial commit: CMP 170HX per-card check tooling Add check_cards.py CLI and cxlib.py host helpers for preparing, unlocking, and PCIe-checking NVIDIA CMP 170HX cards, plus operator README, agent docs, and bundled nvflash tooling. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9020a90 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.pyc +.DS_Store +harness/ +cards/ +unidentified/ +progress.json diff --git a/.serena/.gitignore b/.serena/.gitignore new file mode 100644 index 0000000..2e510af --- /dev/null +++ b/.serena/.gitignore @@ -0,0 +1,2 @@ +/cache +/project.local.yml diff --git a/.serena/project.yml b/.serena/project.yml new file mode 100644 index 0000000..251a32b --- /dev/null +++ b/.serena/project.yml @@ -0,0 +1,133 @@ +# the name by which the project can be referenced within Serena +project_name: "cmp170hx_check-main" + + +# list of languages for which language servers are started; choose from: +# al angular ansible bash clojure +# cpp cpp_ccls crystal csharp csharp_omnisharp +# dart elixir elm erlang fortran +# fsharp go groovy haskell haxe +# hlsl html java json julia +# kotlin lean4 lua luau markdown +# matlab msl nix ocaml pascal +# perl php php_phpactor powershell python +# python_jedi python_ty r rego ruby +# ruby_solargraph rust scala scss solidity +# svelte swift systemverilog terraform toml +# typescript typescript_vts vue yaml zig +# (This list may be outdated. For the current list, see values of Language enum here: +# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py +# For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) +# Note: +# - For C, use cpp +# - For JavaScript, use typescript +# - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root) +# - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm) +# - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three) +# - For Free Pascal/Lazarus, use pascal +# Special requirements: +# Some languages require additional setup/installations. +# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers +# When using multiple languages, the first language server that supports a given file will be used for that file. +# The first language is the default language and the respective language server will be used as a fallback. +# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. +languages: +- python + +# the encoding used by text files in the project +# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings +encoding: "utf-8" + +# line ending convention to use when writing source files. +# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default) +# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings. +line_ending: + +# The language backend to use for this project. +# If not set, the global setting from serena_config.yml is used. +# Valid values: LSP, JetBrains +# Note: the backend is fixed at startup. If a project with a different backend +# is activated post-init, an error will be returned. +language_backend: + +# whether to use project's .gitignore files to ignore files +ignore_all_files_in_gitignore: true + +# advanced configuration option allowing to configure language server-specific options. +# Maps the language key to the options. +# Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available. +# No documentation on options means no options are available. +ls_specific_settings: {} + +# list of additional workspace folder paths for cross-package reference support (e.g. in monorepos). +# Paths can be absolute or relative to the project root. +# Each folder is registered as an LSP workspace folder, enabling language servers to discover +# symbols and references across package boundaries. +# Currently supported for: TypeScript. +# Example: +# additional_workspace_folders: +# - ../sibling-package +# - ../shared-lib +additional_workspace_folders: [] + +# list of additional paths to ignore in this project. +# Same syntax as gitignore, so you can use * and **. +# Note: global ignored_paths from serena_config.yml are also applied additively. +ignored_paths: [] + +# whether the project is in read-only mode +# If set to true, all editing tools will be disabled and attempts to use them will result in an error +# Added on 2025-04-18 +read_only: false + +# list of tool names to exclude. +# This extends the existing exclusions (e.g. from the global configuration) +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html +excluded_tools: [] + +# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default). +# This extends the existing inclusions (e.g. from the global configuration). +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html +included_optional_tools: [] + +# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. +# This cannot be combined with non-empty excluded_tools or included_optional_tools. +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html +fixed_tools: [] + +# list of mode names that are to be activated by default, overriding the setting in the global configuration. +# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. +# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply. +# Otherwise, this overrides the setting from the global configuration (serena_config.yml). +# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply +# for this project. +# This setting can, in turn, be overridden by CLI parameters (--mode). +# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes +default_modes: + +# list of mode names to be activated additionally for this project, e.g. ["query-projects"] +# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. +# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes +added_modes: + +# initial prompt for the project. It will always be given to the LLM upon activating the project +# (contrary to the memories, which are loaded on demand). +initial_prompt: "" + +# time budget (seconds) per tool call for the retrieval of additional symbol information +# such as docstrings or parameter information. +# This overrides the corresponding setting in the global configuration; see the documentation there. +# If null or missing, use the setting from the global configuration. +symbol_info_budget: + +# list of regex patterns which, when matched, mark a memory entry as read‑only. +# Extends the list from the global configuration, merging the two lists. +read_only_memory_patterns: [] + +# list of regex patterns for memories to completely ignore. +# Matching memories will not appear in list_memories or activate_project output +# and cannot be accessed via read_memory or write_memory. +# To access ignored memory files, use the read_file tool on the raw file path. +# Extends the list from the global configuration, merging the two lists. +# Example: ["_archive/.*", "_episodes/.*"] +ignored_memory_patterns: [] diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..b6ae8dc --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,30 @@ +# Project instructions + +## Purpose + +This repository checks one NVIDIA CMP 170HX at a time on a fresh Ubuntu 26.04 x86-64 host. The normal sequence is: prepare driver and tools, check PCIe lane width, install cmpunlocker for stock memory, cold boot, confirm unlocked memory, then check the final PCIe link. The capacitor width check passes at **x16**, even if the link is Gen1. A completed post-unlock card check passes only at **Gen2 x16**. + +## Source and outputs + +- `check_cards.py` owns the CLI, per-card flow, report scoring, and terminal result. +- `cxlib.py` owns host setup, pinned cmpunlocker download, card identity, subprocess logging, PCIe inspection, and optional VBIOS backup. +- `README.md` is the operator guide. Keep its flags, output paths, reboot steps, and exit codes aligned with the code. +- `harness/`, `cards/`, and `unidentified/` contain generated source, logs, reports, or firmware backups. Do not commit their contents. +- The bundled `nvflash_5.867_linux/` contains the optional backup tool. Backups belong under `extras/`, not at repository root. + +## Invariants + +- A card folder is named from an automatically read **physical serial number**. Never request manual serial entry or substitute a GPU UUID or PCI address for a serial. When serial lookup fails, continue the run under `unidentified/` and report the missing serial. +- Match every card-sensitive operation to the detected PCI address. A host may have another NVIDIA GPU even though only one CMP 170HX is being checked. +- Measure capacitor-related lane width before unlocking. The width result is independent of cmpunlocker: Gen1 x16 passes the width check, while x4 fails it. Do not claim the capacitor is physically verified; the width is an observed link result on the current host. +- Keep the final PCIe acceptance check after unlock. Preserve a red `[ERROR]` message and failing exit for any final link other than Gen2 x16. Gen1 x16 passes the earlier width check but fails the final card check. +- Preserve cold-boot state by card identity. GPU UUID may be used internally for this purpose, but not as a folder name. +- The cmpunlocker archive is pinned by commit and SHA256 in `cxlib.py`. Check a pinned version's actual accepted installer flags before changing invocation. The currently pinned v0.3 installer does not accept `--no-passthrough`. +- VBIOS backup is optional and read-only. Select the detected PCI address for nvflash and sysfs ROM operations. Never add a flashing step to the normal workflow. + +## Working in this repository + +- Git is **local only**. Do not run `git push`, `git remote` commands, or add a remote. Local status, diff, and commits are allowed. +- Do not run host setup, unlock, VBIOS, or PCIe mutation commands on a development machine. Those actions belong on the intended Ubuntu host when the user requests them. +- Keep reports and long-running installation logs visible and useful to an operator. Save full output on disk; show installation progress in the terminal. +- If asked to verify without a card, `python3 check_cards.py --demo` is the synthetic path. It does not establish hardware correctness. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..4da0062 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,3 @@ +# Claude project instructions + +Read and follow [AGENTS.md](AGENTS.md). It is the single source of project instructions. diff --git a/README.md b/README.md new file mode 100644 index 0000000..374d2f4 --- /dev/null +++ b/README.md @@ -0,0 +1,95 @@ +# CMP 170HX unlock and PCIe check + +This tool handles **one NVIDIA CMP 170HX at a time** on an Ubuntu 26.04 x86-64 host. It prepares a compatible open NVIDIA driver, measures PCIe lane width before unlocking, installs [cmpunlocker](https://github.com/amoghmunikote/cmpunlocker) if the card still reports stock memory, waits for the required cold boot, then verifies unlocked memory and the final PCIe link. The [170th-Street capacitor mod guide](https://github.com/amoghmunikote/170th-Street/blob/master/modifications/pcie-capacitor-mod.md) documents **x16** as the expected width after the hardware modification, even at Gen1. The final card check requires **Gen2 x16** after cmpunlocker. + +The program downloads a cmpunlocker source archive over HTTPS. It does not run Git remote commands. The source is pinned to upstream v0.3 commit `f51da030ef4260656288edf07e95ae45f339aea8` and checked against the archive SHA256 in `cxlib.py`. The 170th-Street guide supplies the width criterion and is linked above. + +## On a fresh Ubuntu 26.04 host + +1. Connect one card, make sure the host has Internet access, and run from this directory: + + ```bash + python3 check_cards.py + ``` + +2. The first run downloads cmpunlocker, installs `pciutils`, build tools, matching kernel headers, and `nvidia-driver-610-open` when needed. It asks for sudo credentials if necessary. Package installation output streams to the terminal and is saved under `harness/logs/`. When it prints **cold shut down and power on**, shut down fully and start the host again: + + ```bash + sudo shutdown -h now + ``` + +3. Run `python3 check_cards.py` again after each requested cold boot. It reads lane width **before** attempting cmpunlocker. If memory is still stock, it runs the pinned upstream installer with its supported default options. The installer configures its PCIe Gen2 service and IOMMU passthrough settings. Its output streams to the terminal and to `cards//extras/cmpunlocker_install_*.log` (or `unidentified/` if the serial is not readable). Cold shut down and power on again, then rerun the script. Once unlocked memory is active, it measures the final PCIe link. A card that was already unlocked proceeds directly to verification. + +The upstream installer builds and installs patched kernel modules and requires a compatible active **nvidia-open 610** driver, matching headers, root access, and disabled Secure Boot. It downloads NVIDIA source during the build. See the [upstream requirements and installation instructions](https://github.com/amoghmunikote/cmpunlocker#requirements). + +`python3 check_cards.py --preflight` performs setup only. It may also return `4` to signal a required cold boot. If the driver remains inactive after that boot, the script stops with a diagnostic instead of installing it repeatedly. + +If a completed reboot still leaves the driver or unlock inactive, inspect the card report and its installer log. The pending marker in `harness/unlock_pending.json` records the installed card's physical serial or internal GPU UUID when available, so a different card after reboot can be checked independently. The UUID is never used as a folder name. After resolving a failed unlock on the same card, remove that marker only when you intentionally want the script to retry installation. + +## Per-card flow + +1. Detect one CMP 170HX (`10de:20c2` or `10de:2082`) and try to get its **serial number** automatically from `nvidia-smi`, PCI VPD, or a card-scoped query if the installed nvflash supports one. No serial is requested from the operator. If the physical serial is unavailable, checks continue in `unidentified/_/`; a completed run fails with a clear serial error. If the serial appears later in the same run, those diagnostics move to `cards//`. Earlier unidentified runs remain separate. A GPU UUID is not used as a serial. +2. Read the endpoint's `LnkSta` lane width **before** unlocking. An x16 width passes the capacitor width check even at Gen1. An x4 or unreadable width immediately prints `CAPACITOR MOD CHECK FAILED` in red on a terminal. If the current user cannot read link details, the script requests sudo access and retries. The width measurement is independent of cmpunlocker; the normal script flow still completes host preflight before reaching it. +3. Check memory size for that exact PCI address. Stock memory triggers the cmpunlocker installer; unlocked memory continues to verification. Run `cmpunlocker/verify.sh` when its installed GPU inventory matches the current card, then capture the final PCIe link. **Gen2 x16** is the only passing final link result. If the upstream inventory describes an earlier card, the script reports that verifier as skipped and uses live per-card memory and PCIe measurements. +4. Capture PCIe configuration, AER counters, kernel lines that name this card since the run began, and `nvidia-smi` data. Other recent PCIe/GPU kernel lines go in a separate log and do not count as card errors. Optional actions run afterward. + +| Negotiated link | Capacitor width check | Final card check after unlock | +| --- | --- | --- | +| Gen2 x16 | Pass | Pass | +| Gen1 x16 | Pass | Fail: Gen2 is required | +| Gen2 x4 | Fail: x16 is required | Fail | +| Unknown speed, x16 width | Pass | Fail: Gen2 cannot be confirmed | +| Unknown width | Fail: x16 cannot be confirmed | Fail | + +The tool cannot inspect physical capacitors. A passing width result means **x16 was observed on this host**; a passing final card result also requires unlocked memory and Gen2. A failing width result does not by itself identify whether the capacitor mod, slot, or riser limited the link. Failure text remains explicit when color is unavailable in redirected output. + +## Optional actions + +```bash +python3 check_cards.py --bios # VBIOS backup in cards//extras/ +python3 check_cards.py --link-tests # PCIe retrain and bus rescan +python3 check_cards.py --vulkan ask # offer a Vulkan capability probe +python3 check_cards.py --vulkan always # run the optional probe +python3 check_cards.py --no-install # inspect only; no package or unlock install +python3 check_cards.py --no-download # use existing harness/ sources +python3 check_cards.py --demo # synthetic output without a GPU +``` + +VBIOS backup is opt in and never flashes the card. The bundled `nvflash_5.867_linux/x64/nvflash` is used automatically; `--nvflash PATH` selects another binary. Both the nvflash save attempt and sysfs ROM fallback target the detected PCI address. If nvflash cannot save the ROM, the script tries the PCI sysfs ROM for that card. An existing `bios.rom` larger than 512 bytes and containing nonzero data is preserved; the script does not authenticate its contents. + +The Vulkan action is disabled by default and needs `vulkaninfo` installed separately. It lists generic Vulkan capabilities; it does **not** test this card's VRAM or prove that Vulkan uses the CMP 170HX. + +## Output and status + +```text +cards// +├── report.json latest run +├── runs// +│ ├── report.json archived run +│ ├── lspci_before_unlock.txt capacitor width check +│ ├── lspci.txt +│ ├── lspci_after.txt +│ ├── config_space.hex +│ ├── dmesg.log +│ ├── dmesg_other_pcie.log +│ └── verify_unlocker.log +└── extras/ + ├── bios.rom if --bios was requested + └── cmpunlocker_install_*.log if installation was attempted +``` + +If no physical serial is readable, the same report layout is written beneath `unidentified/_/` instead of `cards//`. + +`result` is `PASS`, `WARN`, `FAIL`, or `PENDING_REBOOT`. A link other than Gen2 x16, inactive unlock, or missing physical serial fails the check. Correctable AER errors or failed optional backup produce a warning. + +| Exit code | Meaning | +| --- | --- | +| `0` | `PASS` or `WARN` | +| `1` | `FAIL` | +| `2` | No card selected | +| `3` | Preflight blocked | +| `4` | Cold boot required | + +The tool never flashes the card. The required cmpunlocker installation changes host kernel modules and boot configuration. PCIe retrain and rescan happen only with `--link-tests`. + +The script runs no Git remote commands. Guidance for coding agents is in [AGENTS.md](AGENTS.md). diff --git a/check_cards.py b/check_cards.py new file mode 100755 index 0000000..3d6f977 --- /dev/null +++ b/check_cards.py @@ -0,0 +1,408 @@ +#!/usr/bin/env python3 +"""Unlock one CMP 170HX, then assess its PCIe link and optional diagnostics. + +Exit codes: 0 PASS/WARN, 1 FAIL, 2 no card, 3 preflight blocked, +4 cold reboot required before checking the card. +""" +import argparse +import datetime +import json +import shutil +import sys +import time +from pathlib import Path + +import cxlib as X +from cxlib import ask, detect_gpus, err, info, now_iso, ok, sanitize, step, warn + + +# ---------------------------------------------------------------- scoring + +def score(rep): + fails, warns = [], [] + if rep.get("unlock", {}).get("status") != "unlocked": + fails.append("cmpunlocker memory unlock is not active") + if not rep.get("serial"): + fails.append("physical card serial unavailable; diagnostics kept under unidentified/") + if not rep.get("bdf"): + fails.append("no CMP 170HX device detected on the bus") + link = rep.get("link", {}) + if rep.get("bdf"): + if link.get("lspci_rc") not in (0, None): + fails.append("lspci could not read the device") + if not link_passes(link): + generation = f"Gen{link['lnksta_gen']}" if link.get("lnksta_gen") else "unknown generation" + fails.append(f"PCIe {generation} {link.get('lnksta_width') or 'unknown width'}; " + "Gen2 x16 required after cmpunlocker (check unlock, mod, slot and riser)") + if rep.get("rescan", {}).get("ok") is False: + fails.append("device lost after bus rescan") + if rep.get("retrain", {}).get("ok") is False: + fails.append("link did not survive retrain") + if rep.get("link_after") and not link_passes(rep["link_after"]): + fails.append("link does not report Gen2 x16 after diagnostics") + aer = rep.get("aer_after", {}) + aer_before = rep.get("aer_before", {}) + if (aer.get("ce_active") or aer.get("parent_ce_active") + or (aer.get("aer_correctable_sum", 0) > aer_before.get("aer_correctable_sum", 0))): + warns.append("correctable AER errors present/incremented") + if aer.get("ue_active"): + fails.append(f"uncorrectable AER errors active: {aer['ue_active']}") + if aer.get("aer_fatal_sum", 0) > aer_before.get("aer_fatal_sum", 0): + fails.append("fatal PCIe AER counter increased during the check") + dm = rep.get("dmesg", {}) + if dm.get("errors"): + warns.append(f"{len(dm['errors'])} kernel error line(s) related to this card (see dmesg.log)") + vu = rep.get("vulkan_probe", {}) + if vu.get("requested") and (not vu.get("ran") or vu.get("rc") != 0): + warns.append("optional Vulkan capability probe failed") + if rep.get("bios", {}).get("requested") and not rep["bios"].get("ok"): + warns.append("VBIOS backup failed/incomplete") + ver = rep.get("cmpunlocker_verify", {}) + if ver.get("status") not in ("unlocked-ok", "skipped-stale-inventory"): + warns.append(f"cmpunlocker verify status: {ver.get('status')}") + + result = "FAIL" if fails else ("WARN" if warns else "PASS") + return result, fails, warns + + +def link_passes(link): + return link.get("lnksta_width") == "x16" and link.get("lnksta_gen") == 2 + + +def width_status(link): + width = link.get("lnksta_width") + return "PASS" if width == "x16" else "FAIL" if width else "UNKNOWN" + + +def log_capacitor_width(link, phase): + width = link.get("lnksta_width") + speed = link.get("lnksta_speed") or "unknown speed" + generation = f"Gen{link['lnksta_gen']}" if link.get("lnksta_gen") else "unknown generation" + if width == "x16": + ok(f"CAPACITOR MOD WIDTH CHECK PASS ({phase}): PCIe {generation} x16 ({speed}) negotiated") + else: + err(f"CAPACITOR MOD CHECK FAILED ({phase}): PCIe {generation} {width or 'unknown width'} " + f"({speed}) negotiated; expected x16 width. Check the capacitor mod and host slot/riser.") + + +def log_final_link(link, phase="after unlock"): + width = link.get("lnksta_width") or "unknown width" + generation = f"Gen{link['lnksta_gen']}" if link.get("lnksta_gen") else "unknown generation" + if link_passes(link): + ok(f"FINAL PCIe CHECK PASS ({phase}): Gen2 x16 negotiated") + else: + err(f"FINAL PCIe CHECK FAILED ({phase}): PCIe {generation} {width} negotiated; " + "Gen2 x16 required after cmpunlocker.") + + +def start_card_folder(serial, bdf): + run_id = datetime.datetime.now().strftime("%Y%m%dT%H%M%S%f") + folder = (X.CARDS_DIR / sanitize(serial) if serial else + X.UNIDENTIFIED_DIR / f"{run_id}_{sanitize(bdf)}") + run_dir = folder / "runs" / run_id + run_dir.mkdir(parents=True) + return folder, run_dir + + +def promote_card_folder(folder, run_dir, serial, rep): + """Move a current unidentified run once the physical serial becomes readable.""" + target = X.CARDS_DIR / sanitize(serial) + if target.exists(): + new_run = target / "runs" / run_dir.name + new_run.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(run_dir), str(new_run)) + extras = folder / "extras" + if extras.exists(): + destination = target / "extras" + destination.mkdir(exist_ok=True) + for item in extras.iterdir(): + shutil.move(str(item), str(destination / item.name)) + shutil.rmtree(folder) + else: + folder.rename(target) + new_run = target / "runs" / run_dir.name + if rep.get("unlock", {}).get("install_log"): + old_log = Path(rep["unlock"]["install_log"]) + rep["unlock"]["install_log"] = str(target / "extras" / old_log.name) + rep["unlock"]["note"] = rep["unlock"].get("note", "").replace(str(folder), str(target)) + rep["run_dir"] = str(new_run) + info(f"physical serial found; moved current diagnostics to {target}") + return target, new_run + + +# ---------------------------------------------------------------- card detect + +def wait_for_gpu(): + """Wait for a single CMP 170HX (with rescan retries). Returns BDF or None.""" + while True: + for _ in range(5): + bdfs = detect_gpus() + if bdfs: + if len(bdfs) > 1: + err(f"multiple CMP 170HX devices present: {', '.join(bdfs)}; connect one card") + return None + return bdfs[0] + time.sleep(1) + if not sys.stdin.isatty(): + return None + ans = ask("CMP 170HX not detected - insert/reseat the card, press Enter to rescan " + "(q=quit)", "") + if ans.lower().startswith("q"): + return None + if not X.rescan_bus(): + warn("bus rescan not permitted (no root/sudo) - reseat the card physically") + + +# ---------------------------------------------------------------- one card + +def process_card(args): + step("Card check") + started_at = now_iso() + bdf = wait_for_gpu() + if not bdf: + err("no card detected - nothing to do") + return None + devid = "" + for line in X.run(["lspci", "-D", "-s", bdf, "-nn"])["out"].splitlines(): + if "[10de:" in line: + i = line.index("[10de:") + devid = line[i + 1:i + 10].rstrip("]") + ok(f"device found at {bdf} ({devid})") + + nf = X.nvflash_bin(args) + serial, serial_src = X.get_serial(nf, bdf) + if not serial: + warn("physical card serial unavailable; continuing diagnostics in unidentified/") + folder, run_dir = start_card_folder(serial, bdf) + info(f"diagnostics folder: {folder}") + + rep = { + "serial": serial, "serial_source": serial_src, "started_at": started_at, + "run_dir": str(run_dir), + "bdf": bdf, "device_id": devid, "host": X.run(["uname", "-n"])["out"].strip(), + "kernel": X.run(["uname", "-r"])["out"].strip(), + } + + step("capacitor lane width before unlock") + link_before, lspci_before = X.check_link(bdf) + rep["link_before_unlock"] = link_before + rep["link"] = link_before + rep["capacitor_mod_before_unlock"] = width_status(link_before) + rep["capacitor_mod"] = rep["capacitor_mod_before_unlock"] + (run_dir / "lspci_before_unlock.txt").write_text(lspci_before) + info(link_before.get("lnksta_raw") or "LnkSta: unavailable") + log_capacitor_width(link_before, "before unlock") + + step("cmpunlocker memory unlock") + identity = X.card_identity(bdf, devid, serial) + rep["gpu_uuid"] = identity.get("gpu_uuid") + rep["unlock"] = X.ensure_unlocked(bdf, devid, folder, identity, + allow_install=not args.no_install) + info(f"unlock: {rep['unlock']['status']} - {rep['unlock'].get('note', '')}") + if not serial: + found, source = X.get_serial(nf, bdf) + if found: + folder, run_dir = promote_card_folder(folder, run_dir, found, rep) + serial, serial_src = found, source + rep.update(serial=serial, serial_source=serial_src) + if rep["unlock"]["status"] != "unlocked": + rep["result"] = "PENDING_REBOOT" if rep["unlock"]["status"] == "reboot-required" else "FAIL" + if rep["result"] == "FAIL": + rep["failures"] = [rep["unlock"].get("note", "memory unlock failed")] + if not serial: + rep["failures"].append("physical card serial unavailable; diagnostics kept under unidentified/") + rep["finished_at"] = now_iso() + save_report(rep, folder, run_dir) + return rep + + step("cmpunlocker verification") + rep["cmpunlocker_verify"] = X.run_verify_unlocker(bdf, devid) + v = rep["cmpunlocker_verify"] + (run_dir / "verify_unlocker.log").write_text( + "\n".join(v.get("tail", [])) or (v.get("note") or "not run")) + info("verify: " + (f"rc={v.get('rc')} -> {v.get('status')}" if v.get("ran") + else f"skipped - {v.get('note')}")) + + step("PCIe link + config") + link, lspci_txt = X.check_link(bdf) + rep["link"] = link + rep["capacitor_mod_after_unlock"] = width_status(link) + rep["capacitor_mod"] = rep["capacitor_mod_after_unlock"] + (run_dir / "lspci.txt").write_text(lspci_txt) + (run_dir / "config_space.hex").write_text(X.dump_config(bdf)) + info(f"{link.get('lnksta_raw') or 'LnkSta: unavailable'} " + f"(setpci gen={link.get('setpci_current_gen')})") + log_capacitor_width(link, "after unlock") + log_final_link(link) + rep["aer_before"] = X.aer_snapshot(bdf) + + if args.link_tests: + step("optional retrain + rescan tests") + rep["retrain"] = X.retrain_test(bdf) + info(f"retrain: ok={rep['retrain']['ok']} - {rep['retrain']['note']}") + rep["rescan"] = X.rescan_test(bdf) + info(f"rescan: ok={rep['rescan']['ok']} - {rep['rescan']['note']}") + rep["aer_after"] = X.aer_snapshot(bdf) + link2, lspci2 = X.check_link(bdf) + rep["link_after"] = link2 + rep["capacitor_mod_after_diagnostics"] = width_status(link2) + if rep["capacitor_mod_after_diagnostics"] != "PASS": + rep["capacitor_mod"] = rep["capacitor_mod_after_diagnostics"] + if (link2.get("lnksta_width"), link2.get("lnksta_gen")) != (link.get("lnksta_width"), link.get("lnksta_gen")): + if link2.get("lnksta_width") != link.get("lnksta_width"): + log_capacitor_width(link2, "after diagnostics") + log_final_link(link2, "after diagnostics") + if lspci2: + (run_dir / "lspci_after.txt").write_text(lspci2) + + step("kernel log scrape") + dm = X.dmesg_capture(bdf, rep["started_at"]) + rep["dmesg"] = dm + (run_dir / "dmesg.log").write_text("\n".join(dm.get("related", []))) + (run_dir / "dmesg_other_pcie.log").write_text("\n".join(dm.get("other_pcie", []))) + info(f"{dm.get('note')} (source: {dm.get('source')})") + for e in dm.get("errors", [])[-5:]: + warn(e.strip()[:160]) + + step("nvidia-smi") + rep["nvidia_smi"] = X.nvidia_smi_info() + + if args.bios: + step("optional VBIOS backup") + extras = folder / "extras" + extras.mkdir(exist_ok=True) + rep["bios"] = X.save_bios(nf, extras / "bios.rom", bdf) + rep["bios"]["requested"] = True + (ok if rep["bios"]["ok"] else warn)(f"bios: {rep['bios']['method'] or 'FAILED'} " + f"{rep['bios']['size']} bytes - {rep['bios']['note']}") + else: + rep["bios"] = {"requested": False, "ok": None, "note": "not requested"} + + step("optional Vulkan capability probe") + if args.vulkan == "never": + rep["vulkan_probe"] = {"requested": False, "ran": False, "note": "disabled"} + elif args.vulkan == "always": + rep["vulkan_probe"] = X.run_vulkan_probe() + else: + if ask("Run optional Vulkan capability probe now? (y/N)", "n").lower().startswith("y"): + rep["vulkan_probe"] = X.run_vulkan_probe() + else: + rep["vulkan_probe"] = {"requested": False, "ran": False, "note": "operator declined"} + info("Vulkan probe skipped") + + result, fails, warns = score(rep) + rep.update(finished_at=now_iso(), result=result, failures=fails, warnings=warns) + save_report(rep, folder, run_dir) + + print() + label = serial or f"unidentified card at {bdf}" + if result == "PASS": + ok(f"{label}: PASS") + elif result == "WARN": + warn(f"{label}: WARN") + else: + err(f"{label}: FAIL") + for f in fails: + err(" FAIL: " + f) + for w in warns: + warn(" WARN: " + w) + info(f"report: {folder / 'report.json'}") + return rep + + +def save_report(rep, folder, run_dir): + encoded = json.dumps(rep, indent=2) + (run_dir / "report.json").write_text(encoded) + (folder / "report.json").write_text(encoded) + + +# ---------------------------------------------------------------- demo + +def demo(args): + """Validate report pipeline without hardware: writes cards/DEMO-*.""" + step("Demo mode (no hardware)") + args.vulkan = "never" + args.bios = False + args.link_tests = False + + orig_wait = globals()["wait_for_gpu"] + orig_detect, orig_check = X.detect_gpus, X.check_link + orig_aer, orig_dm, orig_cfg = X.aer_snapshot, X.dmesg_capture, X.dump_config + orig_retrain, orig_rescan = X.retrain_test, X.rescan_test + orig_smi, orig_verify, orig_unlock = X.nvidia_smi_info, X.run_verify_unlocker, X.ensure_unlocked + orig_serial, orig_identity = X.get_serial, X.card_identity + globals()["wait_for_gpu"] = lambda: "0000:0a:00.0" + X.detect_gpus = lambda: ["0000:0a:00.0"] + X.check_link = lambda b: ({"lnksta_speed": "5GT/s", "lnksta_width": "x16", "lnksta_gen": 2, + "lnkcap_speed": "8GT/s", "lnkcap_width": "x16", + "lnksta_raw": "LnkSta: Speed 5GT/s, Width x16, TrErr-", + "setpci_current_gen": 2, "lspci_rc": 0}, "demo lspci output\n") + X.aer_snapshot = lambda b: {"aer_sysfs_correctable": None, "aer_sysfs_fatal": None, + "aer_correctable_sum": 0, "ce_active": [], "ue_active": [], + "parent_bdf": None, "parent_ce_active": []} + X.dmesg_capture = lambda b, since: {"source": "demo", "related": ["demo line"], + "errors": [], "other_pcie": [], "note": "demo"} + X.dump_config = lambda b: "demo config space dump\n" + X.retrain_test = lambda b: {"attempted": True, "ok": True, "note": "x16 -> x16", + "width_before": "x16", "width_after": "x16"} + X.rescan_test = lambda b: {"attempted": True, "ok": True, "note": "re-enumerated"} + X.nvidia_smi_info = lambda: {"available": False} + X.run_verify_unlocker = lambda b, d: {"ran": True, "rc": 0, "status": "unlocked-ok", "tail": ["demo"]} + X.ensure_unlocked = lambda b, d, f, card=None, allow_install=True: {"status": "unlocked", "note": "demo"} + X.get_serial = lambda nf, bdf: ("DEMO-SERIAL-001", "demo") + X.card_identity = lambda b, d, serial=None: {"serial": serial, "gpu_uuid": "DEMO-UUID", "device_id": d, "bdf": b} + try: + rep = process_card(args) + finally: + globals()["wait_for_gpu"] = orig_wait + X.detect_gpus, X.check_link = orig_detect, orig_check + X.aer_snapshot, X.dmesg_capture, X.dump_config = orig_aer, orig_dm, orig_cfg + X.retrain_test, X.rescan_test = orig_retrain, orig_rescan + X.nvidia_smi_info, X.run_verify_unlocker, X.ensure_unlocked = orig_smi, orig_verify, orig_unlock + X.get_serial, X.card_identity = orig_serial, orig_identity + return rep + + +# ---------------------------------------------------------------- main + +def main(): + ap = argparse.ArgumentParser(description="CMP 170HX single-card checker (one card per run)") + ap.add_argument("--preflight", action="store_true", + help="prepare tools, driver and cmpunlocker source, then exit") + ap.add_argument("--demo", action="store_true", help="run pipeline with fake hardware (self-test)") + ap.add_argument("--nvflash", help="path to nvflash binary") + ap.add_argument("--bios", action="store_true", help="save VBIOS in cards//extras/") + ap.add_argument("--link-tests", action="store_true", help="also retrain and rescan PCIe bus") + ap.add_argument("--vulkan", choices=("ask", "always", "never"), default="never", + help="generic Vulkan capability probe policy (default never)") + ap.add_argument("--no-install", action="store_true", + help="do not install packages or cmpunlocker") + ap.add_argument("--no-download", action="store_true", help="use existing harness sources only") + args = ap.parse_args() + + if args.demo: + demo(args) + return 0 + pf = X.preflight(auto_install=not args.no_install, do_download=not args.no_download) + if args.preflight: + return 4 if pf.get("reboot_required") else 0 if pf.get("preflight_ok") else 3 + + if pf.get("reboot_required"): + return 4 + if not pf.get("preflight_ok"): + return 3 + + X.CARDS_DIR.mkdir(exist_ok=True) + rep = process_card(args) + if not rep: + return 2 + return 4 if rep.get("result") == "PENDING_REBOOT" else 1 if rep.get("result") == "FAIL" else 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyboardInterrupt: + print() + warn("interrupted") + sys.exit(130) diff --git a/cxlib.py b/cxlib.py new file mode 100755 index 0000000..322b7cc --- /dev/null +++ b/cxlib.py @@ -0,0 +1,884 @@ +#!/usr/bin/env python3 +"""CMP 170HX setup, unlock, PCIe checks, and optional diagnostics.""" +import datetime +import hashlib +import io +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +import urllib.error +import urllib.request +import zipfile +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +HARNESS_DIR = SCRIPT_DIR / "harness" +CARDS_DIR = SCRIPT_DIR / "cards" +UNIDENTIFIED_DIR = SCRIPT_DIR / "unidentified" + +UNLOCKER_COMMIT = "f51da030ef4260656288edf07e95ae45f339aea8" # upstream v0.3 +UNLOCKER_ZIP_SHA256 = "a10b226ed4667fb2ae4c9c5ad1575938a85ceb707f8c40ea146887778930ce21" +REPOS = {"cmpunlocker": ("commits", UNLOCKER_COMMIT)} +VERIFY_SH = HARNESS_DIR / "cmpunlocker" / "verify.sh" +INSTALL_SH = HARNESS_DIR / "cmpunlocker" / "install.sh" +PENDING_FILE = HARNESS_DIR / "unlock_pending.json" +GPU_IDS = ("20c2", "2082") + +SPEED_TO_GEN = {"2.5GT/s": 1, "5GT/s": 2, "8GT/s": 3, "16GT/s": 4, "32GT/s": 5, "64GT/s": 6} + +USE_COLOR = sys.stdout.isatty() + + +class C: + RESET = "\033[0m" if USE_COLOR else "" + BOLD = "\033[1m" if USE_COLOR else "" + DIM = "\033[2m" if USE_COLOR else "" + GREEN = "\033[32m" if USE_COLOR else "" + YELLOW = "\033[33m" if USE_COLOR else "" + RED = "\033[31m" if USE_COLOR else "" + CYAN = "\033[36m" if USE_COLOR else "" + + +def info(msg): + print(f"{C.CYAN}[*]{C.RESET} {msg}", flush=True) + + +def ok(msg): + print(f"{C.GREEN}[+]{C.RESET} {msg}", flush=True) + + +def warn(msg): + print(f"{C.YELLOW}[!]{C.RESET} {msg}", flush=True) + + +def err(msg): + print(f"{C.RED}[ERROR] {msg}{C.RESET}", flush=True) + + +def step(msg): + print(f"\n{C.BOLD}== {msg} =={C.RESET}", flush=True) + + +def ask(prompt, default=""): + suffix = f" [{default}]" if default else "" + try: + val = input(f"{C.BOLD}?{C.RESET} {prompt}{suffix}: ").strip() + except EOFError: + return default + return val if val else default + + +def now_iso(): + return datetime.datetime.now().isoformat(timespec="seconds") + + +def sanitize(name): + return re.sub(r"[^A-Za-z0-9._-]+", "_", name).strip("._-") or "UNKNOWN" + + +# ---------------------------------------------------------------- subprocess + +def run(cmd, timeout=60, env=None, cwd=None): + try: + p = subprocess.run(cmd, capture_output=True, text=True, errors="replace", + timeout=timeout, env=env, cwd=cwd) + return {"rc": p.returncode, "out": p.stdout, "err": p.stderr, "cmd": cmd} + except FileNotFoundError: + return {"rc": 127, "out": "", "err": f"not found: {cmd[0]}", "cmd": cmd} + except subprocess.TimeoutExpired as e: + out = e.stdout.decode(errors="replace") if isinstance(e.stdout, bytes) else (e.stdout or "") + return {"rc": 124, "out": out, "err": "timeout", "cmd": cmd} + + +def as_root(cmd, timeout=60, cwd=None): + if os.geteuid() == 0: + return run(cmd, timeout=timeout, cwd=cwd) + if shutil.which("sudo"): + r = run(["sudo", "-n"] + cmd, timeout=timeout, cwd=cwd) + if r["rc"] != 0 and "password is required" in (r["err"] + r["out"]).lower(): + r["err"] = "sudo requires a password (sudo -n failed)" + return r + return {"rc": 1, "out": "", "err": "no root: neither uid 0 nor sudo available", "cmd": cmd} + + +def run_logged_as_root(cmd, log_path, timeout, cwd=None): + """Show a long-running command live while keeping its full output on disk.""" + actual = cmd if os.geteuid() == 0 else ["sudo", "-n"] + cmd + log_path.parent.mkdir(parents=True, exist_ok=True) + tail = bytearray() + try: + with log_path.open("wb") as log: + process = subprocess.Popen(actual, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, cwd=cwd, + start_new_session=True) + + def copy_output(): + show_console = True + while True: + try: + chunk = os.read(process.stdout.fileno(), 4096) + except OSError: + break + if not chunk: + break + log.write(chunk) + log.flush() + if show_console: + try: + if hasattr(sys.stdout, "buffer"): + sys.stdout.buffer.write(chunk) + else: + sys.stdout.write(chunk.decode(errors="replace")) + sys.stdout.flush() + except BrokenPipeError: + show_console = False + tail.extend(chunk) + if len(tail) > 4096: + del tail[:-4096] + + def stop_group(sig): + try: + os.killpg(process.pid, sig) + except ProcessLookupError: + pass + + reader = threading.Thread(target=copy_output, daemon=True) + reader.start() + try: + rc = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + stop_group(signal.SIGTERM) + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + stop_group(signal.SIGKILL) + process.wait() + rc = 124 + except KeyboardInterrupt: + stop_group(signal.SIGTERM) + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + stop_group(signal.SIGKILL) + process.wait() + raise + finally: + reader.join(timeout=10) + if reader.is_alive(): + stop_group(signal.SIGTERM) + reader.join(timeout=5) + if reader.is_alive(): + stop_group(signal.SIGKILL) + reader.join(timeout=5) + process.stdout.close() + reader.join(timeout=2) + except OSError as exc: + return {"rc": 127, "out": "", "err": str(exc), "cmd": actual, + "log": str(log_path)} + return {"rc": rc, "out": tail.decode(errors="replace"), "err": "", + "cmd": actual, "log": str(log_path)} + + +def which(name): + return shutil.which(name) + + +def find_nvflash(): + """nvflash binary: repo-bundled copy first (no system install needed), then PATH.""" + import platform + arch_map = {"x86_64": "x64", "amd64": "x64", "aarch64": "aarch64", + "ppc64le": "ppc64", "ppc64": "ppc64", "i686": "x86", "i386": "x86"} + arch = arch_map.get(platform.machine(), "x64") + candidates = [ + *sorted(SCRIPT_DIR.glob(f"nvflash_*_linux/{arch}/nvflash")), + *sorted(SCRIPT_DIR.glob("nvflash_*_linux/x64/nvflash")), + SCRIPT_DIR / "nvflash", + ] + for p in candidates: + if p.is_file() and os.access(p, os.X_OK): + return str(p) + return which("nvflash") + + +def have_root(): + return os.geteuid() == 0 or bool(shutil.which("sudo")) + + +# ---------------------------------------------------------------- preflight + +def ensure_repos(): + """Fetch upstream source archives without changing any Git remotes.""" + HARNESS_DIR.mkdir(exist_ok=True) + complete = True + for name, (kind, ref) in REPOS.items(): + dest = HARNESS_DIR / name + if (dest / ("install.sh" if name == "cmpunlocker" else "README.md")).exists(): + continue + suffix = f"archive/{ref}.zip" if kind == "commits" else f"archive/refs/{kind}/{ref}.zip" + url = f"https://github.com/amoghmunikote/{name}/{suffix}" + info(f"downloading {name} ({ref}) ...") + try: + with urllib.request.urlopen(url, timeout=60) as response: + archive = response.read() + if name == "cmpunlocker" and hashlib.sha256(archive).hexdigest() != UNLOCKER_ZIP_SHA256: + raise ValueError("cmpunlocker archive checksum differs from pinned v0.3 source") + with tempfile.TemporaryDirectory(dir=HARNESS_DIR) as temp: + staging = Path(temp) / name + staging.mkdir() + archive_root = f"{name}-{ref[1:] if kind == 'tags' and ref.startswith('v') else ref}" + with zipfile.ZipFile(io.BytesIO(archive)) as zf: + for member in zf.infolist(): + parts = Path(member.filename).parts + if parts == (archive_root,): + continue + if len(parts) < 2 or ".." in parts or parts[0] != archive_root: + raise ValueError(f"unsafe archive path: {member.filename}") + if (member.external_attr >> 16) & 0o170000 == 0o120000: + raise ValueError(f"archive symlink: {member.filename}") + path = staging.joinpath(*parts[1:]) + if member.is_dir(): + path.mkdir(parents=True, exist_ok=True) + else: + path.parent.mkdir(parents=True, exist_ok=True) + with zf.open(member) as source, path.open("wb") as output: + shutil.copyfileobj(source, output) + path.chmod(((member.external_attr >> 16) & 0o777) or 0o644) + if dest.exists(): + raise ValueError(f"{dest} exists but lacks required source files") + staging.rename(dest) + ok(f"downloaded {name}") + except (OSError, ValueError, zipfile.BadZipFile, urllib.error.URLError) as exc: + err(f"could not download {name}: {exc}") + complete = False + return complete + + +TOOL_CHECKS = { + "lspci": ("critical", "PCIe checks (package: pciutils)"), + "setpci": ("critical", "link generation probe (package: pciutils)"), + "nvidia-smi": ("critical", "GPU state / cmpunlocker verify"), + "nvflash": ("optional", "VBIOS backup + serial (bundled in repo, or on PATH)"), + "dmesg": ("optional", "kernel log scrape"), + "journalctl": ("optional", "kernel log scrape fallback"), + "vulkaninfo": ("optional", "Vulkan capability probe (package: vulkan-tools)"), +} +def os_release(): + data = read_file("/etc/os-release") or "" + out = {} + for line in data.splitlines(): + if "=" in line: + k, v = line.split("=", 1) + out[k] = v.strip().strip('"') + return out + + +def sudo_precondition(interactive=True): + """Check root or working sudo for host setup and privileged inspection. + + Tries passwordless sudo first; falls back to an interactive `sudo -v` so the + credentials get cached, after which plain sudo -n calls succeed. + """ + if os.geteuid() == 0: + return True, "running as root" + if not shutil.which("sudo"): + return False, "sudo is not installed" + r = run(["sudo", "-n", "true"], timeout=15) + if r["rc"] == 0: + return True, "sudo available (credentials cached)" + if interactive and sys.stdin.isatty(): + info("sudo authentication required - enter your password") + try: + p = subprocess.run(["sudo", "-v"]) + except KeyboardInterrupt: + return False, "interrupted" + if p.returncode == 0: + return True, "sudo authenticated just now" + return False, "sudo authentication failed" + return False, "sudo requires a password (non-interactive session)" + + +def boot_id(): + return (read_file("/proc/sys/kernel/random/boot_id") or "").strip() + + +def pending_state(): + try: + return json.loads(PENDING_FILE.read_text()) + except (OSError, ValueError): + return None + + +def mark_pending(stage, card=None): + HARNESS_DIR.mkdir(exist_ok=True) + PENDING_FILE.write_text(json.dumps({"stage": stage, "boot_id": boot_id(), + "card": card}, indent=2)) + + +def card_identity(bdf, device_id, serial=None): + """Identify a card across boots without treating its GPU UUID as a serial.""" + query = run(["nvidia-smi", "-i", bdf, "--query-gpu=uuid", + "--format=csv,noheader"], timeout=30) + uuid = query["out"].strip().splitlines()[0] if query["rc"] == 0 and query["out"].strip() else None + if uuid and uuid.upper() in ("N/A", "UNKNOWN", "NONE"): + uuid = None + return {"serial": serial, "gpu_uuid": uuid, "device_id": device_id, "bdf": bdf} + + +def same_card(previous, current): + """Return None when no stable identity is available for comparison.""" + if not isinstance(previous, dict): + return None + for key in ("serial", "gpu_uuid"): + if previous.get(key) and current.get(key): + return previous[key] == current[key] + if previous.get("device_id") and previous["device_id"] != current["device_id"]: + return False + return None + + +def supported_versions(): + text = read_file(HARNESS_DIR / "cmpunlocker" / "driver" / "VERSION") or "" + return [line.strip() for line in text.splitlines() if re.fullmatch(r"\d+\.\d+\.\d+", line.strip())] + + +def current_driver_version(): + r = run(["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader"], timeout=30) + if r["rc"] == 0: + return r["out"].splitlines()[0].strip() if r["out"].strip() else None + return None + + +def open_driver_active(): + version_text = read_file("/proc/driver/nvidia/version") or "" + return "Open Kernel Module" in version_text + + +def preflight(auto_install=True, do_download=True): + step("Preflight") + report = {"tools": {}, "root": have_root()} + rel = os_release() + report["os"] = {"id": rel.get("ID", ""), "version_id": rel.get("VERSION_ID", ""), + "pretty": rel.get("PRETTY_NAME", "unknown")} + info(f"OS: {report['os']['pretty']}") + if do_download: + report["repos_ok"] = ensure_repos() + versions = supported_versions() + report["supported_driver_versions"] = versions + if not INSTALL_SH.exists() or not VERIFY_SH.exists() or not versions: + report["blocking_missing"] = ["cmpunlocker v0.3 source"] + report["preflight_ok"] = False + err("preflight blocked: cmpunlocker v0.3 source is missing or incomplete") + return report + pending = pending_state() + if (pending and pending.get("stage") == "nvidia-driver-610-open" + and pending.get("boot_id") == boot_id()): + report["reboot_required"] = True + warn(f"{pending['stage']} installed during this boot; cold shut down and power on, then rerun") + return report + driver_rebooted = bool(pending and pending.get("boot_id") != boot_id() + and pending.get("stage") == "nvidia-driver-610-open") + version = current_driver_version() + report["driver_version"] = version + need_driver = not version or version not in versions or not open_driver_active() + if driver_rebooted and need_driver: + report["blocking_missing"] = ["nvidia-driver-610-open installed but inactive after reboot; inspect Secure Boot and driver logs"] + report["preflight_ok"] = False + err(report["blocking_missing"][0]) + return report + if driver_rebooted: + PENDING_FILE.unlink(missing_ok=True) + pkgs = ["pciutils", "build-essential", "patch", "dkms", "curl", + f"linux-headers-{run(['uname', '-r'])['out'].strip()}"] + build_tools = ("lspci", "setpci", "gcc", "make", "patch", "curl") + need_tools = not all(which(t) for t in build_tools) + need_headers = not Path("/lib/modules", run(["uname", "-r"])["out"].strip(), "build").exists() + if auto_install and (need_tools or need_headers or need_driver) and rel.get("ID") == "ubuntu": + allowed, reason = sudo_precondition() + report["sudo"] = {"ok": allowed, "why": reason} + if allowed: + update_log = HARNESS_DIR / "logs" / f"apt_update_{datetime.datetime.now().strftime('%Y%m%dT%H%M%S%f')}.log" + info(f"apt-get update log: {update_log}") + update = run_logged_as_root(["apt-get", "update"], update_log, timeout=300) + if update["rc"] != 0: + err(f"apt-get update failed: {(update['err'] or update['out'])[-300:]}") + else: + to_install = pkgs + (["nvidia-driver-610-open"] if need_driver else []) + info("installing: " + " ".join(to_install)) + install_log = HARNESS_DIR / "logs" / f"apt_install_{datetime.datetime.now().strftime('%Y%m%dT%H%M%S%f')}.log" + info(f"apt-get install log: {install_log}") + install = run_logged_as_root(["apt-get", "install", "-y"] + to_install, + install_log, timeout=1800) + report["install"] = {"rc": install["rc"], "log": str(install_log), + "tail": (install["out"] + install["err"])[-2000:]} + if install["rc"] != 0: + err("package installation failed: " + report["install"]["tail"][-300:]) + elif need_driver: + mark_pending("nvidia-driver-610-open") + report["reboot_required"] = True + warn("driver installed; cold shut down and power on, then rerun") + else: + err(f"sudo unavailable: {reason}") + + for tool, (level, why) in TOOL_CHECKS.items(): + path = find_nvflash() if tool == "nvflash" else which(tool) + report["tools"][tool] = {"found": bool(path), "path": path, "level": level, "why": why} + info(f"{tool}: {path or 'missing'}") + blocking = [t for t in build_tools if not which(t)] + if not report.get("reboot_required") and (current_driver_version() not in versions or not open_driver_active()): + blocking.append("supported active nvidia-open 610 driver") + if need_headers and not Path("/lib/modules", run(["uname", "-r"])["out"].strip(), "build").exists(): + blocking.append("matching kernel headers") + report["blocking_missing"] = blocking + report["preflight_ok"] = not blocking and not report.get("reboot_required") + if blocking and not report.get("reboot_required"): + err("preflight blocked: " + ", ".join(blocking)) + elif report["preflight_ok"]: + ok("preflight complete") + return report + + +# ---------------------------------------------------------------- GPU detect + +def detect_gpus(): + bdfs = [] + for dev_id in GPU_IDS: + r = run(["lspci", "-D", "-d", f"10de:{dev_id}"]) + for line in r["out"].splitlines(): + parts = line.split() + if not parts: + continue + bdf = parts[0] + if re.match(r"^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-f]$", bdf): + bdfs.append(bdf.lower()) + return sorted(set(bdfs)) + + +def rescan_bus(): + r = as_root(["sh", "-c", "echo 1 > /sys/bus/pci/rescan"], timeout=15) + if r["rc"] == 0: + time.sleep(2) + return r["rc"] == 0 + + +def sysfs(bdf): + return Path("/sys/bus/pci/devices") / bdf + + +def read_file(path, binary=False): + try: + with open(path, "rb" if binary else "r", errors="replace") as f: + return f.read() + except (PermissionError, FileNotFoundError, OSError): + return None + + +# ---------------------------------------------------------------- PCIe checks + +def parse_link(text): + out = {"lnksta_speed": None, "lnksta_width": None, "lnksta_gen": None, + "lnkcap_speed": None, "lnkcap_width": None, "lnksta_raw": None} + for line in text.splitlines(): + s = line.strip() + if s.startswith("LnkSta:") and not s.startswith("LnkSta2"): + out["lnksta_raw"] = s + m = re.search(r"Speed\s+([^,\s]+)", s) + if m: + out["lnksta_speed"] = m.group(1) + out["lnksta_gen"] = SPEED_TO_GEN.get(m.group(1)) + m = re.search(r"Width\s+(x\d+)", s) + if m: + out["lnksta_width"] = m.group(1) + elif s.startswith("LnkCap:"): + m = re.search(r"Speed\s+([^,\s]+)", s) + if m: + out["lnkcap_speed"] = m.group(1) + m = re.search(r"Width\s+(x\d+)", s) + if m: + out["lnkcap_width"] = m.group(1) + return out + + +def parse_aer_flags(text): + res = {"CESta": [], "UESta": []} + for line in text.splitlines(): + s = line.strip() + for key in ("CESta", "UESta"): + if s.startswith(key + ":"): + body = s.split(":", 1)[1] + res[key].extend(re.findall(r"([A-Za-z0-9_]+)\+", body)) + return res + + +def parent_bdf(bdf): + path = Path(os.path.realpath(sysfs(bdf))) + while True: + path = path.parent + name = path.name + if re.match(r"^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-f]$", name): + return name.lower() + if name.startswith("pci") or path.parent == path: + return None + + +def aer_snapshot(bdf): + snap = {"aer_sysfs_correctable": None, "aer_sysfs_fatal": None, + "aer_correctable_sum": 0, "aer_fatal_sum": 0, + "ce_active": [], "ue_active": [], + "parent_bdf": None, "parent_ce_active": []} + base = sysfs(bdf) + data = read_file(base / "aer_dev_correctable") + if data is not None: + snap["aer_sysfs_correctable"] = data.strip() + snap["aer_correctable_sum"] = sum(int(x) for x in re.findall(r"\d+", data)) + fatal = read_file(base / "aer_dev_fatal") + if fatal is not None: + snap["aer_sysfs_fatal"] = fatal.strip() + snap["aer_fatal_sum"] = sum(int(x) for x in re.findall(r"\d+", fatal)) + r = run(["lspci", "-vvv", "-D", "-s", bdf], timeout=30) + if r["rc"] == 0: + aer = parse_aer_flags(r["out"]) + snap["ce_active"] = aer["CESta"] + snap["ue_active"] = aer["UESta"] + parent = parent_bdf(bdf) + if parent: + snap["parent_bdf"] = parent + rp = run(["lspci", "-vvv", "-D", "-s", parent], timeout=30) + if rp["rc"] == 0: + snap["parent_ce_active"] = parse_aer_flags(rp["out"])["CESta"] + return snap + + +def link_gen_setpci(bdf): + r = as_root(["setpci", "-s", bdf, "CAP_EXP+12.w"], timeout=15) + val = r["out"].strip() + if re.match(r"^[0-9a-fA-F]{4}$", val): + return int(val, 16) & 0x0F + return None + + +def check_link(bdf): + cmd = ["lspci", "-vvv", "-D", "-s", bdf] + r = run(cmd, timeout=30) + link = parse_link(r["out"]) if r["rc"] == 0 else {} + if (not link.get("lnksta_width") or not link.get("lnksta_gen")) and os.geteuid() != 0: + allowed, _ = sudo_precondition() + if allowed: + privileged = as_root(cmd, timeout=30) + privileged_link = parse_link(privileged["out"]) if privileged["rc"] == 0 else {} + fields = ("lnksta_width", "lnksta_gen") + if sum(bool(privileged_link.get(field)) for field in fields) > sum( + bool(link.get(field)) for field in fields): + r, link = privileged, privileged_link + link["setpci_current_gen"] = link_gen_setpci(bdf) + link["lspci_rc"] = r["rc"] + return link, r["out"] + + +def dump_config(bdf): + r = run(["lspci", "-nnvvxxxx", "-D", "-s", bdf], timeout=30) + return r["out"] if r["rc"] == 0 else f"lspci failed rc={r['rc']}: {r['err']}" + + +def retrain_test(bdf): + result = {"attempted": False, "ok": None, "note": "", "width_before": None, "width_after": None} + link_before, _ = check_link(bdf) + result["width_before"] = link_before.get("lnksta_width") + node = sysfs(bdf) / "link" / "retrain" + if not node.exists(): + result["note"] = "sysfs link/retrain not available on this kernel" + return result + result["attempted"] = True + r = as_root(["sh", "-c", f"echo 1 > {node}"], timeout=15) + if r["rc"] != 0: + result["note"] = f"retrain write failed: {r['err'].strip()}" + return result + time.sleep(3) + if bdf not in detect_gpus(): + result.update(ok=False, note="device vanished after retrain") + return result + link_after, _ = check_link(bdf) + result["width_after"] = link_after.get("lnksta_width") + result["ok"] = result["width_after"] == result["width_before"] and result["width_after"] is not None + result["note"] = f"{result['width_before']} -> {result['width_after']}" + return result + + +def rescan_test(bdf): + result = {"attempted": True, "ok": None, "note": ""} + if not rescan_bus(): + result["note"] = "bus rescan not permitted (no root)" + return result + time.sleep(1) + present = bdf in detect_gpus() + result["ok"] = present + result["note"] = "re-enumerated at same BDF" if present else "device missing after rescan" + return result + + +def dmesg_capture(bdf, since): + """Warn only for recent lines naming this endpoint; keep others separately.""" + since = since.replace("T", " ") + text, source = "", None + for command, label in ( + (["journalctl", "-k", "-b", "--since", since, "--no-pager", "-o", "short-precise"], "journalctl"), + (["dmesg", "--ctime", "--since", since], "dmesg")): + r = as_root(command, timeout=25) + if r["rc"] == 0: + text, source = r["out"], label + break + if source is None: + return {"source": None, "related": [], "errors": [], "other_pcie": [], + "note": "recent kernel log unavailable (root or journal access needed)"} + tokens = (bdf.lower(), bdf[:-2].lower()) + related, errors, other = [], [], [] + for line in text.splitlines(): + low = line.lower() + if any(token in low for token in tokens): + related.append(line) + if any(k in low for k in ("xid", "error", "aer:", "fail", "corrupted", "fallen off")): + errors.append(line) + elif any(k in low for k in ("aer:", "pcieport", "nvrm", "xid")): + other.append(line) + return {"source": source, "related": related[-200:], "errors": errors[-50:], + "other_pcie": other[-200:], "note": f"{len(related)} card lines, {len(other)} other PCIe/GPU lines since {since}"} + + +# ---------------------------------------------------------------- BIOS / serial + +def nvflash_bin(args): + if getattr(args, "nvflash", None): + return args.nvflash + return find_nvflash() + + +def usable_serial(value): + return bool(value and sanitize(value) == value and + value.upper() not in ("N/A", "UNKNOWN", "NONE") and + not re.fullmatch(r"0+", value)) + + +def get_serial(nf, bdf): + """Read the physical board serial; never substitute a GPU UUID or PCI address.""" + smi = run(["nvidia-smi", "-i", bdf, "--query-gpu=serial", + "--format=csv,noheader"], timeout=30) + if smi["rc"] == 0: + serial = smi["out"].strip().splitlines()[0] if smi["out"].strip() else "" + if usable_serial(serial): + info(f"serial auto-read from nvidia-smi: {serial}") + return serial, "nvidia-smi" + + detail = run(["nvidia-smi", "-q", "-i", bdf], timeout=30) + if detail["rc"] == 0: + match = re.search(r"^\s*Serial Number\s*:\s*(\S+)", detail["out"], re.I | re.M) + if match and usable_serial(match.group(1)): + info(f"serial auto-read from nvidia-smi details: {match.group(1)}") + return match.group(1), "nvidia-smi-detail" + + vpd_cmd = ["lspci", "-vvv", "-D", "-s", bdf] + for vpd in (run(vpd_cmd, timeout=30), as_root(vpd_cmd, timeout=30)): + if vpd["rc"] == 0: + match = re.search(r"\[SN\]\s+Serial number:\s*(\S+)", vpd["out"], re.I) + if match and usable_serial(match.group(1)): + info(f"serial auto-read from PCI VPD: {match.group(1)}") + return match.group(1), "pci-vpd" + + if nf: + q = as_root([nf, "--pcisegbus", bdf, "--query"], timeout=40) + serials = {m.group(1) for m in re.finditer( + r"Serial(?:\s*Number)?\s*[:=]\s*(\S+)", q["out"], re.I) + if q["rc"] == 0 and usable_serial(m.group(1))} + if q["rc"] == 0 and len(serials) == 1: + serial = serials.pop() + info(f"serial auto-read from card-scoped nvflash query: {serial}") + return serial, "nvflash-pcisegbus-query" + return None, "unavailable" + + +def normalize_bdf(value): + match = re.search(r"([0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7])$", value) + return match.group(1).lower() if match else None + + +def memory_state(bdf, device_id): + """Identify unlock from this card's reported memory, independent of Gen2 status.""" + r = run(["nvidia-smi", "--query-gpu=pci.bus_id,memory.total", + "--format=csv,noheader,nounits"], timeout=30) + if r["rc"] != 0: + return {"status": "unavailable", "note": (r["err"] or r["out"])[-300:]} + for line in r["out"].splitlines(): + fields = [field.strip() for field in line.split(",")] + if len(fields) != 2 or normalize_bdf(fields[0]) != bdf: + continue + try: + mib = int(fields[1]) + except ValueError: + return {"status": "unavailable", "note": f"invalid memory total: {fields[1]}"} + if device_id == "10de:20c2": + status = "unlocked" if mib >= 60000 else "stock" if 7680 <= mib <= 8704 else "unexpected" + elif device_id == "10de:2082": + status = "unlocked" if 35000 <= mib < 60000 else "stock" if 9728 <= mib <= 10752 else "unexpected" + else: + status = "unsupported" + return {"status": status, "memory_mib": mib} + return {"status": "unavailable", "note": f"{bdf} absent from nvidia-smi"} + + +def ensure_unlocked(bdf, device_id, folder, card=None, allow_install=True): + before = memory_state(bdf, device_id) + result = {"before": before, "status": before["status"], "install_rc": None} + pending = pending_state() + if pending and pending.get("stage") == "cmpunlocker": + if pending.get("boot_id") == boot_id(): + result.update(status="reboot-required", note="cold shut down and power on after cmpunlocker install") + return result + match = same_card(pending.get("card"), card or card_identity(bdf, device_id)) + if match is False: + warn("previous unlock marker belongs to a different card; checking this card anew") + PENDING_FILE.unlink(missing_ok=True) + elif match is None and before["status"] != "unlocked": + result.update(status="failed", note="cannot tie the previous unlock to this card; inspect the pending marker before retrying") + return result + elif before["status"] != "unlocked": + result.update(status="failed", note="cmpunlocker installed but card is not unlocked after reboot") + return result + else: + PENDING_FILE.unlink(missing_ok=True) + if before["status"] == "unlocked": + result.update(status="unlocked", note="unlocked memory is active") + return result + if before["status"] != "stock": + result.update(status="failed", note="card memory/PCI ID is unavailable or unexpected; refusing to select an unlock profile") + return result + if not allow_install: + result.update(status="failed", note="unlock inactive and --no-install was requested") + return result + if not INSTALL_SH.exists(): + result.update(status="failed", note="cmpunlocker install.sh is missing") + return result + allowed, reason = sudo_precondition() + if not allowed: + result.update(status="failed", note=f"root access required: {reason}") + return result + info("installing cmpunlocker for the detected card (this can take several minutes)") + log_path = (folder / "extras" / + f"cmpunlocker_install_{datetime.datetime.now().strftime('%Y%m%dT%H%M%S%f')}.log") + info(f"cmpunlocker install log: {log_path}") + install = run_logged_as_root(["bash", str(INSTALL_SH)], log_path, timeout=3600, + cwd=INSTALL_SH.parent) + result["install_rc"] = install["rc"] + result["install_log"] = str(log_path) + if install["rc"] != 0: + result.update(status="failed", note=f"cmpunlocker installer failed (rc={install['rc']}); see {log_path}") + return result + mark_pending("cmpunlocker", card or card_identity(bdf, device_id)) + result.update(status="reboot-required", note="cmpunlocker installed; cold shut down and power on, then rerun") + return result + + +def save_bios(nf, out_path, bdf): + result = {"method": None, "path": None, "sha256": None, "size": 0, "ok": False, "note": ""} + if out_path.exists(): + data = out_path.read_bytes() + if len(data) > 512 and data.strip(b"\x00"): + result.update(method="existing", path=str(out_path), size=len(data), + sha256=hashlib.sha256(data).hexdigest(), ok=True, + note="existing backup preserved") + else: + result["note"] = "existing backup is empty or invalid; move it before retrying" + return result + if nf: + r = as_root([nf, "--pcisegbus", bdf, "--save", str(out_path)], timeout=90) + if r["rc"] == 0 and out_path.exists() and out_path.stat().st_size > 512: + data = out_path.read_bytes() + if data.strip(b"\x00") == b"": + result["note"] = "nvflash wrote an all-zero image" + else: + result.update(method="nvflash", path=str(out_path), size=len(data), + sha256=hashlib.sha256(data).hexdigest(), ok=True) + return result + else: + result["note"] = f"card-scoped nvflash --save failed or image too small (rc={r['rc']}): {(r['err'] or r['out'])[-200:]}" + out_path.unlink(missing_ok=True) + else: + result["note"] = "nvflash unavailable" + + rom = sysfs(bdf) / "rom" + if as_root(["sh", "-c", f"echo 1 > {rom}"], timeout=15)["rc"] == 0: + command = ["cat", str(rom)] if os.geteuid() == 0 else ["sudo", "-n", "cat", str(rom)] + try: + read = subprocess.run(command, capture_output=True, timeout=30) + data = read.stdout if read.returncode == 0 else None + except (OSError, subprocess.TimeoutExpired): + data = None + finally: + as_root(["sh", "-c", f"echo 0 > {rom}"], timeout=15) + if data and len(data) > 512 and data.strip(b"\x00") != b"": + out_path.write_bytes(data) + result.update(method="sysfs-rom", path=str(out_path), size=len(data), + sha256=hashlib.sha256(data).hexdigest(), ok=True) + result["note"] = (result["note"] + " | saved via sysfs rom").strip(" |") + return result + result["note"] += " | sysfs rom empty/all-zero" + else: + result["note"] += " | sysfs rom enable failed" + return result + + +# ---------------------------------------------------------------- nvidia / verify + +def nvidia_smi_info(): + if not which("nvidia-smi"): + return {"available": False} + q = run(["nvidia-smi", "--query-gpu=name,uuid,memory.total,pcie.link.gen.current," + "pcie.link.gen.max,pcie.link.width.current,temperature.gpu,driver_version", + "--format=csv,noheader"], timeout=30) + if q["rc"] != 0: + return {"available": False, "error": (q["err"] or q["out"])[-300:]} + rows = [[f.strip() for f in line.split(",")] for line in q["out"].strip().splitlines()] + return {"available": True, "gpus": rows} + + +def run_verify_unlocker(bdf, device_id): + if not VERIFY_SH.exists(): + return {"ran": False, "note": "harness/cmpunlocker source missing"} + if not which("nvidia-smi"): + return {"ran": False, "note": "nvidia-smi missing - unlock verify skipped"} + kernel = run(["uname", "-r"])["out"].strip() + inventory = Path("/lib/modules") / kernel / "updates" / "cmpunlocker" / "gpu_inventory" + contents = read_file(inventory) + if contents: + entries = [line.split() for line in contents.splitlines() + if line.strip() and not line.lstrip().startswith("#")] + if len(entries) != 1 or len(entries[0]) < 2 or normalize_bdf(entries[0][0]) != bdf or not entries[0][1].lower().endswith(device_id[-4:]): + return {"ran": False, "status": "skipped-stale-inventory", + "note": f"installed gpu_inventory does not match {bdf} ({device_id}); live per-card memory and PCIe checks remain authoritative"} + r = as_root(["bash", str(VERIFY_SH)], timeout=180) + lines = [l for l in (r["out"] + r["err"]).splitlines() if l.strip()] + status = "unknown" + if r["rc"] == 0 and any("unlocked memory" in l.lower() for l in lines): + status = "unlocked-ok" + elif "STOCK" in r["out"]: + status = "stock-memory" + elif "MISSING" in r["out"]: + status = "gpu-missing-in-smi" + elif r["rc"] != 0: + status = f"verify-failed-rc{r['rc']}" + return {"ran": True, "rc": r["rc"], "status": status, "tail": lines[-30:]} + + +# ---------------------------------------------------------------- optional Vulkan capability probe + +def run_vulkan_probe(): + if not which("vulkaninfo"): + return {"requested": True, "ran": False, "note": "vulkaninfo is missing"} + r = run(["vulkaninfo", "--summary"], timeout=30) + return {"requested": True, "ran": True, "rc": r["rc"], + "note": "generic Vulkan capability listing; does not validate CMP VRAM", + "log_tail": (r["out"] + r["err"]).splitlines()[-40:]} diff --git a/nvflash_5.867_linux.zip b/nvflash_5.867_linux.zip new file mode 100755 index 0000000..2276909 Binary files /dev/null and b/nvflash_5.867_linux.zip differ diff --git a/nvflash_5.867_linux/aarch64/nvflash b/nvflash_5.867_linux/aarch64/nvflash new file mode 100755 index 0000000..c2bff55 Binary files /dev/null and b/nvflash_5.867_linux/aarch64/nvflash differ diff --git a/nvflash_5.867_linux/ppc64/nvflash b/nvflash_5.867_linux/ppc64/nvflash new file mode 100755 index 0000000..2706c1d Binary files /dev/null and b/nvflash_5.867_linux/ppc64/nvflash differ diff --git a/nvflash_5.867_linux/x64/nvflash b/nvflash_5.867_linux/x64/nvflash new file mode 100755 index 0000000..a85aa9b Binary files /dev/null and b/nvflash_5.867_linux/x64/nvflash differ diff --git a/nvflash_5.867_linux/x86/nvflash b/nvflash_5.867_linux/x86/nvflash new file mode 100755 index 0000000..e1b70d8 Binary files /dev/null and b/nvflash_5.867_linux/x86/nvflash differ