omp-docs/tools/pcie-probe.md

@@ -8,7 +8,7 @@ tags:
   - rdna4
   - benchmarking
   - dankserver
-verified: 2026-10-20
+verified: 2026-10-21
 timestamp: 2026-10-20
 ---
 
@@ -40,7 +40,10 @@ scripts/pcie-probe.sh --build-only         # compile, never touch the cards
 
 The guard runs **before anything is built or run** and refuses when the GPU is
 occupied — `gpu_busy_percent` above `PCIE_PROBE_MAX_BUSY` (default 5 %) or any
-process holding `/dev/kfd`. Refusal is exit `3`, and it stops nothing.
+process holding `/dev/kfd`. Refusal is exit `3`, and it stops nothing. If the holder
+probe is missing the guard refuses too rather than assuming idle — see
+[seams](#exit-status-and-seams); `gpu_busy_percent` reads `0` for a service that is
+resident and idle, so a silent fallback is exactly the benchmark this law forbids.
 
 Auto-stopping a production endpoint is not this tool's decision to make, and a
 measurement taken as noise beside serving traffic is worthless anyway: it is
@@ -65,8 +68,11 @@ throughput at *that* size is what matters, not the big-transfer figure:
 - Pinned versus pageable columns apart means pinning is load-bearing; together
   means the copies are already pinned by someone else.
 
-Measure one device at a time (`--device`). A run that lets the runtime pick
-several devices cannot attribute bandwidth to a card at all.
+Measure one device at a time (`--device`, a HIP ordinal). A run that lets the runtime
+pick several devices cannot attribute bandwidth to a card at all. The ordinal goes to
+the probe as an argument and **the driver never sets `HIP_VISIBLE_DEVICES`**: masking
+the device *and* passing its real ordinal asks `hipSetDevice` for a card the runtime
+has hidden, so every card but ordinal 0 fails to measure at all. One selector only.
 
 ## The trap carried in the sources
 
@@ -79,14 +85,18 @@ when touching these files.
 
 ## Exit status and seams
 
-`0` measured · `2` usage or no sources · `3` refused, GPU busy · `4` no HIP
-toolchain or build failed. A client box without the SDK exits `4` loudly rather
-than reporting an empty success; the probes belong on the box with the cards.
+`0` measured · `2` usage or no sources · `3` refused, GPU busy or holders unknowable ·
+`4` no HIP toolchain or build failed · `5` a probe ran and failed. A client box without
+the SDK exits `4` loudly rather than reporting an empty success; the probes belong on
+the box with the cards. A probe that dies (the segfault above, an unfundable
+allocation) exits `5` and names which probe on which device — a sweep that quietly
+swallowed the failure would report success with half its rows missing.
 
 Test seams (all documented in the script header): `PCIE_PROBE_SRC`,
 `PCIE_PROBE_BUILD`, `PCIE_PROBE_SYSFS`, `PCIE_PROBE_KFD`, `PCIE_PROBE_HIPCC`,
-`PCIE_PROBE_MAX_BUSY` — the sysfs/kfd paths are what make a refusal testable on
-a machine with no cards, which is the only honest way to test a refusal.
+`PCIE_PROBE_FUSER`, `PCIE_PROBE_MAX_BUSY` — the sysfs/kfd paths are what make a
+refusal testable on a machine with no cards, which is the only honest way to test a
+refusal.
 
 Measured campaign numbers are not kept here; they belong with the endpoint they
 were taken on, in the serverAppConfigs LLM runbook vault.

scripts/pcie-probe.sh

@@ -15,8 +15,12 @@
 #
 # usage: pcie-probe.sh [--device N]... [--build-only] [--force]
 #
-#   --device N    measure this device; repeat for each card. Without it the probes run once with the
-#                 environment's own device selection — one shared run cannot attribute bandwidth.
+#   --device N    measure HIP device ordinal N; repeat for each card. Without it the probes run
+#                 once with the environment's own device selection — one shared run cannot
+#                 attribute bandwidth. The ordinal is passed to the probes as argv, which is the
+#                 only selector that survives contact with HIP: HIP_VISIBLE_DEVICES=1 makes
+#                 ordinal 0 the sole visible card, so masking AND passing N asks the probe for a
+#                 device that no longer exists. The mask is deliberately not set here.
 #   --build-only  compile and stop; never touches the cards.
 #   --force       measure even though the guard says the GPU is busy.
 #
@@ -27,10 +31,11 @@
 #   PCIE_PROBE_SYSFS   drm sysfs root   (/sys/class/drm)
 #   PCIE_PROBE_KFD     KFD device node  (/dev/kfd)
 #   PCIE_PROBE_HIPCC   compiler         (hipcc)
+#   PCIE_PROBE_FUSER   KFD holder probe (fuser)
 #   PCIE_PROBE_MAX_BUSY  refusal threshold, percent (5)
 #
-# Exit status: 0 measured · 2 usage or missing sources · 3 REFUSED, GPU busy · 4 no toolchain or
-# build failed.
+# Exit status: 0 measured · 2 usage or missing sources · 3 REFUSED, GPU busy or unknowable ·
+# 4 no toolchain or build failed · 5 a probe ran and failed.
 
 set -uo pipefail
 
@@ -39,6 +44,7 @@ BUILD="${PCIE_PROBE_BUILD:-$HOME/.cache/pcie-probe}"
 SYSFS="${PCIE_PROBE_SYSFS:-/sys/class/drm}"
 KFD="${PCIE_PROBE_KFD:-/dev/kfd}"
 HIPCC="${PCIE_PROBE_HIPCC:-hipcc}"
+FUSER="${PCIE_PROBE_FUSER:-fuser}"
 MAX_BUSY="${PCIE_PROBE_MAX_BUSY:-5}"
 
 force=0
@@ -63,9 +69,19 @@ fi
 # ---- the guard, before anything touches the card --------------------------------
 if (( ! force )); then
   busy=""
-  if [[ -e "$KFD" ]] && command -v fuser > /dev/null 2>&1; then
-    holders="$(fuser "$KFD" 2>/dev/null | tr -s '[:space:]' ' ')"
-    [[ -n "${holders// /}" ]] && busy="KFD in use by:${holders}"
+  if [[ -e "$KFD" ]]; then
+    if command -v "$FUSER" > /dev/null 2>&1; then
+      holders="$("$FUSER" "$KFD" 2>/dev/null | tr -s '[:space:]' ' ')"
+      [[ -n "${holders// /}" ]] && busy="KFD in use by:${holders}"
+    else
+      # No fuser (psmisc) means the KFD holders are unknowable — and gpu_busy_percent reads 0
+      # for a service that is resident but idle. Guessing "idle" here is how you end up
+      # benchmarking beside a live endpoint, so the guard fails closed instead.
+      printf 'pcie-probe: REFUSED — cannot see who holds %s: `%s` not found (psmisc)\n' "$KFD" "$FUSER" >&2
+      printf '           install psmisc, or pass --force if you have decided the contention is\n' >&2
+      printf '           acceptable and read gpu_busy_percent yourself first.\n' >&2
+      exit 3
+    fi
   fi
   for card in "$SYSFS"/card*; do
     [[ -f "$card/device/gpu_busy_percent" ]] || continue
@@ -111,7 +127,11 @@ for card in "$SYSFS"/card*; do
 done
 
 # ---- measure, per device --------------------------------------------------------
+# One selector only (see --device above), and every probe's exit status is honoured: a probe
+# that dies must not let the sweep report success. Failures are collected so the remaining
+# cards are still measured — losing the second card's numbers is how a campaign lies.
 (( ${#devices[@]} > 0 )) || devices=(default)
+failures=()
 for d in "${devices[@]}"; do
   if [[ $d == default ]]; then
     printf '\n########## engine device selection ##########\n'
@@ -121,6 +141,15 @@ for d in "${devices[@]}"; do
   for name in hip_smoke pcie_bw pcie_probe; do
     bin="$BUILD/$name"
     [[ -x "$bin" ]] || continue
-    if [[ $d == default ]]; then "$bin"; else HIP_VISIBLE_DEVICES="$d" "$bin" "$d"; fi
+    if [[ $d == default ]]; then
+      "$bin" || failures+=("$name (engine selection)")
+    else
+      "$bin" "$d" || failures+=("$name (device $d)")
+    fi
   done
 done
+if (( ${#failures[@]} > 0 )); then
+  printf 'pcie-probe: %d probe(s) failed:\n' "${#failures[@]}" >&2
+  printf '           %s\n' "${failures[@]}" >&2
+  exit 5
+fi

tests/pcie-probe.bats

@@ -6,7 +6,8 @@
 # built or run, and the tool stops nothing and starts nothing (auto-stopping a production service is
 # the thing it must not do unasked).
 #
-# Exit status: 0 measured · 2 usage · 3 refused (GPU busy) · 4 no toolchain / build failed.
+# Exit status: 0 measured · 2 usage · 3 refused (GPU busy) · 4 no toolchain / build failed ·
+# 5 a probe ran and failed.
 #
 # hipcc and the compiled probes are stubbed: this box has no HIP SDK (it is a client laptop) and a
 # real measurement belongs on a machine with the cards, beside no running model.
@@ -46,7 +47,16 @@ while [ $# -gt 0 ]; do [ "$1" = "-o" ] && out="$2"; shift; done
 : > "$PCIE_PROBE_BUILD/.built"
 { echo '#!/usr/bin/env bash'
   echo ': > "$PCIE_PROBE_BUILD/.ran"'
-  echo 'echo "PROBE RAN dev=${HIP_VISIBLE_DEVICES:-none} bin=$(basename "$0")"'
+  # HIP rule, modelled because it is the trap: with HIP_VISIBLE_DEVICES set, the ONLY
+  # valid ordinal is 0. A driver that masks the device and still passes its real ordinal
+  # asks for a card the runtime cannot see — which is why the stub used to report the
+  # env var instead of the argument, and could not see the collision.
+  echo 'dev="${1:-0}"'
+  echo 'if [ -n "${HIP_VISIBLE_DEVICES:-}" ] && [ "$dev" != 0 ]; then'
+  echo '  printf "setDevice(%s) failed\n" "$dev" >&2; exit 1'
+  echo 'fi'
+  echo '[ "${STUB_PROBE_FAILS:-0}" = 1 ] && { echo "stub probe died mid-measurement" >&2; exit 1; }'
+  echo 'echo "PROBE RAN dev=$dev bin=$(basename "$0")"'
   echo 'echo "  slab 2.344 MiB: 31.4 GB/s"'
 } > "$out"
 chmod +x "$out"
@@ -131,6 +141,35 @@ teardown() { rm -rf "$TEST_TMP"; }
   [[ "$output" == *"dev=1"* ]]
 }
 
+@test "a device ordinal the environment mask would hide is still measured" {
+  # Reproduction, not preference: HIP_VISIBLE_DEVICES=1 leaves ordinal 0 as the only
+  # selectable card, so masking AND passing the ordinal asks hipSetDevice(1) for a
+  # device the runtime cannot see. Per-card bandwidth was unobtainable that way.
+  run "$SCRIPT" --device 0 --device 1
+  [ "$status" -eq 0 ]
+  [[ "$output" == *"PROBE RAN dev=1"* ]]
+  [[ "$output" != *"setDevice"* ]]
+}
+
+@test "a probe that dies is reported as a failed run, not a green one" {
+  # The header promises "0 measured". A probe that segfaults (the documented hazard on
+  # this ROCm build) must not let the sweep exit 0.
+  run env STUB_PROBE_FAILS=1 "$SCRIPT"
+  [ "$status" -eq 5 ]
+  [[ "$output" == *"failed"* ]]
+  [[ "$output" == *"hip_smoke"* ]]
+}
+
+@test "the guard refuses when it cannot see who holds the KFD, instead of assuming idle" {
+  # Without fuser the /dev/kfd half of the guard cannot run, and gpu_busy_percent reads
+  # 0 for a service that is resident but idle — a silent fallback would benchmark beside
+  # a live endpoint, which is the one thing this tool's LAW forbids.
+  run env PCIE_PROBE_FUSER="$TEST_TMP/nope-fuser" "$SCRIPT"
+  [ "$status" -eq 3 ]
+  [[ "$output" == *"fuser"* ]]
+  [ ! -e "$BUILD/.built" ]
+}
+
 @test "no probe sources is a usage-level error, not an empty success" {
   rm -f "$SRC"/*.hip
   run "$SCRIPT"