fix(pcie-probe): select a card by ordinal only, honour probe failures, refuse blind

Per-card measurement was unreachable for every card but ordinal 0: the driver
set HIP_VISIBLE_DEVICES=N and still passed N as the device argument, so the
probe called hipSetDevice(N) on a runtime where the mask left only ordinal 0.
The bats stub reported the env var rather than the argument it received, so
the suite asserted the mask was set and never noticed the collision.

- one selector: the ordinal is passed as argv, the mask is never set
- a probe that fails is now exit 5 and names probe + device; the sweep still
  measures the remaining cards instead of swallowing the failure
- the guard fails closed when `fuser` is missing (gpu_busy_percent reads 0 for
  a resident-but-idle service), with PCIE_PROBE_FUSER as the test seam
- stub probes now model the HIP mask rule; the pre-existing per-device test
  goes red against the old driver, which is the regression proof

Refs #60

tests/pcie-probe.bats

@@ -6,7 +6,8 @@
 # built or run, and the tool stops nothing and starts nothing (auto-stopping a production service is
 # the thing it must not do unasked).
 #
-# Exit status: 0 measured · 2 usage · 3 refused (GPU busy) · 4 no toolchain / build failed.
+# Exit status: 0 measured · 2 usage · 3 refused (GPU busy) · 4 no toolchain / build failed ·
+# 5 a probe ran and failed.
 #
 # hipcc and the compiled probes are stubbed: this box has no HIP SDK (it is a client laptop) and a
 # real measurement belongs on a machine with the cards, beside no running model.
@@ -46,7 +47,16 @@ while [ $# -gt 0 ]; do [ "$1" = "-o" ] && out="$2"; shift; done
 : > "$PCIE_PROBE_BUILD/.built"
 { echo '#!/usr/bin/env bash'
   echo ': > "$PCIE_PROBE_BUILD/.ran"'
-  echo 'echo "PROBE RAN dev=${HIP_VISIBLE_DEVICES:-none} bin=$(basename "$0")"'
+  # HIP rule, modelled because it is the trap: with HIP_VISIBLE_DEVICES set, the ONLY
+  # valid ordinal is 0. A driver that masks the device and still passes its real ordinal
+  # asks for a card the runtime cannot see — which is why the stub used to report the
+  # env var instead of the argument, and could not see the collision.
+  echo 'dev="${1:-0}"'
+  echo 'if [ -n "${HIP_VISIBLE_DEVICES:-}" ] && [ "$dev" != 0 ]; then'
+  echo '  printf "setDevice(%s) failed\n" "$dev" >&2; exit 1'
+  echo 'fi'
+  echo '[ "${STUB_PROBE_FAILS:-0}" = 1 ] && { echo "stub probe died mid-measurement" >&2; exit 1; }'
+  echo 'echo "PROBE RAN dev=$dev bin=$(basename "$0")"'
   echo 'echo "  slab 2.344 MiB: 31.4 GB/s"'
 } > "$out"
 chmod +x "$out"
@@ -131,6 +141,35 @@ teardown() { rm -rf "$TEST_TMP"; }
   [[ "$output" == *"dev=1"* ]]
 }
 
+@test "a device ordinal the environment mask would hide is still measured" {
+  # Reproduction, not preference: HIP_VISIBLE_DEVICES=1 leaves ordinal 0 as the only
+  # selectable card, so masking AND passing the ordinal asks hipSetDevice(1) for a
+  # device the runtime cannot see. Per-card bandwidth was unobtainable that way.
+  run "$SCRIPT" --device 0 --device 1
+  [ "$status" -eq 0 ]
+  [[ "$output" == *"PROBE RAN dev=1"* ]]
+  [[ "$output" != *"setDevice"* ]]
+}
+
+@test "a probe that dies is reported as a failed run, not a green one" {
+  # The header promises "0 measured". A probe that segfaults (the documented hazard on
+  # this ROCm build) must not let the sweep exit 0.
+  run env STUB_PROBE_FAILS=1 "$SCRIPT"
+  [ "$status" -eq 5 ]
+  [[ "$output" == *"failed"* ]]
+  [[ "$output" == *"hip_smoke"* ]]
+}
+
+@test "the guard refuses when it cannot see who holds the KFD, instead of assuming idle" {
+  # Without fuser the /dev/kfd half of the guard cannot run, and gpu_busy_percent reads
+  # 0 for a service that is resident but idle — a silent fallback would benchmark beside
+  # a live endpoint, which is the one thing this tool's LAW forbids.
+  run env PCIE_PROBE_FUSER="$TEST_TMP/nope-fuser" "$SCRIPT"
+  [ "$status" -eq 3 ]
+  [[ "$output" == *"fuser"* ]]
+  [ ! -e "$BUILD/.built" ]
+}
+
 @test "no probe sources is a usage-level error, not an empty success" {
   rm -f "$SRC"/*.hip
   run "$SCRIPT"