diff --git a/omp-docs/tools/pcie-probe.md b/omp-docs/tools/pcie-probe.md index 5a17f05..34acca3 100644 --- a/omp-docs/tools/pcie-probe.md +++ b/omp-docs/tools/pcie-probe.md @@ -8,7 +8,7 @@ tags: - rdna4 - benchmarking - dankserver -verified: 2026-10-20 +verified: 2026-10-21 timestamp: 2026-10-20 --- @@ -40,7 +40,10 @@ scripts/pcie-probe.sh --build-only # compile, never touch the cards The guard runs **before anything is built or run** and refuses when the GPU is occupied — `gpu_busy_percent` above `PCIE_PROBE_MAX_BUSY` (default 5 %) or any -process holding `/dev/kfd`. Refusal is exit `3`, and it stops nothing. +process holding `/dev/kfd`. Refusal is exit `3`, and it stops nothing. If the holder +probe is missing the guard refuses too rather than assuming idle — see +[seams](#exit-status-and-seams); `gpu_busy_percent` reads `0` for a service that is +resident and idle, so a silent fallback is exactly the benchmark this law forbids. Auto-stopping a production endpoint is not this tool's decision to make, and a measurement taken as noise beside serving traffic is worthless anyway: it is @@ -65,8 +68,11 @@ throughput at *that* size is what matters, not the big-transfer figure: - Pinned versus pageable columns apart means pinning is load-bearing; together means the copies are already pinned by someone else. -Measure one device at a time (`--device`). A run that lets the runtime pick -several devices cannot attribute bandwidth to a card at all. +Measure one device at a time (`--device`, a HIP ordinal). A run that lets the runtime +pick several devices cannot attribute bandwidth to a card at all. The ordinal goes to +the probe as an argument and **the driver never sets `HIP_VISIBLE_DEVICES`**: masking +the device *and* passing its real ordinal asks `hipSetDevice` for a card the runtime +has hidden, so every card but ordinal 0 fails to measure at all. One selector only. ## The trap carried in the sources @@ -79,14 +85,18 @@ when touching these files. ## Exit status and seams -`0` measured · `2` usage or no sources · `3` refused, GPU busy · `4` no HIP -toolchain or build failed. A client box without the SDK exits `4` loudly rather -than reporting an empty success; the probes belong on the box with the cards. +`0` measured · `2` usage or no sources · `3` refused, GPU busy or holders unknowable · +`4` no HIP toolchain or build failed · `5` a probe ran and failed. A client box without +the SDK exits `4` loudly rather than reporting an empty success; the probes belong on +the box with the cards. A probe that dies (the segfault above, an unfundable +allocation) exits `5` and names which probe on which device — a sweep that quietly +swallowed the failure would report success with half its rows missing. Test seams (all documented in the script header): `PCIE_PROBE_SRC`, `PCIE_PROBE_BUILD`, `PCIE_PROBE_SYSFS`, `PCIE_PROBE_KFD`, `PCIE_PROBE_HIPCC`, -`PCIE_PROBE_MAX_BUSY` — the sysfs/kfd paths are what make a refusal testable on -a machine with no cards, which is the only honest way to test a refusal. +`PCIE_PROBE_FUSER`, `PCIE_PROBE_MAX_BUSY` — the sysfs/kfd paths are what make a +refusal testable on a machine with no cards, which is the only honest way to test a +refusal. Measured campaign numbers are not kept here; they belong with the endpoint they were taken on, in the serverAppConfigs LLM runbook vault. diff --git a/scripts/pcie-probe.sh b/scripts/pcie-probe.sh index dd1b03d..215fc84 100755 --- a/scripts/pcie-probe.sh +++ b/scripts/pcie-probe.sh @@ -15,8 +15,12 @@ # # usage: pcie-probe.sh [--device N]... [--build-only] [--force] # -# --device N measure this device; repeat for each card. Without it the probes run once with the -# environment's own device selection — one shared run cannot attribute bandwidth. +# --device N measure HIP device ordinal N; repeat for each card. Without it the probes run +# once with the environment's own device selection — one shared run cannot +# attribute bandwidth. The ordinal is passed to the probes as argv, which is the +# only selector that survives contact with HIP: HIP_VISIBLE_DEVICES=1 makes +# ordinal 0 the sole visible card, so masking AND passing N asks the probe for a +# device that no longer exists. The mask is deliberately not set here. # --build-only compile and stop; never touches the cards. # --force measure even though the guard says the GPU is busy. # @@ -27,10 +31,11 @@ # PCIE_PROBE_SYSFS drm sysfs root (/sys/class/drm) # PCIE_PROBE_KFD KFD device node (/dev/kfd) # PCIE_PROBE_HIPCC compiler (hipcc) +# PCIE_PROBE_FUSER KFD holder probe (fuser) # PCIE_PROBE_MAX_BUSY refusal threshold, percent (5) # -# Exit status: 0 measured · 2 usage or missing sources · 3 REFUSED, GPU busy · 4 no toolchain or -# build failed. +# Exit status: 0 measured · 2 usage or missing sources · 3 REFUSED, GPU busy or unknowable · +# 4 no toolchain or build failed · 5 a probe ran and failed. set -uo pipefail @@ -39,6 +44,7 @@ BUILD="${PCIE_PROBE_BUILD:-$HOME/.cache/pcie-probe}" SYSFS="${PCIE_PROBE_SYSFS:-/sys/class/drm}" KFD="${PCIE_PROBE_KFD:-/dev/kfd}" HIPCC="${PCIE_PROBE_HIPCC:-hipcc}" +FUSER="${PCIE_PROBE_FUSER:-fuser}" MAX_BUSY="${PCIE_PROBE_MAX_BUSY:-5}" force=0 @@ -63,9 +69,19 @@ fi # ---- the guard, before anything touches the card -------------------------------- if (( ! force )); then busy="" - if [[ -e "$KFD" ]] && command -v fuser > /dev/null 2>&1; then - holders="$(fuser "$KFD" 2>/dev/null | tr -s '[:space:]' ' ')" - [[ -n "${holders// /}" ]] && busy="KFD in use by:${holders}" + if [[ -e "$KFD" ]]; then + if command -v "$FUSER" > /dev/null 2>&1; then + holders="$("$FUSER" "$KFD" 2>/dev/null | tr -s '[:space:]' ' ')" + [[ -n "${holders// /}" ]] && busy="KFD in use by:${holders}" + else + # No fuser (psmisc) means the KFD holders are unknowable — and gpu_busy_percent reads 0 + # for a service that is resident but idle. Guessing "idle" here is how you end up + # benchmarking beside a live endpoint, so the guard fails closed instead. + printf 'pcie-probe: REFUSED — cannot see who holds %s: `%s` not found (psmisc)\n' "$KFD" "$FUSER" >&2 + printf ' install psmisc, or pass --force if you have decided the contention is\n' >&2 + printf ' acceptable and read gpu_busy_percent yourself first.\n' >&2 + exit 3 + fi fi for card in "$SYSFS"/card*; do [[ -f "$card/device/gpu_busy_percent" ]] || continue @@ -111,7 +127,11 @@ for card in "$SYSFS"/card*; do done # ---- measure, per device -------------------------------------------------------- +# One selector only (see --device above), and every probe's exit status is honoured: a probe +# that dies must not let the sweep report success. Failures are collected so the remaining +# cards are still measured — losing the second card's numbers is how a campaign lies. (( ${#devices[@]} > 0 )) || devices=(default) +failures=() for d in "${devices[@]}"; do if [[ $d == default ]]; then printf '\n########## engine device selection ##########\n' @@ -121,6 +141,15 @@ for d in "${devices[@]}"; do for name in hip_smoke pcie_bw pcie_probe; do bin="$BUILD/$name" [[ -x "$bin" ]] || continue - if [[ $d == default ]]; then "$bin"; else HIP_VISIBLE_DEVICES="$d" "$bin" "$d"; fi + if [[ $d == default ]]; then + "$bin" || failures+=("$name (engine selection)") + else + "$bin" "$d" || failures+=("$name (device $d)") + fi done done +if (( ${#failures[@]} > 0 )); then + printf 'pcie-probe: %d probe(s) failed:\n' "${#failures[@]}" >&2 + printf ' %s\n' "${failures[@]}" >&2 + exit 5 +fi diff --git a/tests/pcie-probe.bats b/tests/pcie-probe.bats index 9c74afc..85af796 100755 --- a/tests/pcie-probe.bats +++ b/tests/pcie-probe.bats @@ -6,7 +6,8 @@ # built or run, and the tool stops nothing and starts nothing (auto-stopping a production service is # the thing it must not do unasked). # -# Exit status: 0 measured · 2 usage · 3 refused (GPU busy) · 4 no toolchain / build failed. +# Exit status: 0 measured · 2 usage · 3 refused (GPU busy) · 4 no toolchain / build failed · +# 5 a probe ran and failed. # # hipcc and the compiled probes are stubbed: this box has no HIP SDK (it is a client laptop) and a # real measurement belongs on a machine with the cards, beside no running model. @@ -46,7 +47,16 @@ while [ $# -gt 0 ]; do [ "$1" = "-o" ] && out="$2"; shift; done : > "$PCIE_PROBE_BUILD/.built" { echo '#!/usr/bin/env bash' echo ': > "$PCIE_PROBE_BUILD/.ran"' - echo 'echo "PROBE RAN dev=${HIP_VISIBLE_DEVICES:-none} bin=$(basename "$0")"' + # HIP rule, modelled because it is the trap: with HIP_VISIBLE_DEVICES set, the ONLY + # valid ordinal is 0. A driver that masks the device and still passes its real ordinal + # asks for a card the runtime cannot see — which is why the stub used to report the + # env var instead of the argument, and could not see the collision. + echo 'dev="${1:-0}"' + echo 'if [ -n "${HIP_VISIBLE_DEVICES:-}" ] && [ "$dev" != 0 ]; then' + echo ' printf "setDevice(%s) failed\n" "$dev" >&2; exit 1' + echo 'fi' + echo '[ "${STUB_PROBE_FAILS:-0}" = 1 ] && { echo "stub probe died mid-measurement" >&2; exit 1; }' + echo 'echo "PROBE RAN dev=$dev bin=$(basename "$0")"' echo 'echo " slab 2.344 MiB: 31.4 GB/s"' } > "$out" chmod +x "$out" @@ -131,6 +141,35 @@ teardown() { rm -rf "$TEST_TMP"; } [[ "$output" == *"dev=1"* ]] } +@test "a device ordinal the environment mask would hide is still measured" { + # Reproduction, not preference: HIP_VISIBLE_DEVICES=1 leaves ordinal 0 as the only + # selectable card, so masking AND passing the ordinal asks hipSetDevice(1) for a + # device the runtime cannot see. Per-card bandwidth was unobtainable that way. + run "$SCRIPT" --device 0 --device 1 + [ "$status" -eq 0 ] + [[ "$output" == *"PROBE RAN dev=1"* ]] + [[ "$output" != *"setDevice"* ]] +} + +@test "a probe that dies is reported as a failed run, not a green one" { + # The header promises "0 measured". A probe that segfaults (the documented hazard on + # this ROCm build) must not let the sweep exit 0. + run env STUB_PROBE_FAILS=1 "$SCRIPT" + [ "$status" -eq 5 ] + [[ "$output" == *"failed"* ]] + [[ "$output" == *"hip_smoke"* ]] +} + +@test "the guard refuses when it cannot see who holds the KFD, instead of assuming idle" { + # Without fuser the /dev/kfd half of the guard cannot run, and gpu_busy_percent reads + # 0 for a service that is resident but idle — a silent fallback would benchmark beside + # a live endpoint, which is the one thing this tool's LAW forbids. + run env PCIE_PROBE_FUSER="$TEST_TMP/nope-fuser" "$SCRIPT" + [ "$status" -eq 3 ] + [[ "$output" == *"fuser"* ]] + [ ! -e "$BUILD/.built" ] +} + @test "no probe sources is a usage-level error, not an empty success" { rm -f "$SRC"/*.hip run "$SCRIPT"