fix(pcie-probe): select a card by ordinal only, honour probe failures, refuse blind
Per-card measurement was unreachable for every card but ordinal 0: the driver set HIP_VISIBLE_DEVICES=N and still passed N as the device argument, so the probe called hipSetDevice(N) on a runtime where the mask left only ordinal 0. The bats stub reported the env var rather than the argument it received, so the suite asserted the mask was set and never noticed the collision. - one selector: the ordinal is passed as argv, the mask is never set - a probe that fails is now exit 5 and names probe + device; the sweep still measures the remaining cards instead of swallowing the failure - the guard fails closed when `fuser` is missing (gpu_busy_percent reads 0 for a resident-but-idle service), with PCIE_PROBE_FUSER as the test seam - stub probes now model the HIP mask rule; the pre-existing per-device test goes red against the old driver, which is the regression proof Refs #60
This commit is contained in:
Frank committed
1 parent
97b65067b1
commit
71a7e5cb9a
3 files changed
+97
-19
No files matched your search
+37
-8
@@ -15,8 +15,12 @@
|
||||
#
|
||||
# usage: pcie-probe.sh [--device N]... [--build-only] [--force]
|
||||
#
|
||||
# --device N measure this device; repeat for each card. Without it the probes run once with the
|
||||
# environment's own device selection — one shared run cannot attribute bandwidth.
|
||||
# --device N measure HIP device ordinal N; repeat for each card. Without it the probes run
|
||||
# once with the environment's own device selection — one shared run cannot
|
||||
# attribute bandwidth. The ordinal is passed to the probes as argv, which is the
|
||||
# only selector that survives contact with HIP: HIP_VISIBLE_DEVICES=1 makes
|
||||
# ordinal 0 the sole visible card, so masking AND passing N asks the probe for a
|
||||
# device that no longer exists. The mask is deliberately not set here.
|
||||
# --build-only compile and stop; never touches the cards.
|
||||
# --force measure even though the guard says the GPU is busy.
|
||||
#
|
||||
@@ -27,10 +31,11 @@
|
||||
# PCIE_PROBE_SYSFS drm sysfs root (/sys/class/drm)
|
||||
# PCIE_PROBE_KFD KFD device node (/dev/kfd)
|
||||
# PCIE_PROBE_HIPCC compiler (hipcc)
|
||||
# PCIE_PROBE_FUSER KFD holder probe (fuser)
|
||||
# PCIE_PROBE_MAX_BUSY refusal threshold, percent (5)
|
||||
#
|
||||
# Exit status: 0 measured · 2 usage or missing sources · 3 REFUSED, GPU busy · 4 no toolchain or
|
||||
# build failed.
|
||||
# Exit status: 0 measured · 2 usage or missing sources · 3 REFUSED, GPU busy or unknowable ·
|
||||
# 4 no toolchain or build failed · 5 a probe ran and failed.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
@@ -39,6 +44,7 @@ BUILD="${PCIE_PROBE_BUILD:-$HOME/.cache/pcie-probe}"
|
||||
SYSFS="${PCIE_PROBE_SYSFS:-/sys/class/drm}"
|
||||
KFD="${PCIE_PROBE_KFD:-/dev/kfd}"
|
||||
HIPCC="${PCIE_PROBE_HIPCC:-hipcc}"
|
||||
FUSER="${PCIE_PROBE_FUSER:-fuser}"
|
||||
MAX_BUSY="${PCIE_PROBE_MAX_BUSY:-5}"
|
||||
|
||||
force=0
|
||||
@@ -63,9 +69,19 @@ fi
|
||||
# ---- the guard, before anything touches the card --------------------------------
|
||||
if (( ! force )); then
|
||||
busy=""
|
||||
if [[ -e "$KFD" ]] && command -v fuser > /dev/null 2>&1; then
|
||||
holders="$(fuser "$KFD" 2>/dev/null | tr -s '[:space:]' ' ')"
|
||||
[[ -n "${holders// /}" ]] && busy="KFD in use by:${holders}"
|
||||
if [[ -e "$KFD" ]]; then
|
||||
if command -v "$FUSER" > /dev/null 2>&1; then
|
||||
holders="$("$FUSER" "$KFD" 2>/dev/null | tr -s '[:space:]' ' ')"
|
||||
[[ -n "${holders// /}" ]] && busy="KFD in use by:${holders}"
|
||||
else
|
||||
# No fuser (psmisc) means the KFD holders are unknowable — and gpu_busy_percent reads 0
|
||||
# for a service that is resident but idle. Guessing "idle" here is how you end up
|
||||
# benchmarking beside a live endpoint, so the guard fails closed instead.
|
||||
printf 'pcie-probe: REFUSED — cannot see who holds %s: `%s` not found (psmisc)\n' "$KFD" "$FUSER" >&2
|
||||
printf ' install psmisc, or pass --force if you have decided the contention is\n' >&2
|
||||
printf ' acceptable and read gpu_busy_percent yourself first.\n' >&2
|
||||
exit 3
|
||||
fi
|
||||
fi
|
||||
for card in "$SYSFS"/card*; do
|
||||
[[ -f "$card/device/gpu_busy_percent" ]] || continue
|
||||
@@ -111,7 +127,11 @@ for card in "$SYSFS"/card*; do
|
||||
done
|
||||
|
||||
# ---- measure, per device --------------------------------------------------------
|
||||
# One selector only (see --device above), and every probe's exit status is honoured: a probe
|
||||
# that dies must not let the sweep report success. Failures are collected so the remaining
|
||||
# cards are still measured — losing the second card's numbers is how a campaign lies.
|
||||
(( ${#devices[@]} > 0 )) || devices=(default)
|
||||
failures=()
|
||||
for d in "${devices[@]}"; do
|
||||
if [[ $d == default ]]; then
|
||||
printf '\n########## engine device selection ##########\n'
|
||||
@@ -121,6 +141,15 @@ for d in "${devices[@]}"; do
|
||||
for name in hip_smoke pcie_bw pcie_probe; do
|
||||
bin="$BUILD/$name"
|
||||
[[ -x "$bin" ]] || continue
|
||||
if [[ $d == default ]]; then "$bin"; else HIP_VISIBLE_DEVICES="$d" "$bin" "$d"; fi
|
||||
if [[ $d == default ]]; then
|
||||
"$bin" || failures+=("$name (engine selection)")
|
||||
else
|
||||
"$bin" "$d" || failures+=("$name (device $d)")
|
||||
fi
|
||||
done
|
||||
done
|
||||
if (( ${#failures[@]} > 0 )); then
|
||||
printf 'pcie-probe: %d probe(s) failed:\n' "${#failures[@]}" >&2
|
||||
printf ' %s\n' "${failures[@]}" >&2
|
||||
exit 5
|
||||
fi
|
||||
Reference in new issue
Block a user