Frank
71a7e5cb9a
Per-card measurement was unreachable for every card but ordinal 0: the driver set HIP_VISIBLE_DEVICES=N and still passed N as the device argument, so the probe called hipSetDevice(N) on a runtime where the mask left only ordinal 0. The bats stub reported the env var rather than the argument it received, so the suite asserted the mask was set and never noticed the collision. - one selector: the ordinal is passed as argv, the mask is never set - a probe that fails is now exit 5 and names probe + device; the sweep still measures the remaining cards instead of swallowing the failure - the guard fails closed when `fuser` is missing (gpu_busy_percent reads 0 for a resident-but-idle service), with PCIE_PROBE_FUSER as the test seam - stub probes now model the HIP mask rule; the pre-existing per-device test goes red against the old driver, which is the regression proof Refs #60
156 lines
7.0 KiB
Bash
Executable File
156 lines
7.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# scripts/pcie-probe.sh — is the decode bottleneck the LINK, or the engine's read path?
|
|
#
|
|
# Instruments live in scripts/probes/: hip_smoke.hip answers "is the host ROCm usable from a plain
|
|
# process", pcie_bw.hip measures host→device bandwidth at the engine's real expert-slab size, and
|
|
# pcie_probe.hip sweeps sizes so a stall can be attributed to the link rather than to the engine.
|
|
# Written during the RDNA4 expert-offload campaign, kept because the same cards are in Dankserver.
|
|
#
|
|
# LAW — never benchmark beside a live service. The guard below runs BEFORE anything is built or run,
|
|
# and it stops nothing: putting a production endpoint down is not this tool's decision to make. When
|
|
# you have deliberately decided that contention is acceptable, say so with --force.
|
|
#
|
|
# Trap carried in the sources: hipGetDeviceProperties SEGFAULTS on this ROCm build even though
|
|
# hipGetDeviceCount works fine and reports every device. Nothing here calls it. Keep it that way.
|
|
#
|
|
# usage: pcie-probe.sh [--device N]... [--build-only] [--force]
|
|
#
|
|
# --device N measure HIP device ordinal N; repeat for each card. Without it the probes run
|
|
# once with the environment's own device selection — one shared run cannot
|
|
# attribute bandwidth. The ordinal is passed to the probes as argv, which is the
|
|
# only selector that survives contact with HIP: HIP_VISIBLE_DEVICES=1 makes
|
|
# ordinal 0 the sole visible card, so masking AND passing N asks the probe for a
|
|
# device that no longer exists. The mask is deliberately not set here.
|
|
# --build-only compile and stop; never touches the cards.
|
|
# --force measure even though the guard says the GPU is busy.
|
|
#
|
|
# Environment (defaults in parentheses) — the sysfs and kfd paths are overridable so the guard can
|
|
# be tested on a machine with no cards, which is the only honest way to test a refusal:
|
|
# PCIE_PROBE_SRC probe sources (<repo>/scripts/probes)
|
|
# PCIE_PROBE_BUILD build output ($HOME/.cache/pcie-probe)
|
|
# PCIE_PROBE_SYSFS drm sysfs root (/sys/class/drm)
|
|
# PCIE_PROBE_KFD KFD device node (/dev/kfd)
|
|
# PCIE_PROBE_HIPCC compiler (hipcc)
|
|
# PCIE_PROBE_FUSER KFD holder probe (fuser)
|
|
# PCIE_PROBE_MAX_BUSY refusal threshold, percent (5)
|
|
#
|
|
# Exit status: 0 measured · 2 usage or missing sources · 3 REFUSED, GPU busy or unknowable ·
|
|
# 4 no toolchain or build failed · 5 a probe ran and failed.
|
|
|
|
set -uo pipefail
|
|
|
|
SRC="${PCIE_PROBE_SRC:-$(dirname "$(realpath "$0")")/probes}"
|
|
BUILD="${PCIE_PROBE_BUILD:-$HOME/.cache/pcie-probe}"
|
|
SYSFS="${PCIE_PROBE_SYSFS:-/sys/class/drm}"
|
|
KFD="${PCIE_PROBE_KFD:-/dev/kfd}"
|
|
HIPCC="${PCIE_PROBE_HIPCC:-hipcc}"
|
|
FUSER="${PCIE_PROBE_FUSER:-fuser}"
|
|
MAX_BUSY="${PCIE_PROBE_MAX_BUSY:-5}"
|
|
|
|
force=0
|
|
build_only=0
|
|
devices=()
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--device) [[ $# -ge 2 ]] || { echo "pcie-probe: --device needs an argument" >&2; exit 2; }
|
|
devices+=("$2"); shift 2 ;;
|
|
--build-only) build_only=1; shift ;;
|
|
--force) force=1; shift ;;
|
|
*) echo "usage: pcie-probe.sh [--device N]... [--build-only] [--force]" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
mapfile -t sources < <(find "$SRC" -maxdepth 1 -name '*.hip' 2>/dev/null | sort)
|
|
if (( ${#sources[@]} == 0 )); then
|
|
echo "pcie-probe: no .hip probe sources in $SRC" >&2
|
|
exit 2
|
|
fi
|
|
|
|
# ---- the guard, before anything touches the card --------------------------------
|
|
if (( ! force )); then
|
|
busy=""
|
|
if [[ -e "$KFD" ]]; then
|
|
if command -v "$FUSER" > /dev/null 2>&1; then
|
|
holders="$("$FUSER" "$KFD" 2>/dev/null | tr -s '[:space:]' ' ')"
|
|
[[ -n "${holders// /}" ]] && busy="KFD in use by:${holders}"
|
|
else
|
|
# No fuser (psmisc) means the KFD holders are unknowable — and gpu_busy_percent reads 0
|
|
# for a service that is resident but idle. Guessing "idle" here is how you end up
|
|
# benchmarking beside a live endpoint, so the guard fails closed instead.
|
|
printf 'pcie-probe: REFUSED — cannot see who holds %s: `%s` not found (psmisc)\n' "$KFD" "$FUSER" >&2
|
|
printf ' install psmisc, or pass --force if you have decided the contention is\n' >&2
|
|
printf ' acceptable and read gpu_busy_percent yourself first.\n' >&2
|
|
exit 3
|
|
fi
|
|
fi
|
|
for card in "$SYSFS"/card*; do
|
|
[[ -f "$card/device/gpu_busy_percent" ]] || continue
|
|
pct="$(cat "$card/device/gpu_busy_percent" 2>/dev/null)"
|
|
[[ "$pct" =~ ^[0-9]+$ ]] || continue
|
|
(( pct > MAX_BUSY )) && busy="${busy:+$busy; }$(basename "$card") at ${pct}%"
|
|
done
|
|
if [[ -n "$busy" ]]; then
|
|
printf 'pcie-probe: REFUSED — GPU busy (%s)\n' "$busy" >&2
|
|
printf ' never benchmark beside a live service. Take the service down deliberately,\n' >&2
|
|
printf ' or pass --force if you have decided the contention is acceptable.\n' >&2
|
|
exit 3
|
|
fi
|
|
fi
|
|
|
|
# ---- build ----------------------------------------------------------------------
|
|
if ! command -v "$HIPCC" > /dev/null 2>&1; then
|
|
printf 'pcie-probe: no HIP toolchain (%s) — run this on the box with the cards, not here\n' "$HIPCC" >&2
|
|
exit 4
|
|
fi
|
|
mkdir -p "$BUILD"
|
|
for src in "${sources[@]}"; do
|
|
bin="$BUILD/$(basename "${src%.hip}")"
|
|
if ! "$HIPCC" -O2 -o "$bin" "$src" 2>&1 | tail -3; then
|
|
printf 'pcie-probe: build failed for %s\n' "$(basename "$src")" >&2
|
|
exit 4
|
|
fi
|
|
done
|
|
if (( build_only )); then
|
|
printf 'pcie-probe: built %s probe(s) into %s\n' "${#sources[@]}" "$BUILD"
|
|
exit 0
|
|
fi
|
|
|
|
# ---- topology: the link the numbers must be read against ------------------------
|
|
printf 'pcie-probe host=%s hipcc=%s probes=%s\n' "$(hostname)" "$HIPCC" "${#sources[@]}"
|
|
for card in "$SYSFS"/card*; do
|
|
[[ -f "$card/device/current_link_width" ]] || continue
|
|
printf ' link %-9s current x%s @ %s max x%s @ %s\n' "$(basename "$card")" \
|
|
"$(cat "$card/device/current_link_width")" \
|
|
"$(cat "$card/device/current_link_speed" 2>/dev/null || echo '?')" \
|
|
"$(cat "$card/device/max_link_width" 2>/dev/null || echo '?')" \
|
|
"$(cat "$card/device/max_link_speed" 2>/dev/null || echo '?')"
|
|
done
|
|
|
|
# ---- measure, per device --------------------------------------------------------
|
|
# One selector only (see --device above), and every probe's exit status is honoured: a probe
|
|
# that dies must not let the sweep report success. Failures are collected so the remaining
|
|
# cards are still measured — losing the second card's numbers is how a campaign lies.
|
|
(( ${#devices[@]} > 0 )) || devices=(default)
|
|
failures=()
|
|
for d in "${devices[@]}"; do
|
|
if [[ $d == default ]]; then
|
|
printf '\n########## engine device selection ##########\n'
|
|
else
|
|
printf '\n########## device %s ##########\n' "$d"
|
|
fi
|
|
for name in hip_smoke pcie_bw pcie_probe; do
|
|
bin="$BUILD/$name"
|
|
[[ -x "$bin" ]] || continue
|
|
if [[ $d == default ]]; then
|
|
"$bin" || failures+=("$name (engine selection)")
|
|
else
|
|
"$bin" "$d" || failures+=("$name (device $d)")
|
|
fi
|
|
done
|
|
done
|
|
if (( ${#failures[@]} > 0 )); then
|
|
printf 'pcie-probe: %d probe(s) failed:\n' "${#failures[@]}" >&2
|
|
printf ' %s\n' "${failures[@]}" >&2
|
|
exit 5
|
|
fi
|