retire frank-medium: tests, planning and observability ground on frank-primary
The frank-medium retirement tail the 2026-09-26 lucebox cutover left uncommitted: the radiance quadlet tree, its swap/fetch tooling and its seam test are deleted (archive copies live in dankserver/.AGENTTRASH/ 20260926-frank-medium-radiance/ and scripts/.AGENTTRASH/20260926- frank-medium-radiance/), the planner/allowlist/test-changed mappings drop the service, and frank-large's chat template moves to scripts/ (its old home was the retired frank-medium dir). Finished here: - tests/frank_primary_unit.bats: the lucebox quadlet seam (boot persistence, host-network bind, HIP_VISIBLE_DEVICES=0,1, HIP-reserved env-name guard, checkout-mounted entrypoint/healthcheck, first-start patience, deployer payload/port gates). - test-changed.sh: explicit .AGENTTRASH/ case — retirement commits move files to trash, and the wrapper refused those additions with "no focused-test mapping". - observability: the vllm: namespace left with the radiance runtime. Rules keep only llamacpp:* series (prefix-cache reuse, spec-decode, queueing, endpoint-down); the dashboard drops the vLLM KV/latency panels; prometheus.yml documents that frank-primary (lucebox) has no /metrics and therefore no scrape job. llm-fleet.json was left corrupted mid-edit (missing opening brace) and is rewritten from HEAD. Suite: 339 green (was 7 red).
This commit is contained in:
Frank committed
1 parent
32d264ccda
commit
d7bf9f119a
22 files changed
+243
-2001
No files matched your search
@@ -21,18 +21,22 @@ load helpers
|
||||
[[ "$output" != *"tests/homepage_pages.bats"* ]]
|
||||
}
|
||||
|
||||
@test "changed-test wrapper maps the frank-medium dir to its focused tests" {
|
||||
@test "changed-test wrapper maps the frank-primary dir to its focused tests" {
|
||||
run bash "$REPO_ROOT/scripts/test-changed.sh" --list --paths \
|
||||
dankserver/frank-medium/quadlet/frank-medium.container
|
||||
dankserver/frank-primary/quadlet/frank-primary.container
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"tests/frank_medium_unit.bats"* ]]
|
||||
[[ "$output" == *"tests/frank_primary_unit.bats"* ]]
|
||||
[[ "$output" == *"tests/deployment_scoping.bats"* ]]
|
||||
[[ "$output" == *"tests/docs_completeness.bats"* ]]
|
||||
[[ "$output" != *"tests/homepage_pages.bats"* ]]
|
||||
# The llama trio's old path must no longer select the container seam: the
|
||||
# mapping was removed with the runtime, and a stale case here would keep
|
||||
# firing tests for an archived service.
|
||||
run bash "$REPO_ROOT/scripts/test-changed.sh" --list --paths scripts/frank-medium.env
|
||||
# The retired frank-medium tree must no longer select any LLM seam: the
|
||||
# mappings were removed with the service, and a stale case would keep
|
||||
# firing tests for an archived runtime. Both archive shapes — the radiance
|
||||
# Quadlet era and the llama.cpp layersplit trio before it — map to nothing.
|
||||
run bash "$REPO_ROOT/scripts/test-changed.sh" --list --paths \
|
||||
dankserver/.AGENTTRASH/20260926-frank-medium-radiance/frank-medium.container \
|
||||
scripts/.AGENTTRASH/20260920-llama-layersplit/serve-frank-medium.sh.llamacpp-layersplit
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" != *"tests/frank_medium_unit.bats"* ]]
|
||||
[[ "$output" != *"tests/frank_primary_unit.bats"* ]]
|
||||
}
|
||||
@@ -67,9 +67,10 @@ load helpers
|
||||
! grep -q 'no proxy was stopped or replaced' "$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
}
|
||||
|
||||
@test "remote runner has separate handlers for both inference services" {
|
||||
@test "remote runner has a handler per inference service" {
|
||||
grep -q 'deploy_llm_large' "$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
grep -q 'deploy_llm_medium' "$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
grep -q 'deploy_llm_primary' "$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
grep -q 'deploy_llm_small' "$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
}
|
||||
|
||||
@test "inference restarts are absent from the top-level dispatcher" {
|
||||
|
||||
@@ -46,25 +46,19 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@test "deployment planner maps the frank-medium container dir to frank-medium only" {
|
||||
run python3 "$PLANNER" --target dankserver --paths \
|
||||
dankserver/frank-medium/quadlet/frank-medium.container \
|
||||
dankserver/frank-medium/radiance-entrypoint.sh \
|
||||
dankserver/frank-medium/frank-medium-healthcheck.py
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$output" = "frank-medium" ]
|
||||
|
||||
# The path is not the assertion: both retired shapes are still in the tree
|
||||
# as archive, and a rule matching either would restart a service nobody
|
||||
# runs. Assert the negatives explicitly, plus the llama trio's old paths,
|
||||
# whose mapping was removed with the runtime.
|
||||
run python3 "$PLANNER" --target dankserver --paths \
|
||||
dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-a.container
|
||||
[ "$status" -eq 0 ]
|
||||
[ -z "$output" ]
|
||||
@test "the retired frank-medium tree plans nothing, from any archive" {
|
||||
# frank-medium left the live tree on 2026-09-26 — first the radiance
|
||||
# Quadlet (and its vLLM replica archive before it), then the llama.cpp
|
||||
# layersplit trio and its fetch tooling. Every shape is archive material
|
||||
# now, and a planner rule matching any of them would restart a service
|
||||
# nobody maintains.
|
||||
run python3 "$PLANNER" --target dankserver --paths \
|
||||
dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-a.container \
|
||||
dankserver/.AGENTTRASH/20260926-frank-medium-radiance/frank-medium.container \
|
||||
dankserver/.AGENTTRASH/20260926-frank-medium-radiance/radiance-entrypoint.sh \
|
||||
scripts/.AGENTTRASH/20260920-llama-layersplit/frank-medium.env.llamacpp-layersplit \
|
||||
scripts/frank-medium.env
|
||||
scripts/.AGENTTRASH/20260926-frank-medium-radiance/download-frank-medium-gguf.sh \
|
||||
scripts/.AGENTTRASH/20260926-frank-medium-radiance/swap-frank-medium.sh
|
||||
[ "$status" -eq 0 ]
|
||||
[ -z "$output" ]
|
||||
}
|
||||
@@ -202,7 +196,7 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
# The runner may restart inference only from the service-dispatch path. It
|
||||
# must not contain a host-wide restart loop.
|
||||
grep -q 'deploy_llm_large' "$REMOTE_DEPLOY"
|
||||
grep -q 'deploy_llm_medium' "$REMOTE_DEPLOY"
|
||||
grep -q 'deploy_llm_primary' "$REMOTE_DEPLOY"
|
||||
grep -q 'deploy_service' "$REMOTE_DEPLOY"
|
||||
! grep -qE 'for .*frank-(small|large|npu)-?' "$REMOTE_DEPLOY"
|
||||
}
|
||||
|
||||
@@ -1,219 +0,0 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the frank-medium vllm-radiance container on dankserver.
|
||||
# frank-medium became a CONTAINER again on 2026-09-20, when the board swap put
|
||||
# both R9700s on CPU root ports and a measured three-arm comparison retired the
|
||||
# mainline-llama.cpp layer split (scripts/.AGENTTRASH/20260920-llama-layersplit/).
|
||||
# What this seam pins is what the flags alone do not say: boot persistence for a
|
||||
# GENERATED unit (this host's v1 quadlet generator cannot enable units, so the
|
||||
# [Install] section is the only thing that survives a reboot), that BOTH cards
|
||||
# work on ONE model through tensor parallelism — the thing the pre-2026-09-20
|
||||
# hardware could not do — the KV pin's coupling to the batch shape it was
|
||||
# calibrated at, and that the pool still holds one full-window request.
|
||||
|
||||
load helpers
|
||||
|
||||
UNIT="$REPO_ROOT/dankserver/frank-medium/quadlet/frank-medium.container"
|
||||
ENTRY="$REPO_ROOT/dankserver/frank-medium/radiance-entrypoint.sh"
|
||||
HEALTH="$REPO_ROOT/dankserver/frank-medium/frank-medium-healthcheck.py"
|
||||
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
LLAMA_TRASH="$REPO_ROOT/scripts/.AGENTTRASH/20260920-llama-layersplit"
|
||||
REPLICA_TRASH="$REPO_ROOT/dankserver/.AGENTTRASH/20260916-vllm-radiance"
|
||||
|
||||
# Measured constants, used by the fit arithmetic.
|
||||
CARD_BYTES=34208743424 # one R9700 as rocm-smi reports it (32624 MiB)
|
||||
KV_BYTES_PER_TOKEN=70656 # --kv-cache-memory 18563072000 => "262,720 tokens"
|
||||
env_value() {
|
||||
sed -nE "s/^Environment=$1=(-?[0-9]+)$/\1/p" "$UNIT" | head -n1
|
||||
}
|
||||
# GPU_UTIL is a fraction, not a count; the integer reader would drop it.
|
||||
env_ratio() {
|
||||
sed -nE "s/^Environment=$1=([0-9.]+)$/\1/p" "$UNIT" | head -n1
|
||||
}
|
||||
# SPEC_METHOD is a string (dflash|mtp), not a count; the integer reader drops it.
|
||||
env_str() {
|
||||
sed -nE "s/^Environment=$1=(.+)$/\1/p" "$UNIT" | head -n1
|
||||
}
|
||||
|
||||
@test "frank-medium's unit, entrypoint and healthcheck exist" {
|
||||
[ -f "$UNIT" ]
|
||||
[ -x "$ENTRY" ]
|
||||
[ -f "$HEALTH" ]
|
||||
}
|
||||
|
||||
@test "frank-medium is boot-persistent and self-healing" {
|
||||
# Generated units cannot be enabled on this host, so WantedBy in the file is
|
||||
# the whole boot story (AGENTS.md: Boot Persistence & Self-Healing).
|
||||
grep -q '^WantedBy=default.target$' "$UNIT"
|
||||
grep -q '^Restart=on-failure$' "$UNIT"
|
||||
grep -q '^RestartSec=' "$UNIT"
|
||||
# First start compiles Triton/inductor kernels before the engine loads ~9.5
|
||||
# GiB per card; a shorter timeout kills a healthy boot.
|
||||
grep -q '^TimeoutStartSec=1800$' "$UNIT"
|
||||
grep -q '^HealthCmd=python3 /usr/local/bin/frank-medium-healthcheck.py$' "$UNIT"
|
||||
grep -q '^HealthStartPeriod=1800s$' "$UNIT"
|
||||
# Boots must not depend on the registry.
|
||||
grep -q '^Pull=never$' "$UNIT"
|
||||
}
|
||||
|
||||
@test "both R9700s serve ONE model as one tensor-parallel process" {
|
||||
grep -q '^Environment=TP=2$' "$UNIT"
|
||||
grep -q '^Environment=ROCR_VISIBLE_DEVICES=0,1$' "$UNIT"
|
||||
grep -q '^Environment=HIP_VISIBLE_DEVICES=0,1$' "$UNIT"
|
||||
grep -q -- '--tensor-parallel-size "$TP"' "$ENTRY"
|
||||
# The iGPU (index 2, 2 GiB) must stay out of the visible set: a third rank
|
||||
# would take layers onto system RAM.
|
||||
! grep -q '^Environment=.*VISIBLE_DEVICES=0,1,2' "$UNIT"
|
||||
# TP needs the all-reduce to actually run. R4D is the collective-free path
|
||||
# this host measured; the entrypoint must still select it, and the RCCL
|
||||
# fallback must not be re-disabled by the pre-board-swap workarounds.
|
||||
grep -q 'RADIANCE_USE_R4D_AR' "$ENTRY"
|
||||
grep -q -- '--attention-backend "$ATTN"' "$ENTRY"
|
||||
! grep -q 'NCCL_P2P_DISABLE' "$UNIT"
|
||||
! grep -q 'RCCL_MSCCL_ENABLE' "$UNIT"
|
||||
}
|
||||
|
||||
@test "HIP never reads one of these tunables as its own setting" {
|
||||
# HIP reads bare TEMP, HOST and GPU_MAX_HEAP_SIZE (2026-08-13 incident:
|
||||
# TEMP=1.0 sent the JIT cache to path "1.0" and segfaulted every load).
|
||||
# Serve scripts export env files with `set -a`; a container inherits the
|
||||
# same hazard from Environment=.
|
||||
! grep -qE '^Environment=(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$UNIT"
|
||||
}
|
||||
|
||||
@test "the speculative drafter matches the serving shape" {
|
||||
local spec_method spec maxseqs chunk
|
||||
spec_method="$(env_str SPEC_METHOD)"
|
||||
spec="$(env_value SPEC)"
|
||||
maxseqs="$(env_value MAXSEQS)"
|
||||
chunk="$(env_value CHUNK)"
|
||||
[ -n "$spec_method" ] && [ -n "$spec" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ]
|
||||
# The chunk is the AR size gate's input (CHUNK x hidden x 2 bytes); a chunk
|
||||
# larger than this row's silently changes which all-reduce path runs.
|
||||
[ "$chunk" -le 8192 ]
|
||||
case "$spec_method" in
|
||||
dflash)
|
||||
# DFlash2 drafts at most 7 tokens per step; deeper is a different
|
||||
# drafter's contract and costs acceptance, not just VRAM.
|
||||
[ "$spec" -eq 7 ]
|
||||
# The MXFP4 / ParoQuant-MXFP6 decode kernel band covers MAXSEQS x
|
||||
# (SPEC+1) rows, and the shipped default covers 64 (the mxfp6 profile
|
||||
# block exports the same RADIANCE_MXFP4_DECODE_MAX_M=64). Exceeding it
|
||||
# without the entrypoint's >8 bump is a launch failure at the worst
|
||||
# moment (first concurrent request).
|
||||
[ $(( maxseqs * (spec + 1) )) -le 64 ]
|
||||
grep -q 'RADIANCE_MXFP4_DECODE_MAX_M' "$ENTRY"
|
||||
;;
|
||||
mtp)
|
||||
# MTP self-speculation at depth 8 (the documented plain-FP8 path; the
|
||||
# model's own single-layer MTP head, applied autoregressively).
|
||||
[ "$spec" -eq 8 ]
|
||||
;;
|
||||
*)
|
||||
false "unknown SPEC_METHOD: $spec_method"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
@test "the KV pool holds a full window and fits beside the weights" {
|
||||
local kv_mem maxseqs chunk maxlen spec kv_dtype
|
||||
kv_mem="$(env_value KV_MEM)"
|
||||
maxseqs="$(env_value MAXSEQS)"
|
||||
chunk="$(env_value CHUNK)"
|
||||
maxlen="$(env_value MAXLEN)"
|
||||
spec="$(env_value SPEC)"
|
||||
kv_dtype="$(env_str KV_DTYPE)"
|
||||
[ -n "$kv_mem" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ] && [ -n "$maxlen" ]
|
||||
# KV_MEM=0 = profile the shape (the documented plain-FP8 path's default;
|
||||
# the upstream docker-compose leaves the pin commented out). The pool is
|
||||
# auto-sized to fit, so there is no pin to verify — only that the shape
|
||||
# is set and the model's weights fit beside an auto-sized pool.
|
||||
if [ "$kv_mem" -eq 0 ]; then
|
||||
# MXFP4 + unquantized (KV_DTYPE=auto -> bf16) KV: MXFP4's freed weights
|
||||
# (19.37 vs 30.9 GB) buy the pool back, so the full 262,144 native window
|
||||
# still fits clean (~290k f16 pool); vLLM boots or the pool is too small.
|
||||
[ "$maxseqs" -eq 8 ] && [ "$chunk" -eq 8192 ] && [ "$maxlen" -eq 262144 ]
|
||||
[ "$kv_dtype" = "auto" ]
|
||||
return 0
|
||||
fi
|
||||
# Pinned pool: the pin must hold at least ONE full-window request or the
|
||||
# endpoint refuses its own --max-model-len, and it must fit on one card
|
||||
# beside the weights.
|
||||
grep -q 'kv-profiles.tsv' "$UNIT"
|
||||
[ $(( kv_mem / KV_BYTES_PER_TOKEN )) -ge "$maxlen" ]
|
||||
# Weights per card: MXFP4 ~9.5 GiB (19 GB / 2), FP8 ~15.5 GiB (30.9 GB / 2).
|
||||
local weights util util_pct floor
|
||||
case "$(env_str SPEC_METHOD)" in
|
||||
dflash) weights=10200547328 ;;
|
||||
*) weights=16646128640 ;;
|
||||
esac
|
||||
util="$(env_ratio GPU_UTIL)"
|
||||
[ -n "$util" ]
|
||||
util_pct="${util#0.}"
|
||||
floor=$(( 1024 * 1024 * 1024 )) # 1 GiB per card
|
||||
[ $(( weights + kv_mem / 2 )) -le $(( CARD_BYTES * util_pct / 100 - floor )) ]
|
||||
}
|
||||
|
||||
@test "the fleet endpoint contract survives the runtime swap" {
|
||||
grep -q '^Environment=PORT=7770$' "$UNIT"
|
||||
grep -q '^Environment=SERVED_NAMES=frank-medium$' "$UNIT"
|
||||
grep -q -- '--port "$PORT"' "$ENTRY"
|
||||
grep -q -- '--served-model-name $SERVED_NAMES' "$ENTRY"
|
||||
# Host network: vLLM binds the port itself, so no PublishPort (and no
|
||||
# pasta bind, which is what made the retired proxy's port replacement
|
||||
# fail at container create).
|
||||
grep -q -- '--network=host' "$UNIT"
|
||||
! grep -q '^PublishPort=' "$UNIT"
|
||||
# Tools and reasoning are part of the harness contract; both parsers must
|
||||
# be selected, and thinking must land in its own field.
|
||||
grep -q -- '--enable-auto-tool-choice' "$ENTRY"
|
||||
grep -q -- '--tool-call-parser qwen3_coder' "$ENTRY"
|
||||
grep -q -- '--reasoning-parser qwen3' "$ENTRY"
|
||||
grep -q -- '--enable-prefix-caching' "$ENTRY"
|
||||
# Requests that send no sampler fields get the predecessor's defaults.
|
||||
grep -q '"temperature":0.7' "$ENTRY"
|
||||
grep -q '"top_p":0.95' "$ENTRY"
|
||||
}
|
||||
|
||||
@test "the healthcheck proves the engine serves, not merely that a process lives" {
|
||||
grep -q '"/health"' "$HEALTH" || grep -q '/health' "$HEALTH"
|
||||
grep -q 'status == 200' "$HEALTH"
|
||||
# PORT-aware, and it must not hard-code a replica port.
|
||||
grep -q 'os.environ.get("PORT"' "$HEALTH"
|
||||
! grep -qE '7771|7772' "$HEALTH"
|
||||
}
|
||||
|
||||
@test "the deploy handler gates on host assets and installs the quadlet" {
|
||||
grep -q 'deploy_llm_medium' "$REMOTE_DEPLOY"
|
||||
grep -q 'install_quadlet frank-medium dankserver/frank-medium/quadlet frank-medium' "$REMOTE_DEPLOY"
|
||||
grep -q 'wait_container_healthy frank-medium' "$REMOTE_DEPLOY"
|
||||
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
|
||||
grep -q '\[\[ "$missing" -eq 0 \]\] || return 1' "$REMOTE_DEPLOY"
|
||||
# The image is only pulled when absent, and re-tagged to the name the unit
|
||||
# actually runs (the quadlet path cannot resolve registry refs at create).
|
||||
grep -q 'podman image exists "$LLM_MEDIUM_IMAGE_LOCAL"' "$REMOTE_DEPLOY"
|
||||
grep -q 'podman tag "$LLM_MEDIUM_IMAGE" "$LLM_MEDIUM_IMAGE_LOCAL"' "$REMOTE_DEPLOY"
|
||||
# The patched libr4d is not optional (the image's own build NaNs this model).
|
||||
grep -q 'patched libr4d absent' "$REMOTE_DEPLOY"
|
||||
}
|
||||
|
||||
@test "a generated unit is never asked to enable, and the llama unit is gone" {
|
||||
# podman 5.x v1 generator: generated units cannot be enabled; asking makes
|
||||
# the deploy fail on a unit that is already boot-persistent via [Install].
|
||||
! grep -q 'systemctl --user enable frank-medium' "$REMOTE_DEPLOY"
|
||||
# ...and the root llama unit must be retired, not relinked: relinking would
|
||||
# resurrect a service whose repo file no longer exists.
|
||||
! grep -q 'relink_unit frank-medium' "$REMOTE_DEPLOY"
|
||||
grep -q 'retire_llama_medium_unit' "$REMOTE_DEPLOY"
|
||||
}
|
||||
|
||||
@test "the llama layer split and the replica trio are archived, not deleted" {
|
||||
[ -f "$LLAMA_TRASH/frank-medium.service.llamacpp-layersplit" ]
|
||||
[ -f "$LLAMA_TRASH/frank-medium.env.llamacpp-layersplit" ]
|
||||
[ -f "$LLAMA_TRASH/serve-frank-medium.sh.llamacpp-layersplit" ]
|
||||
[ ! -f "$REPO_ROOT/scripts/frank-medium.env" ]
|
||||
[ -f "$REPLICA_TRASH/frank-medium-a.container" ]
|
||||
[ -f "$REPLICA_TRASH/frank-medium-proxy.container" ]
|
||||
# The replicas stay retired: the handler stops them but never installs them.
|
||||
grep -q 'systemctl --user stop frank-medium-a.service frank-medium-b.service' "$REMOTE_DEPLOY"
|
||||
! grep -q 'install_quadlet frank-medium-a' "$REMOTE_DEPLOY"
|
||||
}
|
||||
Executable
+94
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: frank-primary — the Dankserver LLM endpoint on :7770.
|
||||
#
|
||||
# This file asserts the LUCEBOX shape (the 2026-09-26 cutover that replaced
|
||||
# frank-medium): a rootless Quadlet running the lucebox DS4.1 engine on the
|
||||
# two R9700s, with the model mmapped out of /home/dank/models and the
|
||||
# entrypoint/healthcheck mounted straight out of this checkout.
|
||||
|
||||
load helpers
|
||||
|
||||
UNIT="$REPO_ROOT/dankserver/frank-primary/quadlet/frank-primary.container"
|
||||
EP="$REPO_ROOT/dankserver/frank-primary/frank-primary-entrypoint.sh"
|
||||
HC="$REPO_ROOT/dankserver/frank-primary/frank-primary-healthcheck.py"
|
||||
|
||||
@test "the frank-primary quadlet file and its mounted scripts exist" {
|
||||
[ -f "$UNIT" ]
|
||||
[ -x "$EP" ]
|
||||
[ -f "$HC" ]
|
||||
# The healthcheck is python; a syntax error would leave the container
|
||||
# permanently unhealthy with the failure hidden inside HealthCmd.
|
||||
python3 -m py_compile "$HC"
|
||||
}
|
||||
|
||||
@test "frank-primary is boot-persistent and self-healing" {
|
||||
# Rootless Podman does not auto-start bare containers and restart:
|
||||
# unless-stopped does not survive shutdown's explicit stop. The unit must
|
||||
# carry both the boot symlink and a restart policy (2026-08-26 incident:
|
||||
# a compose-only Caddy never came back after a reboot).
|
||||
grep -q '^WantedBy=default.target' "$UNIT"
|
||||
grep -qE '^Restart=(always|on-failure)' "$UNIT"
|
||||
}
|
||||
|
||||
@test "frank-primary binds the host network and needs no published port" {
|
||||
# luce_server binds 0.0.0.0:7770 directly, so the container takes the
|
||||
# host netns. A PublishPort would be dead config.
|
||||
grep -q -- '--network=host' "$UNIT"
|
||||
! grep -q '^PublishPort=' "$UNIT"
|
||||
}
|
||||
|
||||
@test "frank-primary reaches both R9700s and hides the Raphael iGPU" {
|
||||
# /dev/kfd + /dev/dri via AddDevice; HIP_VISIBLE_DEVICES=0,1 keeps the
|
||||
# host's Raphael iGPU (index 2) out of the enumeration so the engine's
|
||||
# device picks cannot land on it.
|
||||
grep -q '^AddDevice=/dev/kfd' "$UNIT"
|
||||
grep -q '^AddDevice=/dev/dri' "$UNIT"
|
||||
grep -q '^Environment=HIP_VISIBLE_DEVICES=0,1$' "$UNIT"
|
||||
}
|
||||
|
||||
@test "frank-primary never names a bare HIP-reserved env var" {
|
||||
# The HIP runtime reads bare TEMP for its JIT cache path (the 2026-08-13
|
||||
# incident: TEMP=1.0 sent the JIT cache to path "1.0" and segfaulted every
|
||||
# load). HOST and GPU_MAX_HEAP_SIZE are the same class. The unit's
|
||||
# Environment= lines must not define them; the entrypoint exports
|
||||
# GPU_MAX_HW_QUEUES (explicitly not GPU_MAX_HEAP_SIZE).
|
||||
! grep -qE '^Environment=(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$UNIT"
|
||||
! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$EP"
|
||||
}
|
||||
|
||||
@test "the unit mounts the entrypoint and healthcheck from the checkout" {
|
||||
# deploy.sh git-pulls this checkout before dispatching, so a Volume
|
||||
# reference is how an edit here reaches the container without an image
|
||||
# rebuild. Absolute host paths, read-only.
|
||||
grep -q 'Volume=/home/dank/serverAppConfigs/dankserver/frank-primary/frank-primary-entrypoint.sh:/usr/local/bin/frank-primary-entrypoint.sh:ro' "$UNIT"
|
||||
grep -q 'Volume=/home/dank/serverAppConfigs/dankserver/frank-primary/frank-primary-healthcheck.py:/usr/local/bin/frank-primary-healthcheck.py:ro' "$UNIT"
|
||||
}
|
||||
|
||||
@test "first-start patience is configured on both sides of the health check" {
|
||||
# Cold start page-ins the dense weights and builds the graphs; the
|
||||
# container must not be killed as unhealthy while that happens, and the
|
||||
# deployer must wait the same order of magnitude.
|
||||
grep -q '^HealthStartPeriod=1800s' "$UNIT"
|
||||
grep -q '^TimeoutStartSec=1800$' "$UNIT"
|
||||
grep -q 'wait_container_healthy frank-primary 1800' \
|
||||
"$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
}
|
||||
|
||||
@test "the deployer refuses to start without image and complete payload" {
|
||||
# A 377 GiB partial download must never be served: the handler gates on
|
||||
# the image being present and the model file existing at its exact
|
||||
# declared byte count (stat -c %s, never wc -c — wc read the whole stream
|
||||
# for minutes on 2026-09-26).
|
||||
grep -q 'deploy_llm_primary() {' "$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
grep -q 'podman image exists localhost/lucebox-rocm:gfx1201' \
|
||||
"$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
grep -q 'want_size=405349859328' "$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
grep -q 'stat -c %s' "$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
}
|
||||
|
||||
@test "the deployer gates on the port actually being free" {
|
||||
# 7770 is a host-network bind: the previous holder must release it before
|
||||
# the new container can be created, or pasta's bind fails at create time
|
||||
# and the failure hides behind Restart=.
|
||||
grep -q 'wait_port_free 7770' "$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
}
|
||||
@@ -315,19 +315,24 @@ PAIRS
|
||||
|
||||
@test "prometheus scrapes every LLM endpoint that exports metrics" {
|
||||
# The LLM endpoints are the services whose behaviour this fleet argues about
|
||||
# most (KV pressure, prefix-cache reuse, queue depth). Since frank-medium
|
||||
# moved to radiance/vLLM on 2026-09-20 there are two metric namespaces, and
|
||||
# the three addresses differ by HOW each endpoint is reachable from a
|
||||
# container: netavark DNATs *published* container ports only, so
|
||||
# frank-medium — a rootless Quadlet on another host that publishes 7770 — is
|
||||
# its LAN address, while the two Dankclaw endpoints are HOST processes and
|
||||
# answer only as host.containers.internal. Addressing those two by LAN IP
|
||||
# left both targets red by construction until the 2026-09-20 /targets audit.
|
||||
# most (KV pressure, prefix-cache reuse, queue depth). Only the two Dankclaw
|
||||
# llama.cpp endpoints export metrics today: frank-primary (Dankserver, the
|
||||
# Lucebox DS4.1 engine since the 2026-09-26 cutover) answers 404 on
|
||||
# /metrics, so a job for it would be a permanently-down target — no job,
|
||||
# and its LAN address must not sneak back in with one.
|
||||
# The two scraped endpoints are HOST processes reached through
|
||||
# host.containers.internal: this Prometheus is a bridge-networked
|
||||
# container, and on this netavark host a container can only reach the
|
||||
# host's LAN IP at a PUBLISHED port — a host-process bind like
|
||||
# llama-server's 0.0.0.0:7770 is refused at 192.168.0.143:7770 while it
|
||||
# answers at 169.254.1.2:7770 (why frank-small stayed red by construction
|
||||
# for days until the 2026-09-20 /targets audit).
|
||||
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
||||
grep -q '"192.168.0.235:7770"' "$cfg" || { echo "frank-medium not scraped"; return 1; }
|
||||
! grep -q '"192.168.0.235:7770"' "$cfg" || { echo "frank-primary is scraped but exports no metrics"; return 1; }
|
||||
|
||||
grep -q '"host.containers.internal:7771"' "$cfg" || { echo "frank-small not scraped"; return 1; }
|
||||
grep -q '"host.containers.internal:7770"' "$cfg" || { echo "frank-large not scraped"; return 1; }
|
||||
for job in frank-medium frank-small frank-large; do
|
||||
for job in frank-small frank-large; do
|
||||
grep -q "job_name: \"$job\"" "$cfg" || { echo "no scrape job for $job"; return 1; }
|
||||
done
|
||||
# None of them is reached through a guarded vhost: Prometheus cannot solve
|
||||
@@ -369,45 +374,30 @@ import re, yaml
|
||||
d = yaml.safe_load(open('$rules'))
|
||||
names = [r.get('alert') or r.get('record') for g in d['groups'] for r in g['rules']]
|
||||
for want in ('job:llm_prefix_cache_reuse:rate5m',
|
||||
'job:llm_vllm_prefix_cache_reuse:rate5m',
|
||||
'job:llm_spec_accept_length:rate5m',
|
||||
'LlmRequestsQueueing', 'LlmRequestsWaiting',
|
||||
'LlmKvCacheSaturated', 'LlmPrefixCacheCollapse',
|
||||
'LlmRequestsQueueing', 'LlmPrefixCacheCollapse',
|
||||
'LlmSpecDecodeOff', 'LlmEndpointDown'):
|
||||
assert want in names, want
|
||||
# Every rule must be keyed to a series that actually exists. A rule naming a
|
||||
# series nobody exports evaluates nothing forever and errors never — the failure
|
||||
# this seam exists for. Since 2026-09-20 two engines are scraped, so both
|
||||
# namespaces are legal, but each name must be on the measured list.
|
||||
# series nobody exports evaluates nothing forever and errors never — the
|
||||
# failure this seam exists for. Since the 2026-09-26 cutover the only scraped
|
||||
# LLM runtime is llama.cpp (frank-small, frank-large); the vllm: family left
|
||||
# with frank-medium's radiance runtime and must not survive in any expr.
|
||||
exprs = [r.get('expr', '') for g in d['groups'] for r in g['rules']]
|
||||
# llama.cpp families measured from /metrics on 2026-09-16 (frank-small,
|
||||
# frank-large), vLLM families measured from /metrics on 2026-09-20
|
||||
# (frank-medium, radiance/vLLM). Anything else is a guess.
|
||||
# llama.cpp families measured from /metrics on 2026-09-16 and 2026-09-20
|
||||
# (frank-small, frank-large). Anything else is a guess.
|
||||
live = {'llamacpp:prompt_tokens_total', 'llamacpp:prompt_tokens_cached_total',
|
||||
'llamacpp:requests_deferred', 'llamacpp:requests_processing',
|
||||
'llamacpp:spec_decode_num_drafts_total',
|
||||
'llamacpp:spec_decode_num_accepted_tokens_total',
|
||||
'llamacpp:spec_decode_num_draft_tokens_total',
|
||||
'llamacpp:tokens_predicted_total',
|
||||
'vllm:kv_cache_usage_perc', 'vllm:num_requests_running',
|
||||
'vllm:num_requests_waiting', 'vllm:num_preemptions_total',
|
||||
'vllm:generation_tokens_total', 'vllm:prompt_tokens_total',
|
||||
'vllm:spec_decode_num_accepted_tokens_total',
|
||||
'vllm:spec_decode_num_draft_tokens_total',
|
||||
'vllm:spec_decode_num_drafts_total',
|
||||
'vllm:prefix_cache_hits_total', 'vllm:prefix_cache_queries_total',
|
||||
'vllm:request_success_total'}
|
||||
'llamacpp:predicted_tokens_seconds', 'llamacpp:prompt_tokens_seconds',
|
||||
'llamacpp:n_decode_total', 'llamacpp:n_busy_slots_per_decode'}
|
||||
used = {m for e in exprs
|
||||
for m in re.findall(r'(?:llamacpp|vllm):[a-z_0-9]+', e)}
|
||||
assert used <= live, used - live
|
||||
# Engine-independent rules must cover BOTH families, or the cutover silently
|
||||
# blinds one endpoint: the two rules that describe an engine-agnostic failure
|
||||
# name a series from each namespace.
|
||||
joined = '\n'.join(exprs)
|
||||
for pair in (('LlmSpecDecodeOff', 'llamacpp:spec_decode_num_drafts_total',
|
||||
'vllm:spec_decode_num_drafts_total'),
|
||||
('LlmPrefixCacheCollapse', 'llamacpp:prompt_tokens_total',
|
||||
'vllm:prefix_cache_queries_total')):
|
||||
assert any(pair[1] in e and pair[2] in e for e in exprs), pair[0]
|
||||
assert not any(':vllm' in e for e in exprs), 'vllm: series in rules but no vllm endpoint is scraped'
|
||||
print('rules ok:', len(names))
|
||||
"
|
||||
# A rule file that nothing loads evaluates nothing, and Prometheus reports
|
||||
|
||||
Reference in new issue
Block a user