Frank
d57174c8d3
Three engine knobs that were impossible to set without editing the run line:
FRANK_PRIMARY_SPEC (--speculative-config), FRANK_PRIMARY_COMPCONFIG
(--compilation-config) and FRANK_PRIMARY_GEN_CONFIG
(--override-generation-config). Each holds the whole JSON value; empty or
unset means the flag is absent, so the shipped shape E argv is unchanged.
The seam re-pins the law that justified the omission: "speculative decode is
ABSENT BY MEASUREMENT" compared the wrong pair — c1248d4 retired MTP by
weighing fp8+MTP4 (37.4-40.6 t/s, 54% acceptance) against the bf16 no-spec
boot (36.4-39.4), bf16 then proved unbootable, and fp8 came back WITHOUT MTP.
fp8+spec was never measured against fp8+no-spec on one shape, the no-spec
incumbent runs ~30 t/s under the fleet's 35 t/s target, and every vendor
flash-next recipe ships MTP3.
Also in the seam: a dead assertion is deleted (a failing `! grep` mid-test is
exempt from bats' errexit, so the pre-precise-memory `! grep
--expert-precise-memory` never ran), and the knob test asserts by RENDERING the
flag from the sourced env instead of grepping for an assignment.
362 lines
20 KiB
Bash
Executable File
362 lines
20 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: frank-primary — the Dankserver LLM endpoint on :7770.
|
|
#
|
|
# This file asserts the tcclaviger RDNA4 vLLM ROOT-UNIT shape (the
|
|
# 2026-10-01 BF16 cutover, the overnight redo campaign's verdict): the serve
|
|
# script drives `podman run` DIRECTLY (the image entrypoint execs
|
|
# `vllm serve`), with every engine flag qualified by boot probe and carried
|
|
# in the repo env file. The seam pins what the flags alone do not say: the
|
|
# KV qualification law (the night ladder: 262144/245760 OOM at graph
|
|
# capture; 241664 boots at 732,359 tokens / 3.03x, util 0.98;
|
|
# the bf16-KV trade (double the attention-KV bytes/token, ~14% decode cost,
|
|
# clean cache + 262k windows), the automatic-offload law (the fork plans
|
|
# the tiered-expert offload itself; no flag exists to set), the self-healing
|
|
# law (the run returns before the engine is up, so the serve script must
|
|
# wait on the container for Restart=on-failure to mean anything), the
|
|
# memlock law that makes the pinned host rows legal, the deployer's
|
|
# image/payload gates with the false-green MainPID check, and the measured
|
|
# laws: expert prefetch ABSENT (36% precision), NO speculative decode (the
|
|
# no-spec family measured fastest once the prefetch tax was removed; MTP
|
|
# re-measures pending), gather geometry 32x64 PRESENT (+18.7% decode),
|
|
# MNBT 1024 (Frank's lock), and the client contract (reasoning parser qwen3,
|
|
# tool parser qwen3_xml, preserve_thinking on).
|
|
|
|
load helpers
|
|
|
|
UNIT="$REPO_ROOT/scripts/frank-primary.service"
|
|
SERVE="$REPO_ROOT/scripts/serve-frank-primary.sh"
|
|
ENV_FILE="$REPO_ROOT/scripts/frank-primary.env"
|
|
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh"
|
|
|
|
@test "frank-primary unit, serve script and env exist in scripts/" {
|
|
[ -f "$UNIT" ]
|
|
[ -x "$SERVE" ]
|
|
[ -f "$ENV_FILE" ]
|
|
}
|
|
|
|
@test "frank-primary is boot-persistent, self-healing and unprivileged" {
|
|
# Root unit parity with frank-large/frank-small: enable is the [Install]
|
|
# section, the restart policy survives a crash, and the tiered-expert
|
|
# pin needs an unbounded memlock (a root unit grants it via PID1; the
|
|
# dank session's own hard limit is 8 MiB and cannot be raised without
|
|
# restarting user@1000.service).
|
|
grep -q '^WantedBy=multi-user.target$' "$UNIT"
|
|
# Restart=always, not on-failure: a runtime worker OOM makes vLLM's API
|
|
# server exit CLEAN, so ExecStart returns 0 and on-failure never fires
|
|
# (measured 2026-10-01 — the endpoint stayed down until a human noticed).
|
|
grep -q '^Restart=always$' "$UNIT"
|
|
! grep -q '^Restart=on-failure$' "$UNIT"
|
|
# Exiting the engine takes as long as vLLM's drain: ExecStop runs
|
|
# `podman stop -t 120`, so the stop timeout MUST exceed 120 s or systemd
|
|
# kills the stop and records 'Failed with result timeout' (measured
|
|
# 2026-10-01). Guard the relationship, not just the literal.
|
|
stop_t=$(awk -F= '/^TimeoutStopSec=/{print $2}' "$UNIT")
|
|
[ -n "$stop_t" ] && [ "$stop_t" -gt 120 ]
|
|
grep -q '^ExecStop=.*podman stop -t 120' "$UNIT"
|
|
# StartLimitIntervalSec belongs to [Unit]; a copy in [Service] is an
|
|
# 'Unknown key' warning systemd logs on every load.
|
|
[ "$(grep -c '^StartLimitIntervalSec=' "$UNIT")" = "1" ]
|
|
awk '/^\[Service\]/{in_svc=1} /^StartLimitIntervalSec=/{if(in_svc) exit 1}' "$UNIT"
|
|
grep -q '^RestartSec=' "$UNIT"
|
|
grep -q '^User=dank$' "$UNIT"
|
|
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
|
|
}
|
|
|
|
@test "frank-primary runs the repo's own serve script with the repo's env" {
|
|
grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-primary.sh$' "$UNIT"
|
|
grep -q 'FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env' "$UNIT"
|
|
}
|
|
|
|
@test "the unit stops the container child explicitly" {
|
|
# The serve script blocks on podman wait, which cannot stop the
|
|
# container child; systemd stop goes through ExecStop or the engine
|
|
# outlives the unit and holds both GPUs. The container name must match
|
|
# the env file's identity.
|
|
grep -q '^ExecStop=/usr/bin/podman stop' "$UNIT"
|
|
grep -q 'frank-primary-qwen' "$UNIT"
|
|
grep -q '^FRANK_PRIMARY_CONTAINER=frank-primary-qwen$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the serve shape is direct podman run, not a launcher or shim" {
|
|
# The tcclaviger stack is fully expressed by flags; the R9V launcher,
|
|
# docker->podman shim and setup-state layering are the RETIRED shape.
|
|
grep -q 'podman run' "$SERVE"
|
|
! grep -q 'launch.sh' "$SERVE"
|
|
! grep -q 'podman-shim' "$SERVE"
|
|
! grep -q 'R9V_STATE_FILE' "$SERVE"
|
|
! grep -q -- '--model-draft' "$SERVE"
|
|
! grep -q -- '--spec-type' "$SERVE"
|
|
}
|
|
|
|
@test "the qualified engine shape is in the env file" {
|
|
# 2026-10-02 shape D. History: the 2026-10-01 night ladder put SEQS=3
|
|
# bf16 at 241664 (KV 732,359 / 3.03x); SEQS=6 fp8 does NOT fit (6 x
|
|
# 241664 asked 10.70 GiB against 9.18 GiB free); SEQS=5 fp8 x 262144
|
|
# BOOTED (1,350,182 / 5.15x) but left ~0 bytes free and a real 995-token
|
|
# request killed the worker with `Tried to allocate 50.00 MiB ... 0 bytes
|
|
# free`. UTIL stays 0.98 (lowering it shrinks the engine's own budget
|
|
# instead of buying slack).
|
|
# SEQS=2 is IMPOSSIBLE on this model, measured 2026-10-02 by
|
|
# single-variable boot probe against the then-deployed shape: SEQS 2 fails
|
|
# at BOTH MNBT 1024 and 2048 with torch.fx ConstraintViolationError
|
|
# (query_start_loc.size()[0] marked dynamic but specialized to a constant
|
|
# (3)); SEQS 3 passes at both. model_runner sizes that buffer
|
|
# max_num_reqs + 1, so the width is SEQS + 1 and MNBT only moves the
|
|
# compile RANGE.
|
|
# So the shape is 3 slots with a pool that admits only ~2 full-length
|
|
# conversations: measured live, KV_BYTES 3.15e9/rank reports 398,658
|
|
# tokens = 1.99x for a 200704-token request (7,892 B/token at THIS shape;
|
|
# the per-token price is shape-dependent, 7,701 at 262144). Shorter
|
|
# requests use the spare blocks; a third LONG request queues.
|
|
# The arena is sized by max_model_len, NOT by KV_BYTES: the planner
|
|
# reserves registry 9190 B/token x max_model_len (2.48 GiB at 262144 ->
|
|
# arena 6,248 slots; 1.91 GiB at 200704 -> 6,496). The pool cut buys
|
|
# runtime headroom, not arena.
|
|
grep -q '^FRANK_PRIMARY_MAXLEN=200704$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_SEQS=3$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_UTIL=0.98$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_KV_BYTES=3150000000$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_MNBT=1024$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_KV_DTYPE=fp8$' "$ENV_FILE"
|
|
# --expert-precise-memory IS set (shape E, 2026-10-02). It was refused
|
|
# while the 2 x 262144 bf16 boot sat 0.13 GiB under the 31.86 GiB ceiling
|
|
# (arena 16.65 device-resident, NRestarts=3 at ~6.7 min each) — there it was
|
|
# the last 2.35 GiB over the line, not the cause. Probed on the shape-D base
|
|
# with production stopped: boots healthy in 420 s, plan reports runtime
|
|
# overhead 5.25 GiB (not 7.60), expert cache 17.22 GiB / 7,522 slots
|
|
# (+1,026 / +16% over 6,496). The pool is NOT cut with it (option A): the
|
|
# pool is not an arena lever, so 3.15e9 keeps the measured 1.99x.
|
|
grep -q '^FRANK_PRIMARY_PRECISE_MEMORY=1$' "$ENV_FILE"
|
|
# GPU_MAX_HW_QUEUES caps HSA hw queues per GPU; real knob (the literal is in
|
|
# libhsa-runtime64.so), passed inside the container as a bare name — the env
|
|
# file keeps only the prefixed form.
|
|
grep -q '^FRANK_PRIMARY_GPU_MAX_HW_QUEUES=2$' "$ENV_FILE"
|
|
# The compile cache is a BOOT-COST fix: the image ships no /root/.cache at
|
|
# all, our boot writes 1.1 GB of vLLM AOT artifacts (4,665 files) plus
|
|
# 229 MB of comgr JIT kernels into the ephemeral layer, and every
|
|
# `podman rm -f` throws them away — re-paying `torch.compile took 181.20 s`
|
|
# (173.19 s of it one graph) on every deploy, restart and reboot.
|
|
grep -q '^FRANK_PRIMARY_CACHE_DIR=/home/dank/.cache/frank-primary$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_GATHER_CHUNKS=32$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_GATHER_LANES=64$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_MODEL_SHARDS=131$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the serve script wires the engine flags from the env" {
|
|
grep -q -- '--max-num-seqs "$FRANK_PRIMARY_SEQS"' "$SERVE"
|
|
grep -q -- '--max-model-len "$FRANK_PRIMARY_MAXLEN"' "$SERVE"
|
|
grep -q -- '--kv-cache-memory-bytes "$FRANK_PRIMARY_KV_BYTES"' "$SERVE"
|
|
grep -q -- '--kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE"' "$SERVE"
|
|
grep -q -- '--max-num-batched-tokens "$FRANK_PRIMARY_MNBT"' "$SERVE"
|
|
grep -q -- '-e R4D_LRU_GATHER_CHUNKS="$FRANK_PRIMARY_GATHER_CHUNKS"' "$SERVE"
|
|
grep -q -- '-e R4D_LRU_GATHER_LANES="$FRANK_PRIMARY_GATHER_LANES"' "$SERVE"
|
|
# Speculative decode, the cudagraph capture ladder and the sampling
|
|
# defaults are all ENV-DRIVEN knobs (unset -> the flag is absent, which is
|
|
# the shipped shape today). JSON values carry no shell-special characters
|
|
# that this quoting cannot carry.
|
|
grep -q -- '${FRANK_PRIMARY_SPEC:+--speculative-config "$FRANK_PRIMARY_SPEC"}' "$SERVE"
|
|
grep -q -- '${FRANK_PRIMARY_COMPCONFIG:+--compilation-config "$FRANK_PRIMARY_COMPCONFIG"}' "$SERVE"
|
|
grep -q -- '${FRANK_PRIMARY_GEN_CONFIG:+--override-generation-config "$FRANK_PRIMARY_GEN_CONFIG"}' "$SERVE"
|
|
grep -q -- '--tensor-parallel-size 2' "$SERVE"
|
|
grep -q -- '--enable-prefix-caching' "$SERVE"
|
|
grep -q -- '--trust-remote-code' "$SERVE"
|
|
# DCP is opt-in: the flag is passed only when FRANK_PRIMARY_DCP is set
|
|
# (DCP reuses the TP ranks when PCP is off, so TP=2 + DCP=2 fits two GPUs).
|
|
grep -q -- '--decode-context-parallel-size "$FRANK_PRIMARY_DCP"' "$SERVE"
|
|
! grep -q '^FRANK_PRIMARY_DCP=' "$ENV_FILE"
|
|
# --expert-precise-memory selects the fitted overhead (5.25 GiB) over the
|
|
# default (7.60 GiB = fitted + compile/graph-capture headroom), which only
|
|
# moves the plan's `room` term and therefore the expert arena: ~1,025
|
|
# more slots at 2.344 MiB each.
|
|
grep -q -- '${FRANK_PRIMARY_PRECISE_MEMORY:+--expert-precise-memory}' "$SERVE"
|
|
# Shape E host/CPU wiring: the cache dir is created before it is mounted
|
|
# (a cold boot must have a target) and the mount lands on /root/.cache,
|
|
# whose vllm/ child is the path VLLM_CACHE_ROOT defaults to — the engine log
|
|
# names it as `Using cache directory: /root/.cache/vllm/torch_compile_cache/…`.
|
|
grep -q -- 'mkdir -p "$CACHE_DIR"' "$SERVE"
|
|
grep -q -- '${CACHE_DIR:+-v "$CACHE_DIR":/root/.cache}' "$SERVE"
|
|
grep -q -- '-e GPU_MAX_HW_QUEUES="$FRANK_PRIMARY_GPU_MAX_HW_QUEUES"' "$SERVE"
|
|
}
|
|
|
|
@test "tools and reasoning are wired — the client contract is served" {
|
|
# 2026-10-01: the rollback-era trio served NO tool dialect and left
|
|
# thinking inline in `content` with a dangling `</think>` and a null
|
|
# `reasoning` field (measured: content 'We need answer...Brief.\n</think>
|
|
# \n\n391') — the malformed thinking block clients choke on. The working
|
|
# values are the ones ecaad36 shipped for this model and this fork's
|
|
# registries confirm: reasoning parser `qwen3`, tool parser `qwen3_xml`.
|
|
# preserve_thinking keeps prior thinking blocks across turns.
|
|
grep -q '^FRANK_PRIMARY_REASONING_PARSER=qwen3$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_TOOL_PARSER=qwen3_xml$' "$ENV_FILE"
|
|
grep -q -- '--reasoning-parser "$FRANK_PRIMARY_REASONING_PARSER"' "$SERVE"
|
|
grep -q -- '--enable-auto-tool-choice' "$SERVE"
|
|
grep -q -- '--tool-call-parser "$FRANK_PRIMARY_TOOL_PARSER"' "$SERVE"
|
|
grep -q -- '--default-chat-template-kwargs "$FRANK_PRIMARY_CHAT_TEMPLATE_KWARGS"' "$SERVE"
|
|
grep -q 'preserve_thinking' "$ENV_FILE"
|
|
grep -q 'enable_thinking' "$ENV_FILE"
|
|
}
|
|
|
|
@test "expert offload is AUTOMATIC — the payload cannot fit VRAM and no flag sets it" {
|
|
# ~86 GiB/rank of FP8 trunk against 32 GiB of VRAM: the fork's tiered-
|
|
# expert planner moves the overflow to pinned host rows by itself (the
|
|
# arena 6,574 slots x 2.344 MiB plus host rows). There is no
|
|
# --enable-expert-offload flag in this arm; the legality of the pin is
|
|
# the unit's unbounded LimitMEMLOCK.
|
|
! grep -q -- '--enable-expert-offload' "$SERVE"
|
|
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
|
|
grep -q -- '--ipc=host --ulimit memlock=-1:-1' "$SERVE"
|
|
}
|
|
|
|
@test "speculative decode is an ENV KNOB — the old retirement verdict was invalid" {
|
|
# The 2026-10-02 c1248d4 retirement of MTP compared the WRONG PAIR:
|
|
# fp8+MTP4 (37.4-40.6 tok/s, 54% acceptance) against the bf16 no-spec boot
|
|
# (36.4-39.4), then bf16 proved unbootable and fp8 came back WITHOUT MTP —
|
|
# so fp8+spec was never compared against fp8+no-spec on the same shape. The
|
|
# no-spec incumbent runs ~30 tok/s, under the fleet's 35 tok/s target, while
|
|
# EVERY vendor flash-next recipe ships MTP3, and our MTP4 run used the
|
|
# default GREEDY draft sampling (probabilistic is the other fork option and
|
|
# is what the recipes use under temp>0). So spec is a knob, not a law: the
|
|
# shipped value is whatever a matched-pair re-measurement wins.
|
|
grep -q 'THE RETIREMENT VERDICT WAS INVALID' "$ENV_FILE"
|
|
# The knobs are empty in the shipped env, so the `${VAR:+…}` wiring renders
|
|
# NO spec/compilation flag — asserted by rendering, not by grepping the
|
|
# assignment (shape E today has neither flag in its argv).
|
|
run bash -c 'set -a; . "$1" >/dev/null; set +a
|
|
printf %s "${FRANK_PRIMARY_SPEC:+--speculative-config $FRANK_PRIMARY_SPEC}${FRANK_PRIMARY_COMPCONFIG:+--compilation-config $FRANK_PRIMARY_COMPCONFIG}"' _ "$ENV_FILE"
|
|
[ -z "$output" ]
|
|
# The capture-ladder law: the piecewise graph ladder must contain
|
|
# num_speculative_tokens + 1, so an MTP3 shape captures [4, 8, 12] and the
|
|
# 2026-10-02 SEQS=2 failure (query_start_loc specialized to the constant 3)
|
|
# is why SEQS stays a multiple-compatible value.
|
|
grep -q 'num_speculative_tokens + 1' "$ENV_FILE"
|
|
}
|
|
|
|
@test "KV dtype is fp8 — the incumbent, and the only one that boots" {
|
|
# fp8 costs ~7,701 B/token at 262144 (7,892 at 200704 — shape-dependent),
|
|
# decomposed as 6,138 attention + 1,563 mamba/side (dtype-independent);
|
|
# bf16 costs 13,839 (12,276 + 1,563).
|
|
# bf16 was abandoned 2026-10-01: its 2 x 262144 boot died with 0 bytes
|
|
# free on a real request, and the SEQS=2 slot count it was designed
|
|
# around cannot compile this model at all (see the shape test above).
|
|
grep -q '^FRANK_PRIMARY_KV_DTYPE=fp8$' "$ENV_FILE"
|
|
grep -q -- '--kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE"' "$SERVE"
|
|
}
|
|
|
|
@test "gather geometry 32x64 is PRESENT BY MEASUREMENT" {
|
|
# R4D_LRU_GATHER_CHUNKS=32 / R4D_LRU_GATHER_LANES=64: the overnight
|
|
# sweep's biggest lever, +18.7% decode vs default geometry, and the win
|
|
# GROWS with depth (39.5@166 -> 54.1@1087 -> 49.3@4064 in the fp8 era).
|
|
# g64x128 is past the peak; g8x16 mild. They are container ENV (the
|
|
# fork's knobs), not engine flags — passed through with -e.
|
|
grep -q '^FRANK_PRIMARY_GATHER_CHUNKS=32$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_GATHER_LANES=64$' "$ENV_FILE"
|
|
grep -q -- '-e R4D_LRU_GATHER_CHUNKS="$FRANK_PRIMARY_GATHER_CHUNKS"' "$SERVE"
|
|
}
|
|
|
|
@test "MNBT is at Frank's 1024 with the arena intact" {
|
|
# MNBT 512 doubles TTFT at 4064 ctx (2.46 s vs 1.36 s); 1024-4096 are
|
|
# interchangeable on decode AND identical on arena (the sweep measured
|
|
# 6,499 slots at every rung <=4096); 12288 costs 2.5% plus arena.
|
|
# 1024 is Frank's lock — it keeps the workspace floor so the expert arena
|
|
# stays max. (2026-10-02: MNBT 2048 compiles the first graph in 26 s vs
|
|
# 172 s at 1024 — a startup-cost lever, not a correctness one.)
|
|
grep -q '^FRANK_PRIMARY_MNBT=1024$' "$ENV_FILE"
|
|
grep -q -- '--max-num-batched-tokens "$FRANK_PRIMARY_MNBT"' "$SERVE"
|
|
}
|
|
|
|
@test "expert prefetch is absent BY LAW, not by omission" {
|
|
# Telemetry (2026-09-29): the prefetcher served 13,369 of 36,720
|
|
# prefetched slabs (36% precision) — roughly half the gather traffic was
|
|
# waste, and the "savings" would have been installs anyway. Deleting the
|
|
# flag moved decode 26.8 -> 40.6 tok/s. Any future flag must come with a
|
|
# fresh A/B, so the serve script must NOT wire it unconditionally.
|
|
! grep -q -- '--expert-prefetch' "$SERVE"
|
|
! grep -q 'expert_prefetch' "$ENV_FILE"
|
|
grep -q 'PREFETCH IS ABSENT BY LAW' "$ENV_FILE"
|
|
}
|
|
|
|
@test "TunableOp stays off" {
|
|
# CLAV_TUNABLEOP_SWEEP silently skips under expert offload unless
|
|
# CLAV_TUNABLEOP_WITH_OFFLOAD=1, and forcing it corrupts numerics
|
|
# (repetition loops, early halts, invalid measurements).
|
|
! grep -q 'TUNABLEOP' "$ENV_FILE"
|
|
! grep -q 'TUNABLEOP' "$SERVE"
|
|
}
|
|
|
|
@test "the serve script gates the digest-pinned image and the payload" {
|
|
# The image is pinned by OCI manifest digest (the floating :latest moved
|
|
# once already). The payload gate checks config.json and the exact shard
|
|
# count from the env — a missing shard means a guaranteed long-timeout
|
|
# boot instead of a fast failure.
|
|
grep -q 'FRANK_PRIMARY_IMAGE' "$SERVE"
|
|
grep -q 'RepoDigests' "$SERVE"
|
|
grep -q 'FRANK_PRIMARY_MODEL_SHARDS' "$SERVE"
|
|
grep -q 'config.json' "$SERVE"
|
|
grep -q '/dev/kfd' "$SERVE"
|
|
grep -q -- '--device /dev/dri' "$SERVE"
|
|
! grep -q 'tcclaviger/vllm:latest' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the serve script clears stale containers and self-heals via podman wait" {
|
|
# A crashed unit can leave the container name registered forever after;
|
|
# the legacy identities (the retired R9V container and the ad hoc
|
|
# experiment container) hold the same GPUs and die here too. Then block
|
|
# on the container so a crash is a nonzero ExecStart.
|
|
grep -q 'FRANK_PRIMARY_LEGACY_CONTAINERS' "$ENV_FILE"
|
|
grep -q 'frank-primary-r9v' "$ENV_FILE"
|
|
grep -q 'qwen-fp8' "$ENV_FILE"
|
|
grep -q 'exec podman wait "$FRANK_PRIMARY_CONTAINER"' "$SERVE"
|
|
}
|
|
|
|
@test "frank-primary holds the single fleet port under its own alias" {
|
|
# The scratch identity (qwen3.8-flash-next on :7773) must never leak
|
|
# into production: LAN publish of :7770, the harness's model id.
|
|
grep -q '^FRANK_PRIMARY_PUBLISH=7770:7770$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_SERVED_NAME=frank-primary$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the model id matches the payload's served alias wiring" {
|
|
# The payload dir and its shard count are the fetch script's output
|
|
# layout; the fetch script must write where the env points.
|
|
grep -q '^FRANK_PRIMARY_MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_MODEL_SHARDS=131$' "$ENV_FILE"
|
|
grep -q 'Qwen3.8-Flash-Next-Uncensored-FP8' "$DOWNLOAD"
|
|
grep -q 'orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8' "$DOWNLOAD"
|
|
}
|
|
|
|
@test "no bare HIP-reserved env var is defined" {
|
|
# HIP reads bare TEMP for its JIT cache path (2026-08-13: TEMP=1.0
|
|
# segfaulted every load), and HOST/GPU_MAX_HEAP_SIZE are the same
|
|
# class. Every key here is FRANK_PRIMARY_-prefixed.
|
|
! grep -qE '^(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE"
|
|
! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE"
|
|
}
|
|
|
|
@test "the deployer gates the image digest and payload, then health-checks" {
|
|
# The deployer refuses when the pinned image is absent from rootless
|
|
# storage or the payload is missing/wrong-sharded — then clears stale
|
|
# containers (including the legacy identities), waits out the port,
|
|
# relinks/enables/restarts the root unit and health-gates it.
|
|
grep -q 'LLM_PRIMARY_CONTAINER=frank-primary-qwen' "$REMOTE_DEPLOY"
|
|
grep -q 'FRANK_PRIMARY_IMAGE' "$REMOTE_DEPLOY"
|
|
grep -q "podman images --format '{{.RepoDigests}}'" "$REMOTE_DEPLOY"
|
|
grep -q 'FRANK_PRIMARY_MODEL_SHARDS' "$REMOTE_DEPLOY"
|
|
grep -q 'config.json' "$REMOTE_DEPLOY"
|
|
grep -q 'qwen-fp8' "$REMOTE_DEPLOY"
|
|
grep -q 'glm-probe' "$REMOTE_DEPLOY"
|
|
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
|
|
grep -q 'relink_unit frank-primary' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl enable frank-primary.service' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl restart frank-primary.service' "$REMOTE_DEPLOY"
|
|
grep -q 'wait_for_health frank-primary http://127.0.0.1:7770/health 1800' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the deployer will not report green on a stray port-holder" {
|
|
# The 2026-09-26 false green: a leftover probe held :7770 and satisfied
|
|
# the health curl while the real unit was stopped. The deploy is only
|
|
# green when the unit's own MainPID is live behind the health answer.
|
|
grep -q 'systemctl show -p MainPID --value frank-primary.service' "$REMOTE_DEPLOY"
|
|
grep -q 'stray process holds the port' "$REMOTE_DEPLOY"
|
|
}
|