Frank
0258ae0897
Engine v1.2.0 (Codeberg e95d397, Docker Hub tag 1.2.0, index digest 665775ed...) removes --expert-vs-cache-ratio. The explicit --vram-weights-mib 26071 / --vram-kv-mib 3543 pair transcribes the live 1.0.13 boot ledger at ratio 0.82 (weights 25.46 GiB, KV 3.46 GiB per card); bracket arms scale KV linearly in (1-r): 0.90 -> 1968, 0.95 -> 984. Serve script, autoresearch harness, and bats gates follow; the bats shape asserts are realigned to the deployed seqs=3/spec=2 sweep result (95923d3,df6b5bb) they failed to track.
207 lines
10 KiB
Bash
Executable File
207 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# serve-frank-primary.sh — frank-primary endpoint: radiance (StillDeadcode,
|
|
# Apache-2.0) serving the OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 checkpoint at
|
|
# NATIVE block-FP8 precision from a rad-convert container
|
|
# (/home/dank/models/rad/qwen4exp-fp8-native.rad), on BOTH R9700s (TP=2,
|
|
# wht6 wire), dankserver :7770. 2026-10-05 cutover from FreeToken (PR #587,
|
|
# one card; git tag frank-primary-freetoken-pr587-20261005 is the
|
|
# pre-cutover state).
|
|
#
|
|
# The run line below IS the engine argv: the image's entrypoint is the
|
|
# engine and its tools sit beside it on PATH. Unlike every predecessor
|
|
# there is no per-boot VRAM planner to second-guess: the boot ledger in the
|
|
# log states every budget before anything is allocated (weights vs KV
|
|
# split, the expert host pool's coverage of the worst case, the PLE pin,
|
|
# the P2P answer per pair) — read it after every resize. First-boot shape:
|
|
# 2 decode slots (Frank's call), 40 GiB/rank pinned expert pool, fp8 KV,
|
|
# MTP depth 3, prefix cache host 4 GiB + disk 64 GiB, PLE pinned in RAM.
|
|
#
|
|
# The root unit (scripts/frank-primary.service, User=dank,
|
|
# LimitMEMLOCK=infinity) grants the pin budget: 2 x 40 GiB expert pools +
|
|
# the 47.7 GiB PLE table + 4 GiB prefix host tier — ~132 GiB of pinned
|
|
# pages, which is ALSO why this script refuses to start when ttm's
|
|
# pages_limit/pages_max cap is under FRANK_PRIMARY_TTM_MIN_MIB (see below).
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_PRIMARY_ENV_FILE:-$SCRIPT_DIR/frank-primary.env}"
|
|
|
|
if [ ! -f "$ENV_FILE" ]; then
|
|
echo "ERROR: frank-primary env file absent: $ENV_FILE" >&2
|
|
exit 1
|
|
fi
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
|
|
: "${FRANK_PRIMARY_IMAGE:?FRANK_PRIMARY_IMAGE must name the radiance image}"
|
|
: "${FRANK_PRIMARY_MODEL_DIR:?FRANK_PRIMARY_MODEL_DIR must name the .rad dir}"
|
|
: "${FRANK_PRIMARY_MODEL_FILE:?FRANK_PRIMARY_MODEL_FILE must name the .rad file}"
|
|
: "${FRANK_PRIMARY_STATE_DIR:?FRANK_PRIMARY_STATE_DIR must name the /data state dir}"
|
|
: "${FRANK_PRIMARY_CONTAINER:?FRANK_PRIMARY_CONTAINER must name the container}"
|
|
: "${FRANK_PRIMARY_SERVED_NAME:?FRANK_PRIMARY_SERVED_NAME must name the alias}"
|
|
: "${FRANK_PRIMARY_PORT:?FRANK_PRIMARY_PORT must be the port}"
|
|
: "${FRANK_PRIMARY_TEMPLATE:?FRANK_PRIMARY_TEMPLATE must name the chat template}"
|
|
: "${FRANK_PRIMARY_TP:?FRANK_PRIMARY_TP must be set}"
|
|
: "${FRANK_PRIMARY_TP_WIRE:?FRANK_PRIMARY_TP_WIRE must be set}"
|
|
: "${FRANK_PRIMARY_MAX_SEQS:?FRANK_PRIMARY_MAX_SEQS must be set}"
|
|
: "${FRANK_PRIMARY_MAX_MODEL_LEN:?FRANK_PRIMARY_MAX_MODEL_LEN must be set}"
|
|
: "${FRANK_PRIMARY_PLACEMENT:?FRANK_PRIMARY_PLACEMENT must be set}"
|
|
: "${FRANK_PRIMARY_HOST_POOL_MIB:?FRANK_PRIMARY_HOST_POOL_MIB must be set}"
|
|
: "${FRANK_PRIMARY_GPU_HEADROOM_MIB:?FRANK_PRIMARY_GPU_HEADROOM_MIB must be set}"
|
|
: "${FRANK_PRIMARY_VRAM_WEIGHTS_MIB:?FRANK_PRIMARY_VRAM_WEIGHTS_MIB must be set}"
|
|
: "${FRANK_PRIMARY_VRAM_KV_MIB:?FRANK_PRIMARY_VRAM_KV_MIB must be set}"
|
|
: "${FRANK_PRIMARY_KV_DTYPE:?FRANK_PRIMARY_KV_DTYPE must be set}"
|
|
: "${FRANK_PRIMARY_SPEC_TOKENS:?FRANK_PRIMARY_SPEC_TOKENS must be set}"
|
|
: "${FRANK_PRIMARY_MAX_BATCHED_TOKENS:?FRANK_PRIMARY_MAX_BATCHED_TOKENS must be set}"
|
|
: "${FRANK_PRIMARY_PREFIX_HOST_MIB:?FRANK_PRIMARY_PREFIX_HOST_MIB must be set}"
|
|
: "${FRANK_PRIMARY_PREFIX_DISK_MIB:?FRANK_PRIMARY_PREFIX_DISK_MIB must be set}"
|
|
: "${FRANK_PRIMARY_NGRAM_PLACEMENT:?FRANK_PRIMARY_NGRAM_PLACEMENT must be set}"
|
|
: "${FRANK_PRIMARY_TTM_MIN_MIB:?FRANK_PRIMARY_TTM_MIN_MIB must be set}"
|
|
: "${FRANK_PRIMARY_RAD_MIN_MIB:?FRANK_PRIMARY_RAD_MIN_MIB must be set}"
|
|
|
|
# The image gate is the digest-pinned ref itself: a name@sha256 ref exists
|
|
# in storage only if THAT manifest was pulled — a retagged look-alike or a
|
|
# newer latest does not satisfy it. `podman image exists` takes a
|
|
# name@digest ref in one command — no pipefail/grep race (the 2026-09-30
|
|
# lesson). Runs as dank: the unit's process uses dank's ROOTLESS podman
|
|
# storage (root unit, User=dank — the fleet's frank-primary shape).
|
|
if ! podman image exists "$FRANK_PRIMARY_IMAGE"; then
|
|
echo "ERROR: radiance image absent: $FRANK_PRIMARY_IMAGE" >&2
|
|
echo "frank-primary: podman pull docker://stilldeadcode/radiance:latest (as dank)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# The payload gate: the .rad container file, sized. A truncated convert
|
|
# would otherwise buy a guaranteed long-timeout boot instead of this fast
|
|
# failure. The staging dir / raw checkpoint are NOT this script's concern
|
|
# (scripts/build-frank-primary-rad.sh built the file; its rad-info output
|
|
# is the record).
|
|
RAD="$FRANK_PRIMARY_MODEL_DIR/$FRANK_PRIMARY_MODEL_FILE"
|
|
if [ ! -f "$RAD" ]; then
|
|
echo "ERROR: .rad container missing: $RAD" >&2
|
|
echo "frank-primary: run scripts/build-frank-primary-rad.sh" >&2
|
|
exit 1
|
|
fi
|
|
RAD_MIB=$(( $(stat -c %s "$RAD") / 1048576 ))
|
|
if [ "$RAD_MIB" -lt "$FRANK_PRIMARY_RAD_MIN_MIB" ]; then
|
|
echo "ERROR: .rad container too small: $RAD is ${RAD_MIB} MiB, want >= $FRANK_PRIMARY_RAD_MIN_MIB" >&2
|
|
echo "frank-primary: the convert was truncated — rebuild it" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -f "$FRANK_PRIMARY_TEMPLATE" ]; then
|
|
echo "ERROR: chat template missing: $FRANK_PRIMARY_TEMPLATE" >&2
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "$FRANK_PRIMARY_STATE_DIR/kvcache/frank-primary"
|
|
# The engine's create is NOT recursive: with only the state dir mounted, its
|
|
# "cannot create /data/kvcache/frank-primary: No such file or directory"
|
|
# left the SSD prefix tier off (measured 2026-10-06 boot ledger). Pre-create
|
|
# the full path on the host so the tier opens on first boot.
|
|
|
|
mkdir -p "$FRANK_PRIMARY_STATE_DIR/tune"
|
|
# rad-tune's kernel-selection cache (see the runbook's "Tuning levers"):
|
|
# the engine's selector reads $RADIANCE_HOME/tune/<machine>.tune at boot;
|
|
# $RADIANCE_HOME defaults to /opt/radiance, which lives in the image, so
|
|
# the cache must ride a host mount or it dies with the --rm container
|
|
# that wrote it. The dir is pre-created for the same reason as kvcache
|
|
# (the engine's create is not recursive).
|
|
|
|
if [ ! -e /dev/kfd ] || [ ! -d /dev/dri ]; then
|
|
echo "ERROR: GPU device nodes absent (/dev/kfd, /dev/dri)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# TTM preflight: the pinned ledger (~2 x 40 GiB pools + 47.7 GiB PLE + 4 GiB
|
|
# prefix host = ~132 GiB) rides TTM's shared pages, capped by ttm's
|
|
# pages_limit (the current kernel exposes NO pages_max; the cap function
|
|
# takes the min over the knobs that exist). Below FRANK_PRIMARY_TTM_MIN_MIB
|
|
# the load dies mid-pin with an allocator error instead of this named
|
|
# refusal — raise the knob first (runtime sysfs write; kernel cmdline
|
|
# fallback + one reboot only if the file is read-only), see the runbook.
|
|
# RAISED 2026-10-05 pre-cutover to 52,428,800 pages = 200 GiB.
|
|
ttm_cap_mib() {
|
|
# min over the knobs THIS kernel exposes — the current kernel has
|
|
# pages_limit only (no pages_max); a knob that is absent is skipped,
|
|
# not a zero (a zero would refuse a healthy cap).
|
|
local v cur=""
|
|
for p in /sys/module/ttm/parameters/pages_limit /sys/module/ttm/parameters/pages_max; do
|
|
[ -r "$p" ] || continue
|
|
v="$(cat "$p" 2>/dev/null)" || continue
|
|
[ -n "$v" ] || continue
|
|
v=$(( v * 4 / 1024 ))
|
|
if [ "$v" -lt "${cur:-999999999}" ]; then cur=$v; fi
|
|
done
|
|
echo "${cur:-0}"
|
|
}
|
|
TTM_CAP="$(ttm_cap_mib)"
|
|
if [ "$TTM_CAP" -lt "$FRANK_PRIMARY_TTM_MIN_MIB" ]; then
|
|
echo "ERROR: ttm pages cap is ${TTM_CAP} MiB, the pinned ledger needs >= $FRANK_PRIMARY_TTM_MIN_MIB" >&2
|
|
echo "frank-primary: raise /sys/module/ttm/parameters/pages_limit (and pages_max if the kernel exposes one) to >= $FRANK_PRIMARY_TTM_MIN_MIB in 4 KiB pages" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# A crashed/killed unit can leave the container name registered (systemd
|
|
# SIGKILLs conmon before cleanup runs). Clear OUR name plus the legacy
|
|
# identities (the FreeToken-era frank-primary-qwen, the retired R9V
|
|
# container and the ad hoc experiment containers all hold this GPU); all
|
|
# idempotent.
|
|
for CTR in $FRANK_PRIMARY_LEGACY_CONTAINERS "$FRANK_PRIMARY_CONTAINER"; do
|
|
podman rm -f "$CTR" >/dev/null 2>&1 || true
|
|
done
|
|
|
|
LOG=/home/dank/logs/frank_primary.log
|
|
mkdir -p "$(dirname "$LOG")"
|
|
|
|
# Publisher-vetted container contract (docs/DOCKER.md, compose/common.yaml):
|
|
# --network host (the server binds the host's addresses directly, no NAT),
|
|
# --init (the engine releases the cards on SIGTERM), seccomp=unconfined
|
|
# (Docker/podman's default profile refuses io_uring, which the n-gram
|
|
# table's reads go through, and the NUMA binding calls the ROCm runtime
|
|
# makes for host memory), memlock unbounded (the pinned ledger — legal
|
|
# because the root unit grants LimitMEMLOCK=infinity to this dank process),
|
|
# /dev/kfd + /dev/dri (render/video groups are moot for a root container).
|
|
# The template rides a single-file mount; the .rad dir is ro; /data holds
|
|
# the prefix-cache disk tier.
|
|
podman run -d --name "$FRANK_PRIMARY_CONTAINER" \
|
|
--network host \
|
|
--init \
|
|
--security-opt seccomp=unconfined \
|
|
--ulimit memlock=-1:-1 \
|
|
--device /dev/kfd --device /dev/dri \
|
|
-v "$FRANK_PRIMARY_MODEL_DIR":/models:ro \
|
|
-v "$FRANK_PRIMARY_STATE_DIR":/data \
|
|
-v "$FRANK_PRIMARY_STATE_DIR/tune":/opt/radiance/share/radiance/tune \
|
|
-v "$FRANK_PRIMARY_TEMPLATE":/templates/template.jinja:ro \
|
|
"$FRANK_PRIMARY_IMAGE" \
|
|
--model "/models/$FRANK_PRIMARY_MODEL_FILE" \
|
|
--host :: --port "$FRANK_PRIMARY_PORT" \
|
|
--served-model-name "$FRANK_PRIMARY_SERVED_NAME" \
|
|
--override-chat-template /templates/template.jinja \
|
|
--tp "$FRANK_PRIMARY_TP" --tp-wire "$FRANK_PRIMARY_TP_WIRE" \
|
|
--max-num-seqs "$FRANK_PRIMARY_MAX_SEQS" \
|
|
--max-model-len "$FRANK_PRIMARY_MAX_MODEL_LEN" \
|
|
--placement "$FRANK_PRIMARY_PLACEMENT" \
|
|
--host-pool-mib "$FRANK_PRIMARY_HOST_POOL_MIB" \
|
|
--gpu-headroom-mib "$FRANK_PRIMARY_GPU_HEADROOM_MIB" \
|
|
--vram-weights-mib "$FRANK_PRIMARY_VRAM_WEIGHTS_MIB" \
|
|
--vram-kv-mib "$FRANK_PRIMARY_VRAM_KV_MIB" \
|
|
--kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE" \
|
|
--prefix-cache-host-mib "$FRANK_PRIMARY_PREFIX_HOST_MIB" \
|
|
--prefix-cache-dir /data/kvcache/frank-primary \
|
|
--prefix-cache-disk-mib "$FRANK_PRIMARY_PREFIX_DISK_MIB" \
|
|
--ngram-placement "$FRANK_PRIMARY_NGRAM_PLACEMENT" \
|
|
--num-speculative-tokens "$FRANK_PRIMARY_SPEC_TOKENS" \
|
|
--max-num-batched-tokens "$FRANK_PRIMARY_MAX_BATCHED_TOKENS" \
|
|
${FRANK_PRIMARY_EXTRA_ARGS:+$FRANK_PRIMARY_EXTRA_ARGS} \
|
|
> "$LOG" 2>&1
|
|
|
|
# From here on, this script's lifetime IS the container's: wait returns the
|
|
# container's exit code, so a crash is a nonzero ExecStart and the unit's
|
|
# Restart=always is real. (systemd stop goes through ExecStop in the unit —
|
|
# podman wait alone cannot stop the container child.)
|
|
exec podman wait "$FRANK_PRIMARY_CONTAINER"
|