serverAppConfigs/scripts/serve-frank-primary.sh

#!/usr/bin/env bash
# serve-frank-primary.sh — frank-primary endpoint (FreeToken PR #587 ROCm
# build, orcarouter Qwen3.8-Flash-Next-Uncensored-FP8, ONE R9700 gfx1201,
# dankserver :7770; 2026-10-05 cutover from the tcclaviger vLLM fork — git
# tag frank-primary-vllm-shapeG-20261003 is the pre-cutover state).
#
# The engine is driven by `podman run` DIRECTLY and the run line below IS
# the engine argv: the image carries FreeToken installed at pinned PR head
# 77a7201 (scripts/docker/frank-primary-freetoken.Dockerfile builds it from
# the PR's own docs/install_amd.md stack). Unlike the vLLM fork there is no
# per-boot planner: experts live in pinned host RAM with an LRU GPU cache,
# KV is an explicit token pool, and the boot ledger in the log names
# Shape: 1 GPU (first-boot conservative — the second R9700 sits idle;
# --tensor-parallel-size EXISTS in FreeToken, so a 2-card TP shape is future
# tuning once this single-card shape has a decode baseline), KV pool 300,000
# tokens (24 KiB/token f16 across the 12 full-attention layers = 7.2 GiB;
# clears the harness worst case of 177,356 tokens and admits one NATIVE
# 262,144-token conversation), MoE LRU at 10% of all experts (~12 GiB),
# 2 decode slots. First-boot conservative; `ft ctl cache --kv/--moe` grows
# pools live, and the env comment block carries the resize law. Flag
# surface verified against docs/cli.md of the pinned head AND against
# `ft serve --help` inside the built image; every value here is env-driven
# (frank-primary.env).

# The root unit (scripts/frank-primary.service, User=dank,
# LimitMEMLOCK=infinity) grants the pin budget: ~120 GiB of expert weights
# plus the ~47.7 GiB PLE table pin in host RAM. A plain shell session also
# works, but the unit is the fleet-parity boot-persistent shape.
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FRANK_PRIMARY_ENV_FILE:-$SCRIPT_DIR/frank-primary.env}"

if [ ! -f "$ENV_FILE" ]; then
    echo "serve-frank-primary: env file missing: $ENV_FILE" >&2
    exit 1
fi
set -a
# shellcheck disable=SC1090
source "$ENV_FILE"
set +a

: "${FRANK_PRIMARY_IMAGE:?FRANK_PRIMARY_IMAGE must name the FreeToken image}"
: "${FRANK_PRIMARY_MODEL_DIR:?FRANK_PRIMARY_MODEL_DIR must name the FP8 payload}"
: "${FRANK_PRIMARY_MODEL_SHARDS:?FRANK_PRIMARY_MODEL_SHARDS must give the shard count}"
: "${FRANK_PRIMARY_CONTAINER:?FRANK_PRIMARY_CONTAINER must name the container}"
: "${FRANK_PRIMARY_SERVED_NAME:?FRANK_PRIMARY_SERVED_NAME must name the alias}"
: "${FRANK_PRIMARY_PUBLISH:?FRANK_PRIMARY_PUBLISH must be host:container}"
: "${FRANK_PRIMARY_FT_GPU:?FRANK_PRIMARY_FT_GPU must name the GPU index}"
: "${FRANK_PRIMARY_FT_NUM_TOKENS:?FRANK_PRIMARY_FT_NUM_TOKENS must size the KV pool}"
: "${FRANK_PRIMARY_FT_MAX_RUNNING:?FRANK_PRIMARY_FT_MAX_RUNNING must be set}"
: "${FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS:?FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS must be set}"
: "${FRANK_PRIMARY_FT_MOE_RATE:?FRANK_PRIMARY_FT_MOE_RATE must be set}"

# The image gate is EXISTENCE IN STORAGE, not a registry digest: the tag is
# built ON THE HOST (podman build from
# scripts/docker/frank-primary-freetoken.Dockerfile) and local builds carry
# no RepoDigests entry — the 362-line vLLM-era RepoDigests grep refused
# exactly this shape. Pinning lives in the Dockerfile instead: the FreeToken
# source is checked out at a hardcoded PR-head SHA and the base image is
# digest-pinned there. `podman image exists` accepts both a local tag and a
# name@digest ref, and it is one command — no pipefail/grep race (the
# 2026-09-30 lesson) for either shape.
if ! podman image exists "$FRANK_PRIMARY_IMAGE"; then
    echo "serve-frank-primary: image absent from rootless storage: $FRANK_PRIMARY_IMAGE" >&2
    echo "  build it: podman build -t $FRANK_PRIMARY_IMAGE -f scripts/docker/frank-primary-freetoken.Dockerfile scripts/docker/" >&2
    exit 1
fi

if [ ! -f "$FRANK_PRIMARY_MODEL_DIR/config.json" ]; then
    echo "serve-frank-primary: payload config missing: $FRANK_PRIMARY_MODEL_DIR/config.json" >&2
    exit 1
fi
SHARDS="$(find "$FRANK_PRIMARY_MODEL_DIR" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l)"
if [ "$SHARDS" != "$FRANK_PRIMARY_MODEL_SHARDS" ]; then
    echo "serve-frank-primary: payload shard count wrong: $FRANK_PRIMARY_MODEL_DIR" >&2
    echo "  found $SHARDS safetensors, want $FRANK_PRIMARY_MODEL_SHARDS" >&2
    echo "  re-fetch: scripts/download-frank-primary.sh" >&2
    exit 1
fi
if [ ! -f "$FRANK_PRIMARY_MODEL_DIR/model.safetensors.index.json" ]; then
    echo "serve-frank-primary: weight-map index missing: $FRANK_PRIMARY_MODEL_DIR/model.safetensors.index.json" >&2
    exit 1
fi

if [ ! -e /dev/kfd ] || [ ! -d /dev/dri ]; then
    echo "serve-frank-primary: GPU device nodes absent (/dev/kfd, /dev/dri)" >&2
    exit 1
fi

# A crashed/killed unit can leave the container name registered (systemd
# SIGKILLs conmon before cleanup runs). Clear OUR name plus the legacy
# identities (the vLLM-era frank-primary-qwen, the retired R9V container and
# the ad hoc experiment containers all hold this GPU); all idempotent.
for CTR in $FRANK_PRIMARY_LEGACY_CONTAINERS "$FRANK_PRIMARY_CONTAINER"; do
    podman rm -f "$CTR" >/dev/null 2>&1 || true
done

LOG=/home/dank/logs/frank_primary.log
mkdir -p "$(dirname "$LOG")"

# --device /dev/dri is the WHOLE DIRECTORY (individual renderD nodes enumerate
# without the card nodes and HIP refuses contexts). --ipc=host for the
# engine's shared memory; memlock unbounded for the ~120 GiB pinned expert
# rows + the 47.7 GiB PLE pin (the unit's LimitMEMLOCK=infinity is the other
# half of that law).
# HIP_VISIBLE_DEVICES pins visibility to ONE card so in-container index 0 is
# the right R9700; --gpu 0 selects it inside the visible set.
podman run -d --name "$FRANK_PRIMARY_CONTAINER" \
  --publish "$FRANK_PRIMARY_PUBLISH" \
  --ipc=host --ulimit memlock=-1:-1 \
  --cap-add SYS_PTRACE \
  --device /dev/kfd --device /dev/dri --device /dev/mem \
  -e HIP_VISIBLE_DEVICES="$FRANK_PRIMARY_FT_GPU" \
  -v "$FRANK_PRIMARY_MODEL_DIR":/models:ro \
  "$FRANK_PRIMARY_IMAGE" \
  ft serve --model /models \
    --served-model-name "$FRANK_PRIMARY_SERVED_NAME" \
    --host 0.0.0.0 --port 7770 \
    --gpu "$FRANK_PRIMARY_FT_GPU" \
    --num-tokens "$FRANK_PRIMARY_FT_NUM_TOKENS" \
    --max-running-requests "$FRANK_PRIMARY_FT_MAX_RUNNING" \
    --max-output-tokens "$FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS" \
    --moe-cache-rate "$FRANK_PRIMARY_FT_MOE_RATE" \
    ${FRANK_PRIMARY_FT_EXTRA_ARGS:+$FRANK_PRIMARY_FT_EXTRA_ARGS} \
  > "$LOG" 2>&1

# From here on, this script's lifetime IS the container's: wait returns the
# container's exit code, so a crash is a nonzero ExecStart and the unit's
# Restart=always is real. (systemd stop goes through ExecStop in the unit —
# podman wait alone cannot stop the container child.)
exec podman wait "$FRANK_PRIMARY_CONTAINER"