diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 69bc00f..5f141ee 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -116,8 +116,11 @@ done # --device /dev/mem exists for the fleet's other GPU containers; the GTT # aperture (114 GiB) needs no size knob. Models mount read-only wholesale so -# spec drafts resolve from disk, and the HF cache mount keeps tokenizer -# metadata out of the container's ephemeral layer. +# spec drafts resolve from disk. THE PATH HANDED TO vllm IS THE CONTAINER +# PATH: the engine runs inside this container, and a host path that is not +# mounted at the same location dies as "Repo id must be in the form +# 'namespace/repo_name'" (2026-10-01 first boot, 51 crash-loop restarts). +MODEL_CONTAINER_DIR="/models/${FRANK_LARGE_MODEL_DIR##*/}" podman run -d \ --name "$FRANK_LARGE_CONTAINER" \ --device /dev/kfd --device /dev/dri --device /dev/mem \ @@ -129,7 +132,7 @@ podman run -d \ -p "$FRANK_LARGE_PUBLISH" \ --entrypoint vllm \ "$FRANK_LARGE_IMAGE" \ - serve "$FRANK_LARGE_MODEL_DIR" \ + serve "$MODEL_CONTAINER_DIR" \ --served-model-name "$FRANK_LARGE_SERVED_NAME" \ --host 0.0.0.0 --port "${FRANK_LARGE_PUBLISH##*:}" \ --dtype bfloat16 \ diff --git a/serverAppConfigs-docs/index.md b/serverAppConfigs-docs/index.md index b29de60..0ccb4b0 100644 --- a/serverAppConfigs-docs/index.md +++ b/serverAppConfigs-docs/index.md @@ -213,14 +213,20 @@ The LLM inference fleet on the two hosts. Full index: agentionai/Qwen3.8-Flash-Next-ROCmFP4-FAST-imatrix (87.06 GiB, Q4_0_ROCMFP4_FAST) + own MTP draft, build-qwen38 (retired 2026-08-30; rollback record) -- [[frank-large-gufo-qwen38-27b]] — the ACTIVE frank-large runbook since - 2026-09-28 (root systemd unit, port 7770): gufo (gufo-org/gufo) v0.1.1, - the Strix Halo inference engine, built from source on Dankclaw and - serving unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL (a 27B dense model) with the - z-lab DFlash2 Q4_K_M drafter on the gfx1151 iGPU — one 262,144-token - conversation, adaptive block policy, greedy, text-only, the two - from-source build workarounds, what changed in the client contract, the - measured prefill/decode trade against llama.cpp +- [[frank-large-vllm-mimo-v26-bf16]] — the ACTIVE frank-large runbook + since 2026-10-01 (root systemd unit, port 7770): the digest-pinned + kyuz0/vllm-therock-gfx1151 image (vLLM 0.30, TheRock ROCm 10) serving + XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B in FULL BF16 on the gfx1151 iGPU, + 4 x 262,144 ctx, with CROSS-MODEL MTP spec decode from Qwen/Qwen3.5-9B — + the roof math (13.1 t/s base is ~94% of LPDDR5x BF16), the measured + MTP-vs-DFlash table (19.8 vs 17.4 t/s), the greedy-text numerics trap, + the `reasoning` field change, troubleshooting +- [[frank-large-gufo-qwen38-27b]] — RETIRED 2026-09-28 → 2026-10-01 + (replaced by [[frank-large-vllm-mimo-v26-bf16]]; rollback via tag + frank-large-gufo-q4kxl-20261001): gufo v0.1.1 built from source serving + Qwen3.8-27B GGUF + the z-lab DFlash2 drafter on the gfx1151 iGPU — the + two from-source build workarounds, the client-contract delta, the + measured prefill/decode trade - [[frank-large-unsloth-qwen38-27b-q8xl]] — RETIRED 2026-09-28 (replaced by [[frank-large-gufo-qwen38-27b]]; rollback target): stock llama.cpp (unsloth b11030, ROCm gfx1151) serving diff --git a/serverAppConfigs-docs/log.md b/serverAppConfigs-docs/log.md index e06f2b0..15e3f07 100644 --- a/serverAppConfigs-docs/log.md +++ b/serverAppConfigs-docs/log.md @@ -912,3 +912,16 @@ any repo service. DFlash 2.58 GiB), disk down to ~26 GiB free - reclaim pass due. gufo rollback pinned at git tag `frank-large-gufo-q4kxl-20261001`; engine, Q4_K_XL payload and DFlash2 drafter all remain on the host. +- First deploy attempt #2 crash-looped the container 51 times: the serve + script handed vLLM the HOST model path while the tree is mounted at + `/models` inside the container — transformers rejected it as a malformed + repo id. Fixed by converting to the container path (the spec draft was + already converted; the target was not), pinned by bats. Also fixed the + deployer's own draft gate (same single-file bug as the serve script's). +- Reclaim (Frank approved): Swift-1.5 Q8_K_XL payload (30 G, refetchable via + `scripts/download-frank-large-swift.sh`) and the orphaned flux-2-klein + tree under `models/checkpoints` (23 G; ComfyUI mounts + `comfyui-data/models`, not this) purged via + `~/.AGENTTRASH/20261001-frank-large-reclaim/` — 52 G freed. KEPT the + 1.1 G `Qwen3.8-27B-DFlash2-Q4_K_M.gguf` (it is the gufo rollback's + drafter, not a remnant). diff --git a/tests/frank_large_unit.bats b/tests/frank_large_unit.bats index 8e5a5e8..9589a96 100755 --- a/tests/frank_large_unit.bats +++ b/tests/frank_large_unit.bats @@ -109,6 +109,12 @@ FETCH_MTPSRC="$REPO_ROOT/scripts/download-frank-large-qwen35.sh" grep -q -- '--dtype bfloat16' "$SERVE" grep -q -- '--trust-remote-code' "$SERVE" grep -q -- '--ulimit memlock=-1:-1' "$SERVE" + # vllm runs INSIDE the container: the model path handed to `serve` is + # the /models mount path, never the host path (2026-10-01: the host + # path crash-looped 51 times on "Repo id must be in the form…"). + grep -q 'serve "$MODEL_CONTAINER_DIR"' "$SERVE" + grep -q 'MODEL_CONTAINER_DIR="/models/' "$SERVE" + ! grep -qE '^\s+serve "\$FRANK_LARGE_MODEL_DIR"' "$SERVE" } @test "speculative decode is cross-model MTP BY MEASUREMENT" {