From 7da504a65112cb0b7c1549121b0b5b8cc8f38904 Mon Sep 17 00:00:00 2001 From: Frank Date: Thu, 1 Oct 2026 04:14:25 -0700 Subject: [PATCH] frank-primary: four full windows + Restart=always (runtime survivability) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five 262144-token contexts passed every boot gate (KV 1,350,182 / 5.15x) but left 0 bytes free at runtime: a real client request (995-token prompt, max_tokens=261149, sampling params straight from models.yml) killed Worker_TP1 with torch.OutOfMemoryError 'Tried to allocate 50.00 MiB ... 0 bytes free'. Boot is not proof of runtime survivability. Four slots x 262144 fp8 (KV_BYTES 8,311,013,376/rank) returns ~1.93 GiB per rank for activations/graph capture. Unit: Restart=always instead of on-failure — a runtime worker fault makes vLLM's API server exit clean, so ExecStart returned 0 and on-failure never fired, leaving the endpoint down until a human noticed. Also StartLimitIntervalSec=0 and OOMScoreAdjust=-500. --- AGENTS.md | 2 +- scripts/frank-primary.env | 23 ++++++++------- scripts/frank-primary.service | 23 +++++++++++++-- ...server-qwen38-fp8-expert-offload-tuning.md | 14 +++++---- serverAppConfigs-docs/LLM/index.md | 8 ++--- serverAppConfigs-docs/log.md | 24 +++++++++++---- tests/frank_primary_unit.bats | 29 +++++++++++-------- 7 files changed, 82 insertions(+), 41 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 37f6f9d..1695d1e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,7 @@ merge absorbed the former `vllmScripts` repo, history preserved). | Target | Service | Runtime | Model | Port | |--------|---------|---------|-------|------| -| Dankserver | `frank-primary` | tcclaviger RDNA4 vLLM fork (`docker.io/tcclaviger/vllm@sha256:da4029666…`, digest-pinned, plain `podman run`, root systemd unit) | `orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8` (~172.8 GiB, 131 shards), TP=2, expert offload automatic (fork plans it), prefetch OFF, **fp8 KV** (`--kv-cache-dtype fp8`), **no speculative decode**, MNBT 1024, MoE gather 32×64, **5 seqs × 262144 ctx (the model's full native window)** at KV_BYTES 10,388,766,720/rank (measured pool 1,350,182 tokens / 5.15× concurrency), util 0.98, and the client contract wired: `--reasoning-parser qwen3` (thinking in `reasoning`, never inline), `--enable-auto-tool-choice --tool-call-parser qwen3_xml`, default chat-template kwargs `enable_thinking` + `preserve_thinking`. Six full-length contexts do **not** fit (6×241664 asked 10.70 GiB vs 9.18 GiB free); 5×262144 was the first rung that passed (Frank's subtract-16k ladder). fp8+MTP4 shape restorable from git tag `frank-primary-fp8-mtp-20261001` | 7770 | +| Dankserver | `frank-primary` | tcclaviger RDNA4 vLLM fork (`docker.io/tcclaviger/vllm@sha256:da4029666…`, digest-pinned, plain `podman run`, root systemd unit) | `orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8` (~172.8 GiB, 131 shards), TP=2, expert offload automatic (fork plans it), prefetch OFF, **fp8 KV** (`--kv-cache-dtype fp8`), **no speculative decode**, MNBT 1024, MoE gather 32×64, **4 seqs × 262144 ctx (the model's full native window)** at KV_BYTES 8,311,013,376/rank, util 0.98, and the client contract wired: `--reasoning-parser qwen3` (thinking in `reasoning`, never inline), `--enable-auto-tool-choice --tool-call-parser qwen3_xml`, default chat-template kwargs `enable_thinking` + `preserve_thinking`. Five full-length contexts BOOT but leave ~0 bytes free at runtime — a real client request (995-token prompt, `max_tokens=261149`) killed the worker with `torch.OutOfMemoryError ... 0 bytes free` — so four slots carry the deployed shape and its VRAM headroom. Six do not fit at all (6×241664 asked 10.70 GiB vs 9.18 GiB free). fp8+MTP4 shape restorable from git tag `frank-primary-fp8-mtp-20261001` | 7770 | | Dankclaw | `frank-large` | gufo `gufo-org/gufo` v0.1.1, the Strix Halo engine, BUILT FROM SOURCE (root systemd unit; links the host ROCm 7.2.4) | Swift-1.5-Qwen3.8-27B-Uncensored UD-Q8_K_XL (27B dense, ABLITERATED/uncensored, 29.3 GiB, SWIFT OPEN LICENSE 1.0) with the z-lab DFlash2 Q4_K_M drafter (adaptive block policy, cap 7) on the gfx1151 iGPU (Strix Halo LPDDR, 114 GiB GTT), 262144 ctx single session, greedy, gufo's compiled-in Qwen renderer (NO Qwen-Sharp), text only | 7770 | | Dankclaw | `frank-small` | mainline llama.cpp master 37b53fd (root systemd unit) | HauhauCS Gemma4-12B-QAT-Uncensored-Balanced Q4_K_M (BF16 mmproj + MTP assistant sidecar, `draft-mtp,ngram-mod`) on the RTX 3090, 1048576 pool / 4 slots of 262,144, q8_0 KV, the fleet's fastest vision endpoint | 7771 | diff --git a/scripts/frank-primary.env b/scripts/frank-primary.env index bb52092..a39e8ac 100644 --- a/scripts/frank-primary.env +++ b/scripts/frank-primary.env @@ -60,18 +60,19 @@ FRANK_PRIMARY_PUBLISH=7770:7770 FRANK_PRIMARY_LEGACY_CONTAINERS="frank-primary-r9v qwen-fp8" # --- qualified engine shape (2026-10-01 night ladder, Frank's method) --------- -# SEQS 5 x MAXLEN 262144 at fp8 KV is the DEPLOYED shape (Frank's call: five -# contexts of the model's FULL native window beats six shortened ones). -# Measured: "GPU KV cache size: 1,350,182 tokens, Maximum concurrency for -# 262,144 tokens per request: 5.15x" — 39,462 tokens of margin over the -# 1,310,720-token contract. -# The night's ladder, both contracts: at SEQS=3 bf16, 262144/245760 OOMed at -# graph capture and 241664 was max possible (KV 732,359 / 3.03x). At SEQS=6 -# fp8, 6 x 241664 OOMed (tried 10.70 GiB with 9.18 GiB free) — six -# full-length contexts do NOT fit on this box; five do. +# SEQS 4 x MAXLEN 262144 at fp8 KV is the DEPLOYED shape: four contexts of +# the model's FULL native window, with VRAM headroom the five-slot shape did +# not have. Measured 2026-10-01: 5 x 262144 booted (KV 1,350,182 tokens / +# 5.15x) but left ~0 bytes free at runtime, and a real client request +# (995-token prompt, max_tokens=261149) killed the worker with +# `torch.OutOfMemoryError: Tried to allocate 50.00 MiB ... 0 bytes free`. +# Dropping to four slots returns ~1.93 GiB per rank for activations/graphs. +# At SEQS=3 bf16, 262144/245760 also OOMed at boot and 241664 was max +# possible; at SEQS=6 fp8, six full-length contexts do NOT fit at all +# (6 x 241664 asked 10.70 GiB against 9.18 GiB free). FRANK_PRIMARY_MAXLEN=262144 -FRANK_PRIMARY_SEQS=5 -FRANK_PRIMARY_KV_BYTES=10388766720 +FRANK_PRIMARY_SEQS=4 +FRANK_PRIMARY_KV_BYTES=8311013376 FRANK_PRIMARY_KV_DTYPE=fp8 FRANK_PRIMARY_MNBT=1024 FRANK_PRIMARY_GATHER_CHUNKS=32 diff --git a/scripts/frank-primary.service b/scripts/frank-primary.service index 0a11c56..c1f4b1d 100644 --- a/scripts/frank-primary.service +++ b/scripts/frank-primary.service @@ -8,8 +8,10 @@ # # Same port, same served alias; the harness models.yml id is unchanged. # The serve script drives `podman run` directly (the image entrypoint execs -# `vllm serve`) and then waits on the container, so a crashed engine is a -# nonzero ExecStart and Restart=on-failure is real. +# `vllm serve`) and then waits on the container. NOTE: that wait can return +# 0 even when the engine died mid-request (vLLM's API server drains and +# exits clean on a worker fault), which is why the restart policy is +# `always` — see the [Service] note. # # WHY A ROOT UNIT (not a rootless quadlet): the tiered-expert shape pins tens # of GiB of host RAM (the expert host slabs, ~20.5 GiB per rank of pinned @@ -23,6 +25,9 @@ Description=frank-primary — tcclaviger vLLM Qwen3.8-Flash-Next-Uncensored-FP8 on 2x R9700 After=network-online.target Wants=network-online.target +# Never let the start-rate limiter give up after a crash loop (a failed LLM +# endpoint that systemd refuses to retry is worse than a retrying one). +StartLimitIntervalSec=0 [Service] User=dank @@ -39,8 +44,20 @@ ExecStop=/usr/bin/podman stop -t 120 frank-primary-qwen # Boot cost: FP8 payload page-in plus torch compile/graph capture — # ~15 min cold (measured 2026-09-29), faster warm. TimeoutStartSec=1800 -Restart=on-failure +# Restart=always, NOT on-failure (measured 2026-10-01): a runtime +# torch.OutOfMemoryError kills the engine while vLLM's API server still +# exits 0, so ExecStart returned clean, systemd logged "Deactivated +# successfully", and on-failure NEVER fired — the endpoint stayed down until +# a human noticed. `always` restarts on any container death; an explicit +# `systemctl stop` still wins (systemd does not restart a unit the operator +# stopped), so bench windows and deploys are unaffected. +Restart=always RestartSec=10 +# Do not let the start-rate limiter give up after a crash loop. +StartLimitIntervalSec=0 +# The engine pins tens of GiB of host RAM (expert host slabs); it is the +# last process on this host that should be chosen by the kernel OOM killer. +OOMScoreAdjust=-500 [Install] WantedBy=multi-user.target diff --git a/serverAppConfigs-docs/LLM/dankserver-qwen38-fp8-expert-offload-tuning.md b/serverAppConfigs-docs/LLM/dankserver-qwen38-fp8-expert-offload-tuning.md index f74e89e..6e1db27 100644 --- a/serverAppConfigs-docs/LLM/dankserver-qwen38-fp8-expert-offload-tuning.md +++ b/serverAppConfigs-docs/LLM/dankserver-qwen38-fp8-expert-offload-tuning.md @@ -278,8 +278,8 @@ is only sampled at 3 depths; a 64k-192k-depth probe is the flagged gap. ### The deployed shape (2026-10-01: 5 x 262144, fp8, no spec) ``` -FRANK_PRIMARY_MAXLEN=262144 x SEQS 5 (1,310,720 tokens of contract) -FRANK_PRIMARY_KV_BYTES=10388766720 PER RANK (fp8, pool scales with windows) +FRANK_PRIMARY_MAXLEN=262144 x SEQS 4 (1,048,576 tokens of contract) +FRANK_PRIMARY_KV_BYTES=8311013376 PER RANK (fp8, pool scales with windows) FRANK_PRIMARY_KV_DTYPE=fp8 --kv-cache-dtype fp8 (K and V together) FRANK_PRIMARY_MNBT=1024 --max-num-batched-tokens 1024 R4D_LRU_GATHER_CHUNKS=32 / R4D_LRU_GATHER_LANES=64 (container env) @@ -297,10 +297,14 @@ memory environment). The night searched two concurrency contracts: | 3 × 237568, bf16 | PASS — KV 718,606 tokens | | **3 × 241664, bf16** | **PASS — KV 732,359 tokens, 3.03x** (max possible at 3 contexts) | | 6 × 241664, fp8 | **OOM** — tried 10.70 GiB against 9.18 GiB free: six full-length contexts do NOT fit | -| **5 × 262144, fp8** | **PASS on the first rung — KV 1,350,182 tokens, 5.15x; DEPLOYED** (39,462-token margin over the 1,310,720-token contract) | +| 5 × 262144, fp8 | PASS on the first rung — KV 1,350,182 tokens, 5.15x | +| **4 × 262144, fp8** | **DEPLOYED.** Five slots booted but left 0 bytes free at runtime: a real client request (995-token prompt, `max_tokens=261149`) killed the worker (`Tried to allocate 50.00 MiB ... 0 bytes free`). Dropping one slot returns ~1.93 GiB per rank for activations and graph capture | -The final shape is Frank's call: **five contexts of the model's full native -262,144 window** beat six shortened ones. fp8 KV is what buys the capacity — +Boot is NOT proof of runtime survivability: the five-slot shape passed every +boot gate and still died on the first long-generation request. After any +knife-edge shape change, send a request with a large `max_tokens` before +calling it done. The deployed shape is **four contexts of the model's full +native 262,144 window**. fp8 KV is what buys the capacity — 7,926 B/token against bf16's ~13,900-14,047 — at the cost of bf16's ~4 % decode edge. Six full-length contexts cannot fit at any KV dtype: the 6-seq graph/activation footprint leaves only 9.18 GiB for the pool, and 6 × 262144 diff --git a/serverAppConfigs-docs/LLM/index.md b/serverAppConfigs-docs/LLM/index.md index 69b9643..86f0976 100644 --- a/serverAppConfigs-docs/LLM/index.md +++ b/serverAppConfigs-docs/LLM/index.md @@ -4,9 +4,9 @@ Documentation for the LLM inference fleet on the two hosts (2026-10-01): **Dankserver runs the tcclaviger RDNA4 vLLM fork on frank-primary** (the digest-pinned `tcclaviger/vllm` image driving `podman run` directly, serving orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8 across both Radeon AI PRO R9700s -on :7770, TP=2 with automatic tiered-expert offload, **5 seqs x 262144 ctx -(the model's full native window)**, fp8 KV (6 full-length contexts do not -fit), no speculative decode, MNBT 1024, MoE gather 32x64, util 0.98, +on :7770, TP=2 with automatic tiered-expert offload, **4 seqs x 262144 ctx +(the model's full native window)**, fp8 KV (5 boot but leave no runtime +headroom; 6 do not fit), no speculative decode, MNBT 1024, MoE gather 32x64, util 0.98, reasoning/tool parsers wired (qwen3 / qwen3_xml) with preserve_thinking on, measured 36.4-39.4 tok/s decode — the 2026-10-01 bf16 no-spec cutover plus the night ladder; rollback frank-primary-fp8-mtp-20261001), and Dankclaw @@ -32,7 +32,7 @@ gfx1201, 32 GiB each] B2[tcclaviger RDNA4 vLLM fork, digest-pinned podman run, frank-primary, port 7770] C[Qwen3.8-Flash-Next FP8 bf16 no-spec -TP=2, expert offload automatic, 5 x 262144 ctx, +TP=2, expert offload automatic, 4 x 262144 ctx, fp8 KV, MNBT 1024, gather 32x64, qwen3 parsers] end diff --git a/serverAppConfigs-docs/log.md b/serverAppConfigs-docs/log.md index 208f3c8..1eaa5d0 100644 --- a/serverAppConfigs-docs/log.md +++ b/serverAppConfigs-docs/log.md @@ -722,8 +722,22 @@ <= 262,144) and `preserve_thinking` moved from a bare top-level body field to `chat_template_kwargs` (the form vLLM actually reads). -- Production bring-up: the winner (241,664) boots and serves on :7770 — - engine log verified (KV 732,359 tokens, 3.03x concurrency, MNBT 1024, - gather 32x64). Rollback target for the old fp8+MTP4 shape: git tag - `frank-primary-fp8-mtp-20261001`; the 237,568 fallback is documented in - the runbook. +- **Runtime-survivability correction: five slots -> FOUR.** 5 x 262144 fp8 + passed every boot gate (KV 1,350,182 tokens / 5.15x) but left 0 bytes free + at runtime; a real client request (995-token prompt, `max_tokens=261149`, + sampling params straight from models.yml) killed Worker_TP1 with + `torch.OutOfMemoryError: Tried to allocate 50.00 MiB ... 0 bytes free` and + the engine shut down. Boot is not proof of runtime survivability — after a + knife-edge shape change, send a large-`max_tokens` request before calling it + done. FOUR slots x 262144 fp8 (KV_BYTES 8,311,013,376/rank) returns ~1.93 + GiB per rank for activations/graph capture. Frank's call 2026-10-01. +- `/scripts/frank-primary.service` — **Restart=always, not on-failure** (plus + `StartLimitIntervalSec=0` and `OOMScoreAdjust=-500`). A runtime worker fault + makes vLLM's API server exit CLEAN, so ExecStart returned 0, systemd logged + "Deactivated successfully", and on-failure NEVER fired: the endpoint stayed + down until a human noticed. An explicit `systemctl stop` still wins, so + bench windows and deploys are unaffected. +- Production bring-up: the shape boots and serves on :7770 — engine log + verified (KV cache size, concurrency, MNBT 1024, gather 32x64) plus live + contract probes (reasoning split, tool_calls). Rollback target for the + 2026-09-29 fp8+MTP4 shape: git tag `frank-primary-fp8-mtp-20261001`. diff --git a/tests/frank_primary_unit.bats b/tests/frank_primary_unit.bats index 07c4f11..8e533a7 100755 --- a/tests/frank_primary_unit.bats +++ b/tests/frank_primary_unit.bats @@ -42,7 +42,11 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" # dank session's own hard limit is 8 MiB and cannot be raised without # restarting user@1000.service). grep -q '^WantedBy=multi-user.target$' "$UNIT" - grep -q '^Restart=on-failure$' "$UNIT" + # Restart=always, not on-failure: a runtime worker OOM makes vLLM's API + # server exit CLEAN, so ExecStart returns 0 and on-failure never fires + # (measured 2026-10-01 — the endpoint stayed down until a human noticed). + grep -q '^Restart=always$' "$UNIT" + ! grep -q '^Restart=on-failure$' "$UNIT" grep -q '^RestartSec=' "$UNIT" grep -q '^User=dank$' "$UNIT" grep -q '^LimitMEMLOCK=infinity$' "$UNIT" @@ -75,19 +79,20 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" } @test "the qualified engine shape is in the env file" { - # 2026-10-01 night ladder (Frank's method). Two contracts were searched: - # SEQS=3 bf16 topped out at 241664 (262144/245760 OOMed at graph capture; - # KV 732,359 tokens / 3.03x). SEQS=6 fp8 did NOT fit — 6 x 241664 tried - # 10.70 GiB against 9.18 GiB free, so six full-length contexts are - # impossible here. SEQS=5 fp8 at the model's FULL native 262144 window - # PASSED on the first rung and is DEPLOYED: KV 1,350,182 tokens, - # Maximum concurrency 5.15x, 39,462 tokens of margin over the - # 1,310,720-token contract. UTIL stays 0.98 (lowering it shrinks the - # engine's own budget instead of buying slack). + # 2026-10-01 night ladder (Frank's method), then a runtime-survivability + # correction. SEQS=3 bf16 topped out at 241664 (KV 732,359 / 3.03x); + # SEQS=6 fp8 does NOT fit (6 x 241664 asked 10.70 GiB against 9.18 GiB + # free); SEQS=5 fp8 x 262144 BOOTED (KV 1,350,182 / 5.15x) but left ~0 + # bytes free at runtime and a real client request (995-token prompt with + # max_tokens=261149) killed the worker with `Tried to allocate 50.00 MiB + # ... 0 bytes free`. FOUR slots at the full native 262144 window is the + # deployed shape: ~1.93 GiB more headroom per rank for + # activations/graphs. UTIL stays 0.98 (lowering it shrinks the engine's + # own budget instead of buying slack). grep -q '^FRANK_PRIMARY_MAXLEN=262144$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_SEQS=5$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_SEQS=4$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_UTIL=0.98$' "$ENV_FILE" - grep -q '^FRANK_PRIMARY_KV_BYTES=10388766720$' "$ENV_FILE" + grep -q '^FRANK_PRIMARY_KV_BYTES=8311013376$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_KV_DTYPE=fp8$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_MNBT=1024$' "$ENV_FILE" grep -q '^FRANK_PRIMARY_GATHER_CHUNKS=32$' "$ENV_FILE"