frank-large: roll back to gufo Qwen3.8-27B Q4_K_XL, one session x 262144 f16 KV

frank-small: llama.cpp CUDA (master, sm_86) on MiMo Q6_K GGUF, 2 x 262144,
asymmetric KV (f16 K + q8_0 V, flash-attn pinned for the V quant)
This commit is contained in:
Frank committed 2026-10-01 22:40:59 -07:00
1 parent e452a1afbf
commit 8551bb006b
11 files changed
+729 -816

No files matched your search

+6 -32
View File
@@ -1,46 +1,20 @@
# frank-large.service — vLLM MiMo-V2.6-Distill-Qwen-9B (BF16) on the Strix
# Halo gfx1151 iGPU, :7770. 2026-10-01 cutover from gufo (tag
# frank-large-gufo-q4kxl-20261001 restores the Q4_K_XL + DFlash2 shape).
#
# WHY A ROOT UNIT: parity with frank-primary/frank-small — the unit is
# relink_unit'd from this repo into /etc/systemd/system/, PID1 grants
# LimitMEMLOCK=infinity (vLLM pins host buffers), and OOM protection lands
# on the engine instead of the co-tenants. The container runs in dank's
# ROOTLESS podman storage.
#
# The serve script drives `podman run` directly (image entrypoint is
# overridden to `vllm serve`) and blocks on the container, so a container
# death is a non-zero ExecStart. NOTE from frank-primary's experience:
# vLLM's API server can drain and exit 0 after an engine fault, so the
# restart policy MUST be always, not on-failure.
[Unit]
Description=frank-large — vLLM MiMo-V2.6-Distill-Qwen-9B (BF16) on the gfx1151 iGPU
Description=gufo (gufo-org/gufo, ROCm gfx1151) Qwen3.8-27B UD-Q8_K_XL, DFlash2 spec decode, 262144 ctx / 1 session, root systemd unit (dankclaw, frank-large, Strix Halo iGPU)
After=network-online.target
Wants=network-online.target
# Never let the start-rate limiter give up after a crash loop.
StartLimitIntervalSec=0
[Service]
Type=simple
User=dank
Group=dank
# The engine pins staging buffers; keep the memlock budget unbounded so an
# RLIMIT smaller than the pinned set cannot fail the load. Parity with the
# other LLM units on this fleet.
LimitMEMLOCK=infinity
Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
# systemd stop: podman wait alone cannot stop the container child; stop it
# explicitly or the engine outlives the unit holding the GPU and port.
ExecStop=/usr/bin/podman stop -t 120 frank-large-mimo
# Boot cost: ~200-240 s warm (measured 2026-10-01 bench window); cold torch
# compile is longer. Health stays 503 until the engine is up — do not kill.
TimeoutStartSec=1800
# ExecStop allows 120 s of drain; the stop timeout must exceed it (the
# frank-primary 2026-10-01 lesson: a short TimeoutStopSec records 'Failed
# with result timeout' on every stop).
TimeoutStopSec=180
Restart=always
Restart=on-failure
RestartSec=10
# The engine holds 17.5 GiB of weights plus the KV pool in the GTT
# aperture; it is the last process on this host the kernel should pick.
OOMScoreAdjust=-500
[Install]
WantedBy=multi-user.target