Compare commits
213
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2e86445fc5 | ||
|
|
cf6319aad7 | ||
|
|
a9d45e7012 | ||
|
|
2581b8d4b6 | ||
|
|
09a0251d2d | ||
|
|
066813ea95 | ||
|
|
7daa886dd6 | ||
|
|
bae78e7dae | ||
|
|
a6f55f4cc6 | ||
|
|
a9da6589a6 | ||
|
|
1a6a78ef73 | ||
|
|
c52ba781e3 | ||
|
|
0bf26b0741 | ||
|
|
394fd8535b | ||
|
|
55f87fcdce | ||
|
|
5ced474eb9 | ||
|
|
2992a7ddd6 | ||
|
|
7c77326c33 | ||
|
|
65dec6a80e | ||
|
|
f99b698d25 | ||
|
|
757a90057f | ||
|
|
009e8a885d | ||
|
|
6b677139ad | ||
|
|
184a5c665a | ||
|
|
958b10545e | ||
|
|
c4dca68a03 | ||
|
|
f29dbdf7c7 | ||
|
|
98f11a19c1 | ||
|
|
a30ac9457a | ||
|
|
75695c312a | ||
|
|
c507f91438 | ||
|
|
b175f4afd1 | ||
|
|
971e23a25c | ||
|
|
3ca64e14fc | ||
|
|
f3dfea13ad | ||
|
|
02fa1e22ae | ||
|
|
1aef7418ad | ||
|
|
fa5a444b64 | ||
|
|
90f678582e | ||
|
|
cc0fdd15a1 | ||
|
|
f148ecab4a | ||
|
|
c2f3411342 | ||
|
|
8c5bde7034 | ||
|
|
22a7331100 | ||
|
|
c1e192df00 | ||
|
|
8375567de3 | ||
|
|
79c2e64c2c | ||
|
|
f20da6a6ff | ||
|
|
058ea6cca8 | ||
|
|
3f8531179e | ||
|
|
3c5d77d1fa | ||
|
|
203b0be145 | ||
|
|
74da746c51 | ||
|
|
7066accc25 | ||
|
|
76aaa2543b | ||
|
|
8220030ba1 | ||
|
|
5191c49967 | ||
|
|
f9330fa201 | ||
|
|
4b71c014e9 | ||
|
|
cee90eea52 | ||
|
|
73c37f4301 | ||
|
|
6632d0a2f6 | ||
|
|
caf5e1e0ca | ||
|
|
5278f5be26 | ||
|
|
ec6184de9a | ||
|
|
1d062f06a3 | ||
|
|
b51e078e36 | ||
|
|
68a525740d | ||
|
|
5b7b3ccb90 | ||
|
|
2137080d1c | ||
|
|
f747cb0dab | ||
|
|
db8faa68ff | ||
|
|
cf6c62d44a | ||
|
|
0aafad5f2c | ||
|
|
a3f45ce028 | ||
|
|
45de2fd47f | ||
|
|
fa858f4c10 | ||
|
|
07af1a9366 | ||
|
|
f0d0614ce6 | ||
|
|
9aed9987b6 | ||
|
|
4048dc8630 | ||
|
|
9bc40fd899 | ||
|
|
bb60e8524f | ||
|
|
319dd0f7da | ||
|
|
650dfff5a2 | ||
|
|
de70320cde | ||
|
|
910c54b6ff | ||
|
|
8c887e6fc9 | ||
|
|
49dd2be4cd | ||
|
|
a11f703c49 | ||
|
|
6cdf88f611 | ||
|
|
259fc5505d | ||
|
|
3e46d51df8 | ||
|
|
d062a376b2 | ||
|
|
95219cf4c1 | ||
|
|
fee66187a1 | ||
|
|
0258ae0897 | ||
|
|
9aee00f2b3 | ||
|
|
41d2f56269 | ||
|
|
d8da7a9eb7 | ||
|
|
44e7a8dfaf | ||
|
|
f2575a1018 | ||
|
|
87721ab0fe | ||
|
|
c71a3066c0 | ||
|
|
df6b5bb6e8 | ||
|
|
95923d3a98 | ||
|
|
81e6122891 | ||
|
|
2d89bf97cc | ||
|
|
20d7182f4b | ||
|
|
0f9faf05ea | ||
|
|
31b1ec397b | ||
|
|
7fef3cb907 | ||
|
|
bb3532dfec | ||
|
|
0f7274e55a | ||
|
|
da36fa32cf | ||
|
|
713b126639 | ||
|
|
2d9cecae6d | ||
|
|
fadad18da1 | ||
|
|
306ff32427 | ||
|
|
d39424ffba | ||
|
|
3eb4fa2b62 | ||
|
|
ba67ca5c9e | ||
|
|
8343397584 | ||
|
|
5e166543fa | ||
|
|
812d9ebff5 | ||
|
|
fb1a4fd150 | ||
|
|
90a08ab4b5 | ||
|
|
272acb4c7f | ||
|
|
13112a93f5 | ||
|
|
baf33d016f | ||
|
|
1919ad7f31 | ||
|
|
8617b79a18 | ||
|
|
33eae71d33 | ||
|
|
8618ec6dbe | ||
|
|
e2308dff71 | ||
|
|
6ba19f6f48 | ||
|
|
0ff016a6b2 | ||
|
|
bd5c28c4e9 | ||
|
|
4a02dc0980 | ||
|
|
1f6bdac759 | ||
|
|
a6ad9af708 | ||
|
|
380aafa1cc | ||
|
|
5573ca171f | ||
|
|
1c725ea024 | ||
|
|
1862dc3c0f | ||
|
|
2930431052 | ||
|
|
8f53df7b4f | ||
|
|
9c508218e9 | ||
|
|
77decb545e | ||
|
|
69f10a17bb | ||
|
|
be840b4881 | ||
|
|
b0b840b149 | ||
|
|
bdbe814fac | ||
|
|
87695c5126 | ||
|
|
0900551317 | ||
|
|
fece3ffdd0 | ||
|
|
aa9908257d | ||
|
|
67141c8c02 | ||
|
|
e5893dc30d | ||
|
|
b06b47c7b4 | ||
|
|
2e64f66b82 | ||
|
|
35a45b8a66 | ||
|
|
1fee99c8ce | ||
|
|
b820d341f5 | ||
|
|
9d8b18553d | ||
|
|
a1c4701d7e | ||
|
|
c4603b130c | ||
|
|
9a144f779e | ||
|
|
50459e4e31 | ||
|
|
ff8c0ee264 | ||
|
|
6ec0b1a85d | ||
|
|
b80886e318 | ||
|
|
47a2c1287b | ||
|
|
c67a2aeb80 | ||
|
|
0a4ff30db2 | ||
|
|
c682c6329c | ||
|
|
6584c059d6 | ||
|
|
7d3dd33de0 | ||
|
|
0ae923b879 | ||
|
|
b7c3bdb2be | ||
|
|
d57174c8d3 | ||
|
|
ea1132aa0a | ||
|
|
57f33e5289 | ||
|
|
1ac4d48371 | ||
|
|
812193aa63 | ||
|
|
39f0e0d8d4 | ||
|
|
656012a0fa | ||
|
|
295d5784d3 | ||
|
|
c769ff2f63 | ||
|
|
e460bbbab5 | ||
|
|
30904bac1b | ||
|
|
f8f845d86b | ||
|
|
934285c2b2 | ||
|
|
7fe881de01 | ||
|
|
a02850203d | ||
|
|
8f129180d8 | ||
|
|
72bd4c967e | ||
|
|
125d04967f | ||
|
|
d82c5c03b7 | ||
|
|
4bb10cd20e | ||
|
|
a7c68db9d0 | ||
|
|
8551bb006b | ||
|
|
e452a1afbf | ||
|
|
01ba41cab4 | ||
|
|
cde91c1a00 | ||
|
|
6513672f31 | ||
|
|
5df6ef3166 | ||
|
|
3c339eeba2 | ||
|
|
780c307354 | ||
|
|
de383fe53e | ||
|
|
1127a52078 | ||
|
|
dab483c79f | ||
|
|
b3313340fa |
@@ -1,4 +1,4 @@
|
||||
# Tunables for frank-large (kingjones777/Qwen3.8-Flash-Next-ROCmFP4-
|
||||
# Tunables for frank-dragoon (kingjones777/Qwen3.8-Flash-Next-ROCmFP4-
|
||||
# STRIX-GGUF on the kingjones30/ROCmFPX llama.cpp runtime, Dankclaw,
|
||||
# Strix Halo gfx1151, unified memory).
|
||||
#
|
||||
@@ -56,7 +56,7 @@ CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
|
||||
CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true,"preserve_reasoning":true}'
|
||||
|
||||
# Alias + network.
|
||||
ALIAS=frank-large
|
||||
ALIAS=frank-dragoon
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7770
|
||||
|
||||
+3
-3
@@ -1,5 +1,5 @@
|
||||
[Unit]
|
||||
Description=ROCmFPX (kingjones30 fork) llama.cpp Qwen3.8-Flash-Next ROCmFP4 STRIX 177B MoE (dankclaw, frank-large)
|
||||
Description=ROCmFPX (kingjones30 fork) llama.cpp Qwen3.8-Flash-Next ROCmFP4 STRIX 177B MoE (dankclaw, frank-dragoon)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
@@ -7,8 +7,8 @@ Wants=network-online.target
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
|
||||
Environment="FRANK_DRAGOON_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-dragoon.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-dragoon.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
# is reachable on this port regardless of the boot tag. The NPU is a
|
||||
# single-lane device: generations serialize FIFO, and the first request after a
|
||||
# model swap pays the ~6-11 s load cost. The NPU draws from the unified RAM
|
||||
# pool shared with frank-large (iGPU).
|
||||
# pool shared with frank-dragoon (iGPU).
|
||||
FLM_MODEL=nuextract3:4b
|
||||
FLM_HOST=0.0.0.0
|
||||
FLM_PORT=7771
|
||||
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX-GGUF as
|
||||
# frank-large via the kingjones30/ROCmFPX llama.cpp runtime on
|
||||
# frank-dragoon via the kingjones30/ROCmFPX llama.cpp runtime on
|
||||
# Dankclaw (Strix Halo gfx1151, unified memory).
|
||||
#
|
||||
# Runtime: kingjones30/ROCmFPX (fork of charlie12345/ROCmFPX with the
|
||||
@@ -29,7 +29,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
@@ -44,7 +44,7 @@ fi
|
||||
: "${MMPROJ_PATH:=/home/dank/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST/mmproj/mmproj-Qwen3.8-Flash-Next-f16.gguf}"
|
||||
: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
|
||||
: "${CHAT_TEMPLATE_KWARGS:={}}"
|
||||
: "${ALIAS:=frank-large}"
|
||||
: "${ALIAS:=frank-dragoon}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7770}"
|
||||
: "${CTX_SIZE:=401408}"
|
||||
@@ -4,7 +4,7 @@
|
||||
# Runs `flm serve <tag>` on the XDNA2 NPU (50 TOPS, Dankclaw). The NPU is a
|
||||
# single-lane device: generations serialize in FIFO order, so this is a
|
||||
# background/triage endpoint, not an interactive multi-user chat. The NPU
|
||||
# draws from the unified RAM pool shared with frank-large (the iGPU
|
||||
# draws from the unified RAM pool shared with frank-dragoon (the iGPU
|
||||
# llama.cpp service); see serverAppConfigs-docs/LLM/frank-npu.md
|
||||
# for the memory budget and the model-selection tradeoff.
|
||||
#
|
||||
|
||||
@@ -9,11 +9,12 @@ services run from `/home/dank/serverAppConfigs` on each host (the 2026-08-22
|
||||
merge absorbed the former `vllmScripts` repo, history preserved).
|
||||
|
||||
- **Dankserver** (192.168.0.235) — media/library stack (RomM, Copyparty,
|
||||
Calibre-Web, Jellyfin, Plex) plus the `frank-primary` LLM service.
|
||||
Calibre-Web, Jellyfin, Plex) plus the `frank-archon` LLM service.
|
||||
- **Dankclaw** (192.168.0.143) — AI/search/dev/observability stack (SearXNG
|
||||
via Gluetun, ComfyUI, OpenWebUI, Hermes, Gitea, Grafana, Prometheus, Loki,
|
||||
Alloy, Pi-hole) plus TWO LLM services: `frank-large` (stock llama.cpp on
|
||||
the gfx1151 iGPU) and `frank-small` (SGLang on the RTX 3090). Caddy on
|
||||
Alloy, Pi-hole) plus TWO LLM services: `frank-dragoon` (flash-server
|
||||
container on the gfx1151 iGPU) and `frank-zealot` (llama.cpp CUDA on the
|
||||
RTX 3090). Caddy on
|
||||
Dankclaw reverse-proxies both hosts.
|
||||
- **FrankBook-Pro** (192.168.0.112) — macOS mini running the *arr stack
|
||||
(Prowlarr, Sonarr, Radarr, Lidarr, Readarr, qBittorrent) as quadlets inside
|
||||
@@ -24,9 +25,9 @@ merge absorbed the former `vllmScripts` repo, history preserved).
|
||||
|
||||
| Target | Service | Runtime | Model | Port |
|
||||
|--------|---------|---------|-------|------|
|
||||
| Dankserver | `frank-primary` | tcclaviger RDNA4 vLLM fork (`docker.io/tcclaviger/vllm@sha256:da4029666…`, digest-pinned, plain `podman run`, root systemd unit) | `orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8` (~172.8 GiB, 131 shards), TP=2, expert offload automatic (fork plans it), prefetch OFF, **bf16 KV** (`--kv-cache-dtype auto`), **no speculative decode**, MNBT 2048, MoE gather 32×64, **2 seqs × 262144 ctx (the model's full native window)** at KV_BYTES 7,300,000,000/rank (bf16 measures 13,839 B/token), with **`--expert-precise-memory`** — the fitted 5.25 GiB overhead instead of the 7.60 GiB default, which hands the compile/graph-capture headroom to the expert arena (~7,278 slots vs 6,248) — util 0.98, and the client contract wired: `--reasoning-parser qwen3` (thinking in `reasoning`, never inline), `--enable-auto-tool-choice --tool-call-parser qwen3_xml`, default chat-template kwargs `enable_thinking` + `preserve_thinking`. Capacity history: five full-length contexts boot but leave ~0 bytes free at runtime (a real request with `max_tokens=261149` killed the worker: `Tried to allocate 50.00 MiB ... 0 bytes free`); six never fit (6×241664 asked 10.70 GiB vs 9.18 GiB free). DCP cannot help here — it requires TP > the model's 2 KV heads, i.e. four GPUs. fp8+MTP4 shape restorable from git tag `frank-primary-fp8-mtp-20261001` | 7770 |
|
||||
| Dankclaw | `frank-large` | gufo `gufo-org/gufo` v0.1.1, the Strix Halo engine, BUILT FROM SOURCE (root systemd unit; links the host ROCm 7.2.4) | `unsloth/Qwen3.8-27B-GGUF` **UD-Q4_K_XL** (27B dense, **vanilla — no longer abliterated**, 16.35 GiB, Apache-2.0) with the z-lab DFlash2 Q4_K_M drafter (adaptive block policy, cap 7) on the gfx1151 iGPU (Strix Halo LPDDR, 114 GiB GTT), **4 sessions x 163840 ctx, unquantized (f16) KV** — gufo has no KV-quant lever — greedy, gufo's compiled-in Qwen renderer (NO Qwen-Sharp), text only. Swapped 2026-10-01 from ajgazin's abliterated Swift-1.5 UD-Q8_K_XL (29.3 GiB): ~13 GiB less weights is what funds four f16 sessions, at the cost of the uncensored behaviour | 7770 |
|
||||
| Dankclaw | `frank-small` | SGLang in a self-contained python3.12 venv (root systemd unit; the venv carries its own CUDA 13 stack — see the runbook) | letechlead MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound (qwen3_5 hybrid, 9B dense, 12.0 GiB, INT8 W8A16 AutoRound, in-checkpoint vision tower, NO MTP weights shipped so no spec decode) on the RTX 3090, fp8 KV, pinned 606,208-token pool / 2 slots of 262,144 (native window; 2026-10-01 reslice from 4 x 151,552 for compaction headroom), bf16 mamba states, eager prefill, 83.9 tok/s decode | 7771 |
|
||||
| Dankserver | `frank-archon` | **radiance** (StillDeadcode, **pin 1.0.13** digest-pinned `stilldeadcode/radiance@sha256:4c3f484564…`, plain `podman run`, root systemd unit) | ukisai/Swift1.5-Qwen3.8-Flash-Next BF16 (335.3 GiB, 397/397 sha256-verified) converted by `scripts/convert-swift15-w4dn8.sh` to `qwen4exp-swift15-w4dn8-i8-hc8m.rad` (**116,416 MiB**): **all 49,152 experts GPTQ w4** (10M-token calibration on engine 1.2.0) + **24 whole bf16-protected experts/layer** (energy-share law), int8 trunk/lm_head, fp8 HC, PLE n-gram fp8 fixed-scale, on **BOTH R9700s TP=2** (`--tp-wire wht6`), experts in a **44 GiB/rank pinned host pool** + VRAM cache (`--placement expert_tiered`, explicit `--vram-weights-mib 25600 --vram-kv-mib 5120` = 30,720 of the **30,969 MiB** the engine's own budget table makes claimable: card 32,624 MiB, settle floor 209 MiB), **47.7 GiB n-gram copy pinned to host RAM** (`--ngram-placement ram`; root unit + `LimitMEMLOCK=infinity`), **fp8 KV** (pool 5,120 MiB = the 3-slot contract + 1.3 % — the only dtype that funds 3 × 200,000 without gutting the slab, since bf16 bills 13.8 KiB/token), **MTP spec decode depth 2**, `--max-num-seqs 3` (the capacity contract is 3 × 200,000; 3 × (1+2) = 9 all-reduce rows, inside the 25-row wht6 exactness bound), `--max-model-len 200000`, MNBT 2048, Froggeric v22.5 template. KLD-gated (mean KL 0.0131, top-1 97.92 % vs BF16); decode ~190 t/s live (gate bar 30.8); prefill is **slab-limited and pays for it** — 22,208 → 3,397, 24,000 → 3,290, 25,600 → **3,584** t/s at 43.5k single-stream, 3-way 197k aggregate 3,051 → **3,589** t/s, with a cliff at ~26 GiB (26,071 → 4,379). Cutover 2026-10-08 from fp8-native (payload on disk = rollback); 3-slot fp8-KV shape 2026-10-13, re-cut twice in the slab's favour (22208/6912 at 4 slots → 24000/5120 → 25600/5120); the 4-slot shape (22208/6912, `--max-num-seqs 4`) is the documented rollback. | 7770 |
|
||||
| Dankclaw | `frank-dragoon` | **halogen-flash-server 0.16.2** (`ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84…`, digest-pinned, plain `podman run` via `scripts/serve-frank-dragoon.sh`, root systemd unit) | Quat3rnion/halogen-swift1.5-qwen3.8-flash-next-v2-abliterated **`qwen38-flash-next-v2-swift15-abliterated.hgn`** (66,687,678,464 B = compacted r2 of the card's 89,775,868,032 B original, ~62 GiB resident; the v2 `.hgn` format loads in NO other engine), **+ REQUIRED `qwen38-flash-next-ngram.hgn`** (51,200,246,144 B, peonist @5cc17cea — read beside the checkpoint) **+ vision tower** (897,916,416 B, @e053f488 — **vision ON**, `image_url` parts only, ≥256 px probes), **MTP spec decode ALWAYS-ON (`HALOGEN_SPEC_ADAPT=0` — the adaptive default disables the drafter per request and HALVES decode)**, MTP head INSIDE the checkpoint (`HALOGEN_MTP_HEAD` unset; no sidecar fetched), **2 slots × 262,144 ctx** (KV pool 524,288 positions ≈ 14.0 GiB at ~28 KiB/position; the 27B slots>1-disables-drafting law does NOT carry over — batching and speculation coexist, the card's quickstart ships 2 with the head), harness window 217,088 = ceiling − 45k estimate headroom, **server-side sampler 1.0/0.95/20** (card recipe; models.yml extraBody matches), coupled cap/timeout 65536/7200, prompt cache ON by default (warm 100k turn ≈ 2 s vs ≈ 88 s cold)|7770|
|
||||
| Dankclaw | `frank-zealot` | llama.cpp master (0.5.0-dev, commit `5fc4f3c`), CUDA sm_86 build (root systemd unit, relinked) | HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive **Q6_K** (7,359,259,008 B; dense `qwen35` hybrid: 32 layers, 8 full-attention + 24 Gated DeltaNet, 4 KV heads × 256 dim, vocab 248,320, native 262,144, **no MTP tensors in the file**) + the publisher's own **BF16 mmproj** (921,704,448 B) for **vision**, on the RTX 3090, **TWO 200,704-token slots** (`--ctx-size 401408 --parallel 2`), **exact f16 K + f16 V** under flash attention (32 KiB/token over the 8 full layers = 12.25 GiB pool), batch 4096 / **ubatch 2048** (measured CUDA optimum), `--log-verbosity 4`, **MTP OFF** (forced: no `nextn` tensors exist). Priced from the MiMo Q6_K boots (same geometry, same card, same shape): 21,276/2,849 at ubatch 1024, 21,846/2,279 at 2048. Cutover 2026-10-07 (late) from Gemma4-26B-A4B Q4_K_M 1×200,704. All three endpoints route harness **compaction** here. | 7771 |
|
||||
|
||||
Retired endpoints (`frank-npu`, and every prior runtime) are in
|
||||
the [runbook index](#runbook-index). The full swap history lives in
|
||||
@@ -40,7 +41,11 @@ the [runbook index](#runbook-index). The full swap history lives in
|
||||
- `scripts/` — `deploy.sh` / `deploy-plan.py` / `deploy-remote.sh`, LLM serve
|
||||
scripts, systemd units, env tunables, bench harnesses
|
||||
- `old/` — retired LLM scripts (reference only)
|
||||
- `serverAppConfigs-docs/` — knowledge base (Obsidian vault, OKF format)
|
||||
- `serverAppConfigs-docs/` — knowledge base (Obsidian vault, OKF format).
|
||||
The managed skill library is distilled there one page per skill-cluster:
|
||||
`LLM/index.md` → *Cross-Cutting References*, plus `Operations/index.md`,
|
||||
`Agent/index.md`, and the Infrastructure/Dankclaw/FrankBook-Pro service
|
||||
pages (`serverAppConfigs-docs/log.md` carries the wave plan and results)
|
||||
- `tests/` — bats suite + Playwright e2e
|
||||
|
||||
## Git Workflow
|
||||
@@ -61,19 +66,27 @@ All edits flow through git. Never edit files on a remote machine directly.
|
||||
./scripts/deploy.sh dankclaw
|
||||
./scripts/deploy.sh dankserver
|
||||
./scripts/deploy.sh dankclaw --service searxng
|
||||
./scripts/deploy.sh dankserver --service frank-primary
|
||||
./scripts/deploy.sh dankserver --service frank-archon
|
||||
./scripts/deploy.sh dankclaw --dry-run
|
||||
```
|
||||
|
||||
The planner maps changed paths to services; the remote runner restarts only
|
||||
the selected service and records its revision outside the checkout.
|
||||
Documentation-only changes are a no-op. LLM services are selected only by
|
||||
their own files — every endpoint (`frank-large`/`frank-small`/`frank-primary`)
|
||||
their own files — every endpoint (`frank-dragoon`/`frank-zealot`/`frank-archon`)
|
||||
by its unit/env/serve trio in `scripts/` — so
|
||||
unrelated
|
||||
container changes cannot interrupt inference. Set `FRANK_SUDO_PW` for headless
|
||||
runs on a host without passwordless sudo; the value never enters the repo.
|
||||
|
||||
**`deploy.sh <target>` with no `--service` is a fleet-wide DRAIN, not a sync.**
|
||||
It also walks every per-service revision marker under the deploy-state dir and
|
||||
restarts each service whose marker is behind HEAD *and* whose paths changed
|
||||
since that marker — so a backlog accumulated by service-scoped deploys fires
|
||||
all at once, and one failing health gate aborts the run with every later
|
||||
service left pending. Name the service unless a drain is what you want
|
||||
(`./scripts/deploy.sh dankserver --service frank-archon`).
|
||||
|
||||
### Deployment completeness (MANDATORY)
|
||||
|
||||
A deploy is **not complete** when `deploy.sh` exits green. It requires:
|
||||
@@ -82,7 +95,14 @@ A deploy is **not complete** when `deploy.sh` exits green. It requires:
|
||||
2. the Playwright e2e suite (`tests/e2e/run.sh`) — runs **automatically** as
|
||||
the `[3/3]` gate at the end of `deploy.sh` against the LIVE fleet; a
|
||||
behavior-affecting deploy that fails it exits `DEPLOY INCOMPLETE`. Bypass
|
||||
only with `--skip-e2e` in an emergency.
|
||||
only with `--skip-e2e` in an emergency. The gate runs the conditional
|
||||
`frankbook` Playwright project (the `*arr`/qBittorrent walk,
|
||||
`tests/e2e/specs/19-frankbook.spec.ts`) ONLY when a frankbook blast-radius
|
||||
path changed since origin/main — that spec, `frankbook-pro/**`,
|
||||
`shared/homepage/services.json`, or the `dankclaw/Caddyfile` serving the
|
||||
arr vhosts — so a sleeping FrankBook-Pro never fails an unrelated deploy.
|
||||
The default project set excludes it everywhere else (`E2E_INCLUDE_FRANKBOOK=1`
|
||||
opts in; see `serverAppConfigs-docs/Architecture/E2E Suite.md`).
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
@@ -209,14 +229,17 @@ archive.
|
||||
**STOP the production service BEFORE benchmarking** — these servers serve
|
||||
real users, and a bench beside a live service starves the GPU and corrupts
|
||||
results. Use the runtime's own harness: `llama-bench` for the llama.cpp
|
||||
endpoints, `gufo bench` for frank-large. Rolling your own bench
|
||||
endpoints; a live llama.cpp server also reports honest per-request numbers in the `/completion` `timings` object (prefill/decode rates, `draft_n`, `draft_n_accepted`), which is how frank-dragoon and frank-zealot were measured. Rolling your own bench
|
||||
requires asking first.
|
||||
Scratch ports are not used on LLM hosts — a
|
||||
test process cannot share the single GPU anyway; validate against the service
|
||||
itself.
|
||||
|
||||
**`gufo bench` cannot rank speculative-decode settings, and will invert the
|
||||
ranking.** Measured 2026-09-28 on frank-large at depth 32768: its corpus
|
||||
(A gufo-era law; frank-dragoon ran ROCmFPX llama.cpp 2026-10-02 → 10-05, the
|
||||
halogen 27B container 2026-10-05 → 10-06, and is flash-server since, but the
|
||||
rollback tags keep it live.) **`gufo bench` cannot rank speculative-decode
|
||||
settings, and will invert the ranking.**
|
||||
Measured 2026-09-28 on frank-dragoon at depth 32768: its corpus
|
||||
reports `acceptance=0.933 ± 0.000` with an **identical trace sha256** across
|
||||
draft widths, i.e. the same highly-predictable text every run, and 93 % of
|
||||
every proposal accepted. It reported 29–37 t/s decode where real traffic
|
||||
@@ -241,12 +264,12 @@ one: `ip neigh show <ip>` from a peer — FAILED means off/NIC dead, while a
|
||||
merely wedged kernel usually still answers ARP.
|
||||
|
||||
1. Stop and mask (self-healing would restart it):
|
||||
- Dankclaw `frank-large` (root unit, gufo on the iGPU):
|
||||
`sudo systemctl stop frank-large && sudo systemctl mask frank-large`
|
||||
- Dankclaw `frank-small` (root unit, SGLang on the 3090):
|
||||
`sudo systemctl stop frank-small && sudo systemctl mask frank-small`
|
||||
- Dankserver `frank-primary` (root unit, tcclaviger vLLM on the 2x R9700):
|
||||
`sudo systemctl stop frank-primary && sudo systemctl mask frank-primary`
|
||||
- Dankclaw `frank-dragoon` (root unit, halogen podman on the iGPU):
|
||||
`sudo systemctl stop frank-dragoon && sudo systemctl mask frank-dragoon`
|
||||
- Dankclaw `frank-zealot` (root unit, llama.cpp on the 3090):
|
||||
`sudo systemctl stop frank-zealot && sudo systemctl mask frank-zealot`
|
||||
- Dankserver `frank-archon` (root unit, radiance on both R9700s, TP=2):
|
||||
`sudo systemctl stop frank-archon && sudo systemctl mask frank-archon`
|
||||
2. Bench.
|
||||
NOTE: `systemctl mask` FAILS on a relinked unit (the unit file in
|
||||
`/etc/systemd/system/` is itself a symlink into the checkout — mask
|
||||
@@ -263,9 +286,18 @@ deletes the unit file from `/etc/systemd/system/`, not just the symlink.
|
||||
The harness reaches the fleet through `.omp/agent/models.yml` in the
|
||||
operator's home — `/home/blank/.omp/agent/models.yml` on the dev box, which is
|
||||
where the harness runs; there is no copy on the servers. Each provider is one
|
||||
HOST:PORT; each model `id` is the served alias: `frank-primary` →
|
||||
`192.168.0.235:7770`, `frank-large` → `192.168.0.143:7770`, `frank-small` →
|
||||
`modelRoles`.
|
||||
HOST:PORT; each model `id` is the served alias. **Official model ids are
|
||||
the StarCraft unit names (2026-10-08), on the wire since 2026-10-09, and
|
||||
since 2026-10-10 they are also the SERVICE names:** `frank-archon` →
|
||||
`192.168.0.235:7770` (`FRANK_ARCHON_SERVED_NAME`), `frank-dragoon` →
|
||||
`192.168.0.143:7770` (`HALOGEN_MODEL_ID`), `frank-zealot` →
|
||||
`192.168.0.143:7771` (`ALIAS`). The 2026-10-10 cutover renamed the units,
|
||||
scripts, env files, containers, deploy services and runbooks to match —
|
||||
the three former service names are fully retired and exist nowhere in
|
||||
the repo; historical text in runbooks uses the current names. Verify a
|
||||
rename on `/v1/models`, never
|
||||
on a chat response: radiance echoes the **requested** model name, so a 200
|
||||
proves nothing about the served id (llama.cpp and halogen ignore it).
|
||||
|
||||
Resolution rules: the default model resolves as a `defaultProvider` +
|
||||
`defaultModel` pair (the provider must serve the model); a subagent override
|
||||
@@ -277,12 +309,22 @@ When renaming a service, retiring an endpoint, or moving a model: update the
|
||||
provider block, model `id`, `contextWindow`, `maxTokens`, `input` in
|
||||
`models.yml`, then fix every `modelRoles` entry that named the old endpoint —
|
||||
a role pointing at a provider the file does not define fails at call time. As
|
||||
of 2026-10-01 both `frank-small` and `frank-primary` accept images
|
||||
(`input: [text, image]`) — frank-primary loads the BF16 mmproj explicitly
|
||||
(this build never autoloads projectors), and frank-small's MiMo payload
|
||||
bakes its vision tower INTO the checkpoint (no projector file at all);
|
||||
`frank-large` remains text-only because its gufo deployment loads no
|
||||
vision projector. Note also that
|
||||
of 2026-10-06 **all three endpoints accept images** (`input: [text,
|
||||
image]`) — `frank-dragoon` returned to vision with the flash-server v2 swap
|
||||
(the v2 vision tower loads beside the checkpoint; the text-only halogen
|
||||
27B window closed 2026-10-06) — while
|
||||
`frank-archon` takes them natively (radiance runs the checkpoint's
|
||||
27-layer Qwen-VL tower — patching, pad-token expansion and 3-part rotary
|
||||
are the engine core; `--mm-max-patches` available),
|
||||
and `frank-zealot` through
|
||||
the publisher's own BF16 mmproj passed as `--mmproj` — the model is natively
|
||||
multimodal but the Q6_K GGUF carries no vision tensors (same arrangement the
|
||||
Gemma4 era used). `frank-zealot`'s `contextWindow` is
|
||||
**200704** — the PER-SLOT window (`--ctx-size 401408 --parallel 2` since the
|
||||
2026-10-07 Qwen3.5-9B swap), not the pool and not the model's native 262144:
|
||||
the harness must compact before llama.cpp answers a longer prompt with a 400.
|
||||
Note also
|
||||
that
|
||||
the reasoning channel differs by runtime: llama.cpp AND SGLang (qwen3
|
||||
parser, live-verified) emit `reasoning_content`, vLLM emits `reasoning` —
|
||||
models.yml sets `reasoningContentField` per endpoint to match.
|
||||
@@ -290,197 +332,309 @@ There is **no audio**
|
||||
input anywhere in the fleet: that capability left with the FLM NPU
|
||||
endpoint on 2026-09-11, returned 2026-09-17 with the Gemma4 projector's
|
||||
audio encoder, and left again with the Gemma4 retirement on 2026-10-01
|
||||
(the MiMo tower is vision-only).
|
||||
(neither the MiMo nor the Qwen3.5 tower carries audio).
|
||||
|
||||
### frank-large — gufo, Swift-1.5-Qwen3.8-27B-Uncensored UD-Q8_K_XL (Dankclaw, ACTIVE)
|
||||
### frank-dragoon — halogen-flash-server 0.16.2, Qwen3.8-Flash-Next v2 Swift1.5 ABLITERATED (Dankclaw, ACTIVE)
|
||||
|
||||
**gufo** (`gufo-org/gufo`) v0.1.1 — "the Strix Halo inference engine", a
|
||||
vertical engine built for exactly this hardware (Ryzen AI MAX+ 395 /
|
||||
Radeon 8060S, `gfx1151`) — as a **root systemd unit**
|
||||
(`scripts/frank-large.service`, `User=dank`), serving
|
||||
`ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF`'s `UD-Q8_K_XL`
|
||||
(27B dense, 29.3 GiB, sha256-verified; **abliterated/uncensored**, and under
|
||||
the *non-OSI* SWIFT OPEN LICENSE 1.0) on port 7770 from the gfx1151 iGPU.
|
||||
One session, full 262,144 ctx (the model's native window, `SESSIONS=1`),
|
||||
DFlash2 spec decode (the z-lab `Qwen3.8-27B-DFlash2-Q4_K_M` drafter, gufo's
|
||||
**adaptive** block policy, `--draft-tokens 7`), greedy sampling, reasoning
|
||||
extracted to `reasoning_content`, and **no vision projector**.
|
||||
The **flash-server engine** (`ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84…`,
|
||||
digest-pinned, published tag 0.16.2, 3.6 GB image) as a podman container
|
||||
driven by `scripts/serve-frank-dragoon.sh` under a **root systemd unit**
|
||||
(`scripts/frank-dragoon.service`, `User=dank`, `Restart=always`,
|
||||
`ExecStop=podman stop -t 60 frank-dragoon-halogen`, the script runs `podman
|
||||
run` FOREGROUND — the container's exit code is the service's exit so
|
||||
`Restart=always` stays real, and the engine ledger lands on disk) on port
|
||||
7770 from the Strix Halo iGPU. Engine stdout goes to
|
||||
`/home/dank/logs/frank_dragoon.log` — **the boot ledger (weights resident, KV
|
||||
pool, MTP head, prompt cache, any clamp) lives there, not in the journal**.
|
||||
Cutover 2026-10-06 from the halogen 27B-dense era
|
||||
([[frank-dragoon-halogen27b-dflash2]], retired the same window).
|
||||
|
||||
**Two swaps landed on 2026-09-28**, both same day: the engine (from the
|
||||
unsloth llama.cpp b11030 prebuilt, which had replaced Kairic Edge ROCmFPX)
|
||||
and then the payload (from `unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL`, still on
|
||||
disk and the payload-only rollback). The two targets are structurally
|
||||
interchangeable for gufo — same `qwen35` architecture, the same 866 tensors
|
||||
in the same order, the same per-tensor Q8_0/F32/BF16 layout, the same MTP
|
||||
head at `blk.64` — verified by parsing and diffing both GGUF headers before
|
||||
either was loaded. **The DFlash2 drafter did not move**, so it has its own
|
||||
`DRAFTER_DIR` and the deployer's stray-run sweep keys on the drafter's
|
||||
filename, not the target's.
|
||||
Serving `Quat3rnion/halogen-swift1.5-qwen3.8-flash-next-v2-abliterated`:
|
||||
`qwen38-flash-next-v2-swift15-abliterated.hgn` (66,687,678,464 B ≈ 62.1
|
||||
GiB, sha256 `9c880b8b…` — a compacted r2 of the card's 89,775,868,032 B
|
||||
append-only original; the abliteration workflow appends and the repo's
|
||||
tooling drops dead padding, live tensors byte-identical) + the **REQUIRED**
|
||||
n-gram lookup table `qwen38-flash-next-ngram.hgn` (51,200,246,144 B,
|
||||
peonist @5cc17cea) + the vision tower `qwen38-flash-next-vision.hgn`
|
||||
(897,916,416 B, @e053f488) + the flat `tokenizer/` directory (6 files,
|
||||
fetched fresh from peonist). Cutover 2026-10-06 from the halogen
|
||||
27B-dense shape, chosen by Frank for the ABLITERATED base.
|
||||
|
||||
**The runtime is built from source on the host** —
|
||||
`/home/dank/GitPackages/gufo-v0.1.1/`, reproducible with
|
||||
`scripts/build-gufo.sh` (`--verify` re-checks an installed tree). gufo ships
|
||||
no prebuilt Linux binaries and the v0.1.0/v0.1.1 release assets are empty.
|
||||
Unlike the retired llama.cpp prebuilt, which bundled its own ROCm, **gufo
|
||||
links the host ROCm 7.2.4 through its RPATH**, so the HIP dev packages must
|
||||
stay installed. Two host workarounds the build script encodes:
|
||||
**The format is the lock.** The v2 `.hgn` loads ONLY in the
|
||||
flash-server line — the 27B-era halogen 0.1.4 container refuses it, so the
|
||||
engine moved with the model. The MTP head is INSIDE the checkpoint
|
||||
(`HALOGEN_MTP_HEAD` stays unset; the separate `qwen38-flash-next-mtp.hgn`
|
||||
sidecar is deliberately not fetched).
|
||||
|
||||
- **Ubuntu's `librocwmma-dev` must be absent.** It installs a truncated
|
||||
rocWMMA header set into `/usr/include` that shadows AMD's complete one
|
||||
and breaks the build with `'internal/accessors.hpp' file not found`. A
|
||||
`-I` cannot fix it — in HIP mode the host pass searches the system
|
||||
directories first. The package has no reverse dependencies.
|
||||
- **`--gcc-install-dir` must name GCC 15.** Ubuntu 26.04 ships
|
||||
`/usr/include/c++/16`, which the ROCm clang prefers; its `<atomic_wait.h>`
|
||||
needs a `std::__detail::__notify_impl` symbol version the installed
|
||||
`libstdc++.so.6` (GLIBCXX_3.4.35) does not export, and the link fails.
|
||||
**Product facts vs the 27B era, both regressions FIXED:** the base is
|
||||
**abliterated** (the 27B was a censored-base finetune; refusals were an
|
||||
accepted trade there) and **vision is ON** (the 27B engine silently
|
||||
ignored its encoder — frank-zealot owned images for exactly one day at this
|
||||
endpoint).
|
||||
|
||||
`gufo diagnose` is the preflight and must report `[PASS]` with `gfx1151`.
|
||||
It also self-caps compute queues (`GPU_MAX_HW_QUEUES`, 2 resident for
|
||||
`serve llm`) after reading `/sys/class/kfd/kfd/proc/*/queues/*/type`, which
|
||||
matters because ComfyUI holds queues on the same device; four HIP processes
|
||||
is the device ceiling.
|
||||
**Engine laws that shape the config** (all in `scripts/frank-dragoon.env`
|
||||
with evidence): `HALOGEN_SPEC_ADAPT=0` is MANDATORY — the default adaptive
|
||||
policy disables the MTP head for whole requests after low acceptance and
|
||||
HALVES decode; the head stays on and temp-1.0 traffic pays some acceptance
|
||||
for it (accepted). `HALOGEN_CTX=262144` is the PER-SLOT hard ceiling (400
|
||||
naming the limit, never truncation) — RAISED from 200192 on 2026-10-07
|
||||
under the 400-zone law: the harness clamps max_tokens to
|
||||
`min(45056, contextWindow − est − 4096)`, its chars/4 est UNDER-counts
|
||||
the rendered prompt by ~3 % (measured: est 152,938 → server 157,388,
|
||||
request 43,158 vs room 42,804), so a harness window == engine ceiling
|
||||
made every request over ~146k est a guaranteed 400. `models.yml`
|
||||
`contextWindow: 217088` leaves the 45k gap that absorbs the estimate
|
||||
error (worst-case request ≈ 219k ≤ 262144); compaction fires at 85 % of
|
||||
the harness window = 184,524 est ≈ 190k measured (Frank's 180k target).
|
||||
`HALOGEN_KV_SLOTS=2` is LEGAL here — the 27B "slots>1 disables the drafter"
|
||||
law does NOT carry over; batching and speculation coexist (the card's
|
||||
quickstart ships 2 slots with the head), so 2 is concurrent capacity.
|
||||
`HALOGEN_KV_POOL_POSITIONS=524288` (explicit 2 × ctx; ~28 KiB/position ⇒
|
||||
14.0 GiB). `HALOGEN_MAX_TOK=8192` is only the per-request DEFAULT (the
|
||||
harness sends at most its `maxTokens` 45056); `HALOGEN_MAX_TOKENS_CAP=65536` +
|
||||
`HALOGEN_QUEUE_TIMEOUT=7200` are a coupled pair (FLAGS.md table).
|
||||
**A server-side sampler EXISTS** (`HALOGEN_TEMPERATURE/TOP_P/TOP_K` =
|
||||
1.0/0.95/20, the card's thinking recipe) — an absent temperature is no
|
||||
longer greedy; `models.yml` extraBody carries the same values, consistent
|
||||
not load-bearing. Prompt cache is ON by default (`PROMPT_CACHE=2`): a warm
|
||||
100k follow-up costs ~2 s vs ~88 s cold — the 27B engine's manual
|
||||
`HALOGEN_CACHE_MB` pin era is over. `response_format` is unsupported.
|
||||
Thinking lands in **`reasoning_content`**; the v2 chat template rejects
|
||||
`reasoning_effort` outside `xhigh|medium|low`, so the `models.yml` fold
|
||||
stays mandatory. Thinking tokens COUNT against the budget (empty content +
|
||||
`finish_reason: "length"` when exhausted mid-thought). Vision: OpenAI
|
||||
`image_url` content parts only; ≥256 px probe images or the model answers
|
||||
"video".
|
||||
|
||||
**The client contract changed, and the chat template is the sharp edge.**
|
||||
**gufo has no Jinja engine at all** — `src/models/qwen/chat_template.cpp` is
|
||||
587 lines of hand-written C++ that appends strings, with no
|
||||
`--chat-template-file`, no env override, and no template body to point at.
|
||||
`scripts/qwen-sharp-v22.4.0.jinja` therefore **cannot** be supplied to
|
||||
frank-large: that is a port, not a config line. Given the choice on
|
||||
2026-09-28 the decision was to ship on gufo's stock renderer and accept the
|
||||
loss — the `terse` system-prompt injection, the in-band `<|think_*|>`
|
||||
control, the `json|xml` `tool_call_format` switch, the tool-argument
|
||||
truncation and `add_vision_id` prefixes are all gone. gufo's own renderer
|
||||
does keep the effort instruction strings and `preserve_thinking`, which are
|
||||
what the harness depends on. **This is the thing to watch**: the endpoint
|
||||
now runs an *abliterated* model without the prompt that tells it to stop
|
||||
waffling. The cheap fix, if verbosity shows up, is to send the `terse` text
|
||||
as an ordinary system message from `models.yml` — not a template change.
|
||||
`reasoning_effort` accepts the full OpenAI set (all six levels verified
|
||||
200), so `models.yml`'s `reasoningEffortMap` fold is no longer *required* —
|
||||
it is kept deliberately, so that removing it does not silently move every
|
||||
`high` role from `xhigh` to `high` effort inside a swap. And there is no
|
||||
KV-quantization or prompt-cache-RAM flag; prompt reuse is on by default and
|
||||
`--cache-disk` is opt-in and off.
|
||||
**Memory ledger** (121.2 GiB unified; the referee is the boot ledger):
|
||||
weights ~62 + KV 14.0 + arena ~8 (`3.8 + 8.4 × MAX_TOK/16384`) + vision
|
||||
~2 ≈ 86 GiB engine-side + `HALOGEN_HOST_RESERVE_GIB=20` → ~15 GiB
|
||||
margin over baseline co-tenancy. The 51.2 GB n-gram table rides as mmap'd
|
||||
page cache beside the checkpoint — the card's quickstart arithmetic does
|
||||
not bill it as resident, but READ the boot ledger and re-size if the
|
||||
engine prints a clamp.
|
||||
|
||||
**Read the perf trade before assuming a win.** gufo's own Q8_K_XL tables
|
||||
(vs llama.cpp b11069, DFlash2) show prefill +37 % to +80 %, cold load 4.5×
|
||||
faster, decode +7 % at zero depth and +40 % at 4 users — but **−14 % at
|
||||
65,536 tokens of depth and −24 % at 131,072**. On the 8-bit target the
|
||||
crossover is ~32k of cached context, and frank-large runs one 262,144-token
|
||||
conversation, so long agent sessions sit on the losing side of that line.
|
||||
The Q4_K_XL target shows no such regression; revisiting the quantization is
|
||||
a separate decision from the runtime. Peak memory is 50.58 GiB at the full
|
||||
262,144 context against the iGPU's 114 GiB GTT aperture.
|
||||
Fetch: `scripts/download-frank-dragoon-flashnext.sh` into
|
||||
`/home/dank/models/halogen-flashnext-v2/` (`-f` + exact byte counts + LFS
|
||||
sha256 + `--verify-only`, no token on the command line; resume goes
|
||||
against the RESOLVED CDN URL because curl drops `Range` across HF's 302).
|
||||
The serve gates mirror the deployer EXACTLY (byte-exact checkpoint +
|
||||
n-gram + vision tower, flat tokenizer with all six files as regular
|
||||
files) — the 2026-10-01 crash-loop lesson.
|
||||
|
||||
The serve script (`scripts/serve-frank-large.sh`) validates runtime → target
|
||||
GGUF → drafter and exits with a pointer message if any is missing, then
|
||||
`exec`s `gufo serve --host 0.0.0.0 --port 7770 --sessions 1 llm --model
|
||||
…--served-model-name frank-large --context 262144 --speculative dflash2 …`.
|
||||
The deployer's `deploy_llm_large` gates on the **runtime binary** as well
|
||||
as both payload paths, and returns before it stops anything.
|
||||
Runbook:
|
||||
`serverAppConfigs-docs/LLM/frank-dragoon-flash-server-v2-swift15.md`.
|
||||
Rollback: revert the swap commit and redeploy — the 27B payload
|
||||
(`/home/dank/models/halogen-qwen3.8-27b/`), its fetch script and the
|
||||
halogen 0.1.4 image digest all remain on disk; the ROCmFPX door behind
|
||||
that (`scripts/build-rocmfpx.sh`, pin `36e9acd`) is untouched.
|
||||
|
||||
Fetch: `scripts/download-frank-large-swift.sh` (the target; per-file byte
|
||||
count + sha256 against the Hub LFS oid, no auth header — the repo is public
|
||||
and a token on the curl command line is world-readable) and
|
||||
`scripts/download-frank-large.sh` (the DFlash2 drafter, which did not move).
|
||||
Both take `--verify-only`.
|
||||
### frank-zealot — llama.cpp on the RTX 3090 (Dankclaw, ACTIVE)
|
||||
|
||||
Runbook: `serverAppConfigs-docs/LLM/frank-large-gufo-qwen38-27b.md`.
|
||||
Rollback, two ways: **payload only** — re-fetch with
|
||||
`scripts/download-frank-large.sh` and repoint `MODEL_FILE` at
|
||||
`/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-UD-Q8_K_XL.gguf`; that
|
||||
29.3 GiB target was **reclaimed on 2026-09-28** (staged to
|
||||
`~/.AGENTTRASH/20260928-retired-models/`, restorable from there), so it is
|
||||
no longer a one-minute flip. **Engine** —
|
||||
[[frank-large-unsloth-qwen38-27b-q8xl]], the b11030 prebuilt at
|
||||
`/home/dank/GitPackages/unsloth-llama-b11030/`; the only way to get
|
||||
Qwen-Sharp back. The DFlash2 drafter was never reclaimed and still lives in
|
||||
`Qwen3.8-27B-UD-Q8_K_XL/`.
|
||||
|
||||
### frank-small — SGLang on the RTX 3090 (Dankclaw, ACTIVE)
|
||||
|
||||
SGLang (latest) in a **self-contained python3.12 venv**
|
||||
(`/home/dank/GitPackages/sglang-cu128/`, torch 2.13.0+cu130) as a **root
|
||||
systemd unit** (repo `scripts/frank-small.service`, `relink_unit`ed into
|
||||
**llama.cpp master** (0.5.0-dev, commit `5fc4f3c`; CUDA sm_86 build
|
||||
`/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server`,
|
||||
`GGML_CUDA_FA_ALL_QUANTS=ON`) as a **root systemd unit**
|
||||
(`scripts/frank-zealot.service`, `relink_unit`ed into
|
||||
`/etc/systemd/system/`), serving
|
||||
`letechlead/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound` (9B dense
|
||||
qwen3_5 hybrid — 8 full-attention + 24 Gated DeltaNet layers — INT8 W8A16
|
||||
AutoRound, 12.0 GiB, vision tower baked into the checkpoint). Port 7771.
|
||||
`HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive` at **Q6_K**
|
||||
(7,359,259,008 B, sha256 `c0ba7beb…` computed locally — the Hub carries no
|
||||
LFS oid) **plus the publisher's own BF16 mmproj** (921,704,448 B), both
|
||||
`/home/dank/models/Qwen35-9B-HauhauCS-Q6K/`, on port 7771. Cutover
|
||||
2026-10-07 (late) from the Gemma4-26B-A4B Q4_K_M 1×200,704 shape (same day).
|
||||
|
||||
**The shape is memory-pinned, and the sharp edge is the mamba pool.** The
|
||||
DeltaNet ssm state pool autofills to a fixed ~5.1 GB budget *regardless of
|
||||
dtype* (fp32 at 110 slots, bf16 at 217 — same 5.1 GB), so both dials are
|
||||
pinned: `--mamba-ssm-dtype bfloat16 --max-mamba-cache-size 32` (795 MiB).
|
||||
The KV pool is then **pinned at 606,208 tokens** (`--max-total-tokens`) —
|
||||
the engine's autofit picks 687,388 and leaves 0.53 GB, which Triton's
|
||||
deferred kernel loads eat and the first long request OOMs on. Two
|
||||
262,144-token slots at the model's NATIVE window (2026-10-01 reslice from
|
||||
4 x 151,552: 2 x 262,144 = 524,288 leaves ~82k tokens of pool slack, and
|
||||
a compaction request — frank-primary's trigger is 183,501 tokens plus a
|
||||
32,768-token summary budget = 216,269 — now fits ONE request); a single
|
||||
request may grow to the whole pool. Prefill runs **eager**
|
||||
(`--disable-prefill-cuda-graph`) — there is no headroom for capture;
|
||||
decode graphs are on.
|
||||
**Architecture: qwen35 dense hybrid** — read from the GGUF's own header, not
|
||||
the model card: 32 layers, `full_attention_interval = 4` ⇒ **8 full-attention
|
||||
layers (3, 7, 11, …, 31) + 24 Gated DeltaNet linear layers**, 16 attention
|
||||
heads / **4 KV heads** × 256 dim, `ssm` conv_kernel 4 / state_size 128 /
|
||||
group_count 16 / time_step_rank 32 / inner_size 4096, native 262,144, vocab
|
||||
248,320. Tensor types are Q6_K × 250 + F32 × 177 — standard quants only, so
|
||||
the stock master build loads it (arch `qwen35` is registered in this build's
|
||||
`src/llama-arch.cpp`). **KV accounting:** the pool scales with the 8 full
|
||||
layers only — 8 × 4 × 256 = 8,192 elements/token per cache, so f16 K + f16 V
|
||||
bills **32 KiB/token** = 12.25 GiB at the 401,408 pool; the 24 DeltaNet
|
||||
layers bill a fixed recurrent state per SLOT, not per token, which is exactly
|
||||
why two 200k slots fit where the 26B MoE fit one.
|
||||
|
||||
**No speculative decode — a checkpoint fact.** `text_config` declares
|
||||
`mtp_num_hidden_layers: 1`, but the shipped safetensors (base BF16 and this
|
||||
INT8 conversion alike) carry zero `mtp.*` tensors; NEXTN would draft with
|
||||
an uninitialised 4.24 GiB ghost model at 0% acceptance. The seam gates
|
||||
that no `--speculative-*` flag reaches the engine.
|
||||
**The shape (2026-10-07):** `--ctx-size 401408 --parallel 2` = TWO
|
||||
conversations of 200,704 (Frank's cap; the harness `contextWindow` quotes
|
||||
the per-slot number), `--flash-attn on`, **f16 K + f16 V** (exact KV),
|
||||
batch 4096 / **ubatch 2048** (the measured CUDA optimum: +4.6 % prefill,
|
||||
decode flat, 616 MiB — MiMo-era A/B on this card), `--log-verbosity 4`,
|
||||
`--mmproj` for vision, **speculative decoding OFF** (`SPEC_TYPE=none`).
|
||||
|
||||
**The venv carries a CUDA 13.0 nvcc extracted from debs** (system nvcc is
|
||||
12.4) so the sgl-kernel/flashinfer JIT compiles for sm86; three local
|
||||
patches ride in the tree (GCC-16 shim, CCCL symlink, rsqrt noexcept).
|
||||
Details in the runbook.
|
||||
**Why MTP is off**: it is not a choice — **this Q6_K file carries no
|
||||
`nextn`/`mtp` tensors** (verified from the tensor-name list), and MTP is a
|
||||
property of the file, not of the flags. The publisher card advertises "MTP
|
||||
support" for the base architecture; the quantized export dropped the head.
|
||||
|
||||
Verified at deploy (2026-10-01): `/v1/models` reports `frank-small`;
|
||||
chat returns content plus `reasoning_content` (SGLang's qwen3 parser uses
|
||||
llama.cpp's field name); a red-square image answers "Red"
|
||||
(`image_tokens: 64`). At the 4 x 151,552 shape: 4 concurrent 54k-token
|
||||
requests answered; a 170,019-token prompt prefilled in 50.5 s (~3,370 t/s
|
||||
eager). At the 2 x 262,144 shape (reslice, same day): boot flags read
|
||||
`context_len=262144 max_running_requests=2 max_total_num_tokens=606208`;
|
||||
a 200,000-token prompt — well past frank-primary's 183,501 compaction
|
||||
trigger — prefilled and completed in ONE request (97.3 s wall, ~2,056
|
||||
tok/s eager); two concurrent 50k-token requests both answered. Decode
|
||||
**83.9 tok/s**.
|
||||
Runbook: `serverAppConfigs-docs/LLM/frank-small-sglang-mimo-v26.md`.
|
||||
Rollback: [[frank-small-gemma4-12b]] (payload and binary intact on disk).
|
||||
**Measured boots** (nvidia-smi, the referee): the deployed boot is
|
||||
**21,436 used / 2,689 free** (ubatch 2048) — the pricing boots were the
|
||||
MiMo-V2.6-Distill-Qwen-9B pair at this exact geometry (**21,276 / 2,849** at
|
||||
ubatch 1024, **21,846 / 2,279** at 2048), so the 415 MiB lighter weights
|
||||
bought back what the projection predicted. The engine ledger confirms the KV
|
||||
law to the MiB: `CUDA0 KV buffer size = 12544.00 MiB` (`K (f16): 6272.00`,
|
||||
`V (f16): 6272.00`), compute buffers 1,136 MiB on CUDA0 + 848 MiB host, the
|
||||
mmproj 879 MiB + 248 MiB. Decode is ubatch-invariant (Swift-era 3090 law). The
|
||||
Q6_K rung costs ~10 % decode vs Q5_K_M (weight bandwidth; MiMo-measured 94.4
|
||||
vs 104.8 t/s).
|
||||
|
||||
### frank-primary — tcclaviger vLLM, Qwen3.8-Flash-Next FP8 + MTP4 (Dankserver, ACTIVE)
|
||||
**Vision**: the publisher's own BF16 mmproj via `--mmproj` (the model is
|
||||
natively multimodal; the Q6_K file carries no vision tensors). Probe traps
|
||||
carry over: ≥256 px probe images or the model answers "video", and only the
|
||||
OpenAI `image_url` content part reaches the model.
|
||||
|
||||
The **tcclaviger RDNA4 vLLM fork** (`docker.io/tcclaviger/vllm@sha256:da4029666…`,
|
||||
digest-pinned) serving **`orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8`**
|
||||
(~172.8 GiB, 131 shards) on **both** Radeon AI PRO R9700s (gfx1201), **TP=2**,
|
||||
**MTP4** speculative decoding, **expert prefetch OFF**, 200,000 ctx x 3 seqs,
|
||||
**fp8 KV**. Deployed 2026-10-01 by rollback from commit `2339fae` after the
|
||||
GLM/DFlash2 experiment was declared failed (same day).
|
||||
**Compaction hub**: `models.yml` routes `compactionModel` for
|
||||
`frank-archon`, `frank-dragoon` and `frank-zealot` (the 3090 endpoint) here — 200,704 is the
|
||||
only window in the fleet that fits a worst-case compaction input (~170k
|
||||
tokens). With `--parallel 2` a compaction request no longer serializes behind
|
||||
a live generation (the Gemma4 era's `--parallel 1` law).
|
||||
|
||||
Measured 2026-09-29 tuning campaign: **decode 37.4-40.6 tok/s, acceptance
|
||||
~54 %** (MTP4 + prefetch off — the fleet's only endpoint that meets the
|
||||
35 tok/s target). 256k ctx arm measured 34.1-36.5; 2-way concurrency sums
|
||||
BELOW parity (25.49 vs 26.8 single) — slots are a capacity feature, not a
|
||||
throughput one. V-only q8 KV is NOT possible via config: stock
|
||||
`--kv-cache-dtype` is K+V together; a V-only quant needs a fork patch.
|
||||
Runbook:
|
||||
`serverAppConfigs-docs/LLM/frank-zealot-llama-qwen35-9b-q6k-2x200704.md`.
|
||||
Rollback: revert the swap commit — the Gemma4 payload
|
||||
(`/home/dank/models/Gemma4-26B-A4B/`), the MiMo Q6_K payload
|
||||
(`/home/dank/models/MiMo-Q6K/`), the Swift payload
|
||||
(`/home/dank/models/Swift15-UncIQ3XXS/`) and the Ornith Q8_0 payload
|
||||
(`/home/dank/models/Ornith-Q8/`) all remain on disk.
|
||||
|
||||
The engine auto-sets the attention block size (mamba-page floor); the
|
||||
expert plan's registry prices KV per token; the expert arena (6,574 slots
|
||||
x 2.344 MiB = 15.05 GiB/rank) is the decode-critical region — cutting it
|
||||
for KV headroom collapses the offload hit rate.
|
||||
|
||||
Unit and self-healing: root systemd unit (`scripts/frank-primary.service`,
|
||||
`User=dank`, `LimitMEMLOCK=infinity`), `ExecStop=podman stop -t 120
|
||||
frank-primary-glm`. The serve script drives `podman run` directly, gates
|
||||
the digest-pinned image, the payload's `config.json` and its exact
|
||||
131-shard count and the GPU device nodes, clears stale container
|
||||
identities, then blocks on `podman wait`. `deploy_llm_primary` gates the
|
||||
same, waits out :7770, relinks/enables/restarts the unit and health-gates
|
||||
with the MainPID check. Boot cost ~7 min warm; health returns 503 while
|
||||
loading.
|
||||
### frank-archon — Radiance (StillDeadcode), Swift1.5-Qwen3.8-Flash-Next GPTQ w4, TP=2 (Dankserver, ACTIVE)
|
||||
|
||||
Runbook: `serverAppConfigs-docs/LLM/dankserver-qwen38-fp8-expert-offload-tuning.md`.
|
||||
Retired GLM experiment: `serverAppConfigs-docs/LLM/frank-primary-glm53-flash-w4a16.md`.
|
||||
The **radiance engine** (StillDeadcode, Apache-2.0; image
|
||||
`stilldeadcode/radiance@sha256:4c3f484564…` digest-pinned, **pin 1.0.13**
|
||||
for serve/convert/KLD; `libr4d` gfx1201 kernels, plain `podman run`, root
|
||||
systemd unit, container name = unit name) serving
|
||||
**ukisai/Swift1.5-Qwen3.8-Flash-Next (BF16, 335.3 GiB, 131 shards,
|
||||
397/397 sha256-verified)** converted by `scripts/convert-swift15-w4dn8.sh`
|
||||
with recipe `scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe` to
|
||||
`qwen4exp-swift15-w4dn8-i8-hc8m.rad` (~114 GiB): **all 49,152 experts GPTQ
|
||||
w4** (act-order, damp 0.05, 10,000,000-token calibration captured on engine
|
||||
**1.2.0** via `RADIANCE_CALIB_DIR`/`RADIANCE_CALIB_TOKENS` — the only pin
|
||||
with the token knob; the RADGRAM2 grams are format-identical across pins and
|
||||
cross the boundary by design), **24 whole experts per layer cast bf16** (the
|
||||
PROTECTED set, derived from the calibration grams' energy share),
|
||||
**int8 trunk/lm_head**, **fp8 HC mixtures** (`--hc-mix-dtype`), PLE n-gram
|
||||
table at fp8 fixed scale (47.7 GiB in the container, from 95.4 GiB bf16).
|
||||
The plugin's **one-table-one-encoding law** (measured at rad-convert
|
||||
declare): one expert table takes ONE encoding; only whole bf16 experts are
|
||||
legal beside it — no w4/fp8 split of the down/gate_up pair (24-value
|
||||
`rad_arch_quant_desc` array, one per layer). KLD gate vs the BF16
|
||||
reference (recorded on the same 1.0.13 pin) is the precision referee;
|
||||
the throughput gate (decode ≥ 30.8 t/s) was passed at 4.5× the bar
|
||||
(122–173 t/s at 44.5k depth). Measured decode/prefill in the runbook.
|
||||
Rollback: revert the swap commit — the fp8-native `.rad`
|
||||
(`qwen4exp-fp8-native.rad`) and every prior payload remain on disk.
|
||||
|
||||
**Shape (both R9700s, TP=2)**: `--tp-wire wht6`, `--max-num-seqs 3`,
|
||||
`--max-model-len 200000` (the harness contextWindow must quote this),
|
||||
`--placement expert_tiered --host-pool-mib 44928` + **explicit VRAM
|
||||
partition `--vram-weights-mib 25600 --vram-kv-mib 5120`** (the vram-pair
|
||||
replaced `--expert-vs-cache-ratio`; the two are ONE budget —
|
||||
`weights + kv ≤ 30,969 MiB`, derived from the engine's own budget table: card
|
||||
32,624 MiB, the pair at 29,120 settling 2,058 MiB free against the engine's
|
||||
209.19 MiB floor (`--gpu-headroom-mib 96` + 113.19 static), so 1,849 MiB is
|
||||
claimable and the old 29,120 bound was my hand-picked 1 GiB reserve, not a
|
||||
hardware wall), `--kv-cache-dtype` **`fp8` — quantized KV, restored 2026-10-13
|
||||
because it is the only dtype that funds the slot count at this window**,
|
||||
`--num-speculative-tokens 2`, `--max-num-batched-tokens 2048`, prefix cache
|
||||
8 GiB pinned host + 64 GiB disk at `/home/dank/radiance-state`,
|
||||
**`--ngram-placement ram`**, Froggeric v22.5 chat template vendored
|
||||
(`scripts/qwen38-froggeric-v22.5.jinja`, `--override-chat-template`). Boot
|
||||
ledger (2026-10-13, pair 25600/5120): `kv cache 5.00 GiB … backs 3 sequences
|
||||
all at 200000 tokens at once, which is the worst case` — `kv_attn` page
|
||||
26.00 KiB (fp8) at 607,740 tokens (the sizing law predicted 608,027),
|
||||
`kv_qsa_bkey` 3.25 KiB at the same token count, `kv_gdn_state` 54 MiB × 6 slots
|
||||
(3 sequences × 2 copies) = 324 MiB; the sampler prints **9 rows**
|
||||
(3 × (1+2), inside the 25-row `wht6` exactness bound); the slab filled to
|
||||
563 KiB of slack with **17,548 slots** (16,235 at 24,000, 14,753 at 22,208 —
|
||||
every 1,792 MiB buys ~1,300 resident experts, and the pinned host pool hands
|
||||
the same experts back: 8,893 → 7,580 slots); `unclaimed 477.30 MiB`, cards
|
||||
settled ~336 MiB free against the engine's 209.19 MiB floor, KV lending
|
||||
4.22 GiB to the expert cache. Prefill tracks this slab only up to a
|
||||
**threshold**: 22,208 → 3,397 and 24,000 → 3,290 t/s at 43.5k are one band
|
||||
(±3 %), and the 25,600 arm measured **3,290** — the third step bought nothing
|
||||
(the 3,584 arm did not reproduce at 3 slots); the cliff is at ~26 GiB
|
||||
(26,071 → 4,379 on 2026-10-11's arm), so below that the slab is a threshold,
|
||||
not a slope.
|
||||
|
||||
**Decode ladder (2026-10-13, 512 tokens, greedy, this shape)**: 99.0 t/s
|
||||
single-stream at 43.5k, 93.3 at 197k — decode is nearly window-free (−6 % for
|
||||
4.5× depth); **3 concurrent streams give 113.9/43.3/10.9 t/s = 18.4 t/s
|
||||
aggregate**, so the slot count is paid in decode concurrency (a fused-step
|
||||
lottery), not in prefill; leg D of `probe-frank-archon-kv.py` measures it.
|
||||
|
||||
Shape notes carried from the fp8-native era and unchanged: wht6 gains,
|
||||
MNDT 2048 law (2560 collapses decode to 2.6–7.7 t/s), the 64 GiB disk
|
||||
prefix cache, and the rule that a prompt over `--max-model-len` is
|
||||
refused, never truncated. What changed: the fp8-native era's NVMe expert
|
||||
streaming (the 3.5 GB/s wall) is gone — the w4 resident set fits the
|
||||
host pool, so decode ran 122–173 t/s in the campaign arms and ~190 t/s
|
||||
on the fp8 shape (512-token streamed probe). The campaign also
|
||||
measured the int8-KV + spec-3 arm (136.0 t/s median at 44.5k depth,
|
||||
accept 0.7246).
|
||||
|
||||
**The KV dtype law (measured 2026-10-11, applied at 4 slots 2026-10-13)**:
|
||||
`--kv-cache-dtype` takes ONLY `bf16|fp8` — one dtype for the whole cache, so
|
||||
llama.cpp's exact-K/quantized-V pair (what frank-zealot runs) has NO radiance
|
||||
equivalent; the rung under bf16 is fp8 on both halves. Measured per rank from
|
||||
the two boot ledgers: `kv_attn` (13 layers, 4-token blocks) pages 26.00 → 52.00
|
||||
KiB, while `kv_qsa_bkey` (13 layers) stays 3.25 KiB at BOTH dtypes (the QSA
|
||||
block key is an index, not attention state) — so a token costs 7.3125 KiB at fp8
|
||||
and 13.8125 KiB at bf16: **1.89×, not 2×**. Pool sizing must also pay the
|
||||
dtype-independent terms that live in the same pool (~110 MiB a sequence of
|
||||
recurrent state + 448 MiB of prefix checkpoints):
|
||||
`(vram_kv − 110·seqs − 448) MiB / rate` — bf16 at 6,912 gives 462,946 where the
|
||||
engine reported 462,752 (0.04 % page rounding); fp8 at 6,912 with four slots
|
||||
gives 843,566, i.e. the deployed capacity contract 4 × 200,000. **That is why
|
||||
the dtype is fp8: the pool must back `seqs × max-model-len` as a hard limit, and
|
||||
bf16 at four slots costs 11,679 MiB — the slab would fall to 17,441 MiB (−33 %),
|
||||
where fp8 leaves it at 22,208.** The cost is paid twice on this engine, and
|
||||
half-refunded: idle KV VRAM is also the expert cache's spillover, and the bigger
|
||||
pool *lends* 6.08 GiB to the slab where the 3,543 MiB fp8 pool lent 2.69 GiB.
|
||||
`--max-num-seqs` rides 4 × (1 + spec) = 12 fused-all-reduce rows, under the
|
||||
25-row bound where wht6 is bit-exact (the "seqs 2 cannot boot" law belongs to
|
||||
the retired tcclaviger fork's torch.compile, not radiance). Disk-tier sizing
|
||||
law: the engine forces ~9.6 GiB of disk-tier weight buffers + arena onto the
|
||||
weights side, so `weights + kv` is capped near 29.6 GiB/card; boot with the
|
||||
pair, never `--kv-cache-size-mib`.
|
||||
|
||||
Shape notes carried from the fp8-native era and unchanged: wht6 gains,
|
||||
MNDT 2048 law (2560 collapses decode to 2.6–7.7 t/s), the 64 GiB disk
|
||||
prefix cache, and the rule that a prompt over `--max-model-len` is
|
||||
refused, never truncated. What changed: the fp8-native era's NVMe expert
|
||||
streaming (the 3.5 GB/s wall) is gone — the w4 resident set fits the
|
||||
host pool, so decode ran 122–173 t/s in the campaign arms and ~190 t/s
|
||||
on the fp8 shape (512-token streamed probe). The campaign also
|
||||
measured the int8-KV + spec-3 arm (136.0 t/s median at 44.5k depth,
|
||||
accept 0.7246).
|
||||
|
||||
**The pinned-RAM ledger is why this is a root unit**: 2 × 44 GiB expert
|
||||
pools + the 47.7 GiB n-gram copy (`--ngram-placement ram` is pinned RAM
|
||||
the card reads) + 8 GiB prefix host ≈ **147 GiB** charged against
|
||||
`RLIMIT_MEMLOCK` — the TTM floor is set just above it.
|
||||
`ttm.pages_limit`/`pages_max` cap GPU-mappable pinned pages, and **the knob is
|
||||
runtime state whose default is half of RAM**: `scripts/ttm-preflight.sh` (sourced
|
||||
by the serve script) therefore RAISES it to `FRANK_ARCHON_TTM_TARGET_MIB=204800`
|
||||
before refusing, because the 2026-10-12 power failure reset it to 125,988 MiB
|
||||
and the preflight re-refused 6,919 times. A manual sysfs write in the stop
|
||||
window is a convenience, not the repair; kernel cmdline persistence
|
||||
(`ttm.pages_limit=52428800`) is the belt to it.
|
||||
|
||||
**Contract facts** (source-verified): the HTTP listener binds AFTER the
|
||||
full load, so a bare `curl /health` gate is honest (`/health` == `/ping`,
|
||||
200 serving / 503 stopping); sampling params temperature…preserve_thinking
|
||||
accepted, **unknown params 400** (models.yml's extraBody set is fully
|
||||
inside it); `response_format` supported; `logprobs` REFUSED; reasoning in
|
||||
`reasoning_content` and tool calls in the qwen3.8 dialect via the auto
|
||||
chat format — no parser flags; vision native (the 27-layer Qwen-VL tower,
|
||||
`--mm-max-patches`); `/metrics` carries vLLM-compatible names +
|
||||
`radiance:*` families — the Dankclaw Prometheus scrape is re-added.
|
||||
|
||||
Runbook: `serverAppConfigs-docs/LLM/frank-archon-radiance-swift15-w4dn8.md`.
|
||||
Retired fp8-native era: `serverAppConfigs-docs/LLM/frank-archon-radiance-qwen38-native-fp8.md`.
|
||||
Retired FreeToken era: `serverAppConfigs-docs/LLM/frank-archon-freetoken-pr587.md`.
|
||||
Retired vLLM era: `serverAppConfigs-docs/LLM/dankserver-qwen38-fp8-expert-offload-tuning.md`.
|
||||
Retired GLM experiment: `serverAppConfigs-docs/LLM/frank-archon-glm53-flash-w4a16.md`.
|
||||
|
||||
## Key Lessons Learned
|
||||
Durable operational lessons from the fleet's history. Full detail in the
|
||||
@@ -642,19 +796,71 @@ linked runbooks.
|
||||
- Swap resize ordering: create and `swapon` the replacement BEFORE `swapoff`
|
||||
of the old one, or live pages must land in RAM that `MemAvailable`
|
||||
overstates. Dankclaw swap (64 GiB at `/swap.img`, fstab-driven) protects
|
||||
the co-tenants, not frank-large — pinned weights don't swap, and paging
|
||||
the co-tenants, not frank-dragoon — pinned weights don't swap, and paging
|
||||
the KV pool is worse than the OOM it prevents.
|
||||
- Host swap state is invisible to `deploy.sh` (no repo file) — check
|
||||
`swapon --show` before reasoning about headroom.
|
||||
- **A sysfs raise is a countdown to the next reboot.** `ttm.pages_limit` defaults
|
||||
to half of RAM, so the manual raise to 200 GiB (2026-10-05) vanished with the
|
||||
2026-10-12 23:17 POWER FAILURE on dankserver: the cap fell to 125,988 MiB,
|
||||
frank-archon's pinned-ledger preflight (floor 150,000 MiB) refused the start,
|
||||
and `Restart=always` re-ran that refusal **6,919 times in ~20 h** — a unit in
|
||||
`activating (auto-restart)` that exits 1 in 50 ms with no container and no
|
||||
engine log is a PREFLIGHT refusal, not an engine fault. Repo law now:
|
||||
`scripts/ttm-preflight.sh` (sourced by `serve-frank-archon.sh`) raises the knob
|
||||
to `FRANK_ARCHON_TTM_TARGET_MIB` itself, re-reads, and refuses by name only
|
||||
after a write the kernel rejected. Never prescribe a manual sysfs write as the
|
||||
recovery procedure for a service that can boot before a human reaches it.
|
||||
- **Silent RAM corruption outranks every software theory.** frank-archon's
|
||||
~40k-token output corruption (2026-10-04/05, repetition/garbage in long
|
||||
agent conversations) was marginal DDR5 at 4800 MT/s on four DIMMs, fixed
|
||||
by 4600. Consumer non-ECC memory raises **no MCE** for a silent flip, and
|
||||
a short in-band pattern check (60 s, 8 GiB of 246 GiB) is not a stability
|
||||
proof — combined, those two instruments said "ruled out" while the RAM
|
||||
was the cause. "4800 is the JEDEC floor on 5600-rated DIMMs" is a rating,
|
||||
not a margin guarantee. When the fault is data-shaped (NaN states,
|
||||
garbage tokens) and a software revert doesn't move it, price in the RAM
|
||||
before the runtime — and remember a revert tested under the same marginal
|
||||
hardware is itself confounded. See the runbook's 2026-10-05 section.
|
||||
|
||||
**Runtime-specific guards**
|
||||
|
||||
- **Rootless podman cannot RAISE rlimits, and `--ulimit nofile=1048576`
|
||||
dies at container create, not at runtime.** runc reports
|
||||
`error setting rlimit type 7: operation not permitted` (type 7 =
|
||||
RLIMIT_NOFILE) when the requested hard limit exceeds the host process's
|
||||
hard limit — on Dankclaw that ceiling is 524288. Inherited-equal limits
|
||||
are fine, so `memlock=-1:-1` works iff the unit grants
|
||||
`LimitMEMLOCK=infinity` (systemd is privileged; the value flows through
|
||||
podman to runc without any raise). Upstream halogen's own run lines pass
|
||||
NO `--ulimit` at all — copy vendor run lines before adding flags
|
||||
(measured 2026-10-05 on the frank-dragoon cutover).
|
||||
- **Kairic Edge** (if ever re-deployed): the pinned CK commit needs
|
||||
`third_party/composable_kernel/include/ck/config.h` generated manually
|
||||
(untracked; `git clean` removes it), and its greedy-argmax fastpath rejects
|
||||
non-greedy requests and reasoning budgets. Re-test chat before production.
|
||||
- **FastMTP sidecar**: build must be commit `4df29be4` + the HauhauCS patch;
|
||||
stock binaries fail the draft load with a tensor-shape mismatch.
|
||||
- **A GGUF's custom tensor type picks the runtime — read the histogram before
|
||||
promising a file to a loader.** `Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf` is
|
||||
ftype 115 with 311 tensors of type 103 (`GGML_TYPE_Q8_0_ROCMFPX`); stock
|
||||
llama.cpp and gufo refuse it at load, so ROCmFPX was not a preference but
|
||||
the only door. Counting tensor types out of the GGUF KV section takes one
|
||||
python read and settles a runtime argument in minutes. The build identity
|
||||
matters as much as the fork: the 2026-08-25 ROCmFPX builds SIGSEGV in
|
||||
`ggml_compute_forward_set_rows` on this same file, so
|
||||
`scripts/build-rocmfpx.sh` pins commit `36e9acd` by SHA.
|
||||
- **Speculative-decode flags move as a SET.** Handing the ROCmFPX fork
|
||||
`--model-draft` with no `--spec-type` (the naive way to build a "no MTP"
|
||||
bench arm) segfaulted at load, core dumped, no diagnostic (2026-10-02). A
|
||||
baseline arm must strip `--model-draft` AND every `--spec-draft-*` AND
|
||||
`--spec-type`, or it measures a crash instead of the absence of drafting.
|
||||
- **An orphaned scratch server taxes the whole host.** A bench child left
|
||||
alive by a client-side exception kept ~26 GiB of weights resident on a
|
||||
121 GiB box, pushed `available` to 3 GiB, and every later rate was suspect.
|
||||
Record the child pid and kill it in a `finally`, then prove `free` came back
|
||||
before believing a number. Kill by pid — never `pkill -f llama-server`,
|
||||
which also hits frank-zealot.
|
||||
|
||||
**llama.cpp serving**
|
||||
|
||||
@@ -664,19 +870,26 @@ linked runbooks.
|
||||
the MAIN model's weights (`shares_model`), so there is no second copy of a
|
||||
12 GiB model in VRAM and no `-md` argument. The plain build has no MTP
|
||||
tensors at all, and pointing `--model-draft` at the file would load it twice.
|
||||
- **What can be chained, and what cannot.** `--spec-type a,b` builds every
|
||||
enabled implementation and chains drafting through them, in a fixed internal
|
||||
order (ngram types first, then the drafters — CLI order is irrelevant). The
|
||||
gate is `ctx_dft`: **draftless** types (the ngram family) are added
|
||||
unconditionally, while `draft-mtp`, `draft-dflash`, `draft-dspark` and
|
||||
`draft-eagle3` all require a draft CONTEXT. So a chain may hold one
|
||||
context-backed drafter plus any number of ngram types — `draft-dflash,ngram-mod`
|
||||
is legal and measured (2.6x the drafter alone on prompt-echo traffic, byte-
|
||||
identical to it on prose at the fleet's 48/64 window) — but
|
||||
`draft-mtp,draft-dflash` dies at load: *"context type MTP requested but model
|
||||
doesn't contain MTP layers"*, because MTP builds its context against whatever
|
||||
model the draft context holds. Two context-backed types cannot share one
|
||||
process, and MTP's head only loads when MTP **is** the draft model.
|
||||
- **What can be chained, what cannot, and what a chain costs on a hybrid.**
|
||||
`--spec-type a,b` builds every enabled implementation and chains drafting
|
||||
through them, in a fixed internal order (ngram types first, then the
|
||||
drafters — CLI order is irrelevant). The gate is `ctx_dft`: **draftless**
|
||||
types (the ngram family) are added unconditionally, while `draft-mtp`,
|
||||
`draft-dflash`, `draft-dspark` and `draft-eagle3` all require a draft
|
||||
CONTEXT. So a chain may hold one context-backed drafter plus any number of
|
||||
ngram types — `draft-dflash,ngram-mod` is legal and measured (2.6x the
|
||||
drafter alone on prompt-echo traffic, byte-identical to it on prose at the
|
||||
fleet's 48/64 window) — but `draft-mtp,draft-dflash` dies at load: *"context
|
||||
type MTP requested but model doesn't contain MTP layers"*, because MTP
|
||||
builds its context against whatever model the draft context holds. Two
|
||||
context-backed types cannot share one process, and MTP's head only loads
|
||||
when MTP **is** the draft model. **A LEGAL chain can still lose on a
|
||||
recurrent model:** `draft-mtp,ngram-mod` on frank-dragoon's qwen35 hybrid
|
||||
measured −15 % prose / −7 % block-copy (2026-10-03) — adding the ngram layer
|
||||
logs `recurrent state rollback is not compatible with 'ngram-mod' -
|
||||
disabling rollback support`, and MTP verification depends on that rollback,
|
||||
so acceptance drops on ALL traffic even where the matcher fires. Measure
|
||||
chains per architecture, never by analogy.
|
||||
- **A drafter's KV scales with the TARGET's context, and its window policy
|
||||
decides how much that costs.** The draft context is built with
|
||||
`cparams.n_ctx = llama_n_ctx(ctx_tgt)` (`common/speculative.cpp`) — there is
|
||||
@@ -693,7 +906,8 @@ linked runbooks.
|
||||
trip moved the balance point FOUR layers (~4.3 GiB of headroom, 40:25 ->
|
||||
36:29), which is the practical size of the effect. See
|
||||
[[frank-medium-llamacpp-dflash2]].
|
||||
- **`--ubatch-size` has an OPTIMUM on this stack, not a plateau — and its
|
||||
- **`--ubatch-size` has an OPTIMUM on the 2x R9700 layer split, not a plateau —
|
||||
and the curve is BACKEND-SPECIFIC, so never carry it across (next bullet). Its
|
||||
compute buffers land about TWICE as heavy on the card that holds the output
|
||||
layers and the drafter.** Measured on frank-medium 2026-09-16 at the deployed
|
||||
shape (262144 over two slots, a 64k prefill measured at 64k depth,
|
||||
@@ -711,6 +925,19 @@ linked runbooks.
|
||||
its 2020 MiB pp buffer here under the DSpark drafter, when ROCm1 had 0.08 GiB
|
||||
free. That was a CONTIGUITY failure, not a total-bytes one, and no totals
|
||||
model — the sweep's or the seam's — can see it.)
|
||||
- **On the CUDA 3090 the same knob has NO interior optimum up to 2048, so
|
||||
frank-zealot ships 4096/2048.** Measured 2026-10-05 by boot-pair A/B on the
|
||||
live service: at 8,928 prompt tokens ubatch 1024 gives 3,972 / 4,077 t/s and
|
||||
ubatch 2048 gives 4,207 / 4,212 — **+4.6 %** — for **616 MiB** (free 2,849 →
|
||||
2,279). Decode did not move (91.9–93.0 vs 93.8–94.3, i.e. noise), and a
|
||||
279-token prompt, which is below either ubatch, came back identical — that is
|
||||
the mechanical proof the knob was the only variable, since a prompt shorter
|
||||
than the ubatch cannot see it. The logical `--batch-size` was again free. And
|
||||
the predecessor's "ubatch 2048 failed the scratch boot" was a shape
|
||||
observation, not a law: the same file boots 2048 clean at 2 × 200,704. Bench
|
||||
it by restarting the SERVICE through a transient `FRANK_ZEALOT_ENV_FILE`
|
||||
drop-in, never a scratch port — one card holds one server. See
|
||||
[[frank-zealot-llama-mimo-q6k-2x200704]].
|
||||
- **One `-ts` step is one LAYER, and a hybrid stack's layers are not the same
|
||||
size.** qwen35 is 16 x (3 Gated DeltaNet + 1 Gated Attention), so a step is
|
||||
worth ~0.43 GiB on a DeltaNet boundary and ~1.43 GiB on a full-attention one
|
||||
@@ -727,15 +954,50 @@ linked runbooks.
|
||||
host-RAM hash table) and its window decides which traffic it can see: too
|
||||
short and it proposes on ordinary prose fragments, which costs verification
|
||||
without acceptance.
|
||||
- **`--parallel` divides the pool, so MORE conversations cost WINDOW, and the
|
||||
per-slot number is the contract — not `ctx-size`.** `n_ctx_slot =
|
||||
CTX_SIZE / PARALLEL`, and llama.cpp refuses a prompt longer than the SLOT
|
||||
(`400 exceed_context_size_error: request (184001 tokens) exceeds the
|
||||
available context size (180224 tokens)`) even when the pool has room. So
|
||||
frank-zealot's 2 → 3 reslice moved each conversation from the model's native
|
||||
262,144 down to 180,224, and `models.yml` `contextWindow` had to follow or
|
||||
the harness would discover the ceiling as a 400 on a long conversation. Side
|
||||
effect worth knowing: the shorter slot made decode ~5 % FASTER (106.8 vs
|
||||
100.4 tok/s) — less attention to scan and one fewer DeltaNet state resident.
|
||||
The 2026-10-03 reslice went the other way (3 → 2 slots, 180,224 → 200,704,
|
||||
pool 401,408) to fund unquantized f16 V, and `contextWindow` followed again;
|
||||
decode landed 104.8 tok/s — the ~2 % off 106.8 is the exact-V tax. The same
|
||||
day the freed space also bought the Q6_K rung (now 94.4 tok/s — the rung
|
||||
itself is a further measured −10 %, weight-bandwidth, not slot geometry).
|
||||
All moves measured; both slot directions cost or buy window, never pool.
|
||||
See [[frank-zealot-llama-mimo-q6k-2x200704]].
|
||||
- **Price a VRAM budget from two boots, never from a formula — and an mmproj
|
||||
costs ~1.5x its file size.** The frank-zealot 3090 fit was solved from two
|
||||
measurements: 2 × 262,144 with no projector = 20,190 MiB used (KV 12,800 ⇒
|
||||
**non-KV 7,390**), 3 × 196,608 with the f16 projector = 22,972 used (KV
|
||||
14,113 ⇒ **non-KV 8,859**). The difference is the vision tower: a 918 MB file
|
||||
that bills ~1.4 GiB, because the encoder's activation and compute buffers
|
||||
ride with it. Subtracting the two rows is exact; extrapolating weights+KV+
|
||||
"a bit for buffers" is what leaves 1.1 GiB free when you meant 2.7. Then
|
||||
solve `KV ≤ card − non-KV − headroom` and round to a multiple of
|
||||
`PARALLEL × 4096`. Recorded in `scripts/frank-zealot.env` so the next resize
|
||||
starts from measurement.
|
||||
- **`images` and `image_url` are NOT interchangeable: only the content part
|
||||
reaches the model.** `--mmproj` loads, `usage.prompt_tokens` rises, and the
|
||||
model still invents an answer if the client used the legacy top-level
|
||||
`images` field on `/v1/chat/completions` — frank-zealot answered "Brown." for
|
||||
a solid-red PNG that way and "Red." through the OpenAI `image_url` part. The
|
||||
failure is silent because a hallucinated color is a plausible answer. Probe
|
||||
vision with a solid color and expect an exact word.
|
||||
- **`--jinja` is what `tools` needs.** On 2026-09-16 a subagent died with
|
||||
`500 tools param requires --jinja flag`, from the ik_llama.cpp runtime that
|
||||
then served frank-large: every tool-using agent on that endpoint was broken
|
||||
then served frank-dragoon: every tool-using agent on that endpoint was broken
|
||||
and the failure read like a model fault. Mainline llama.cpp 0.4.1 defaults
|
||||
`--jinja` on and the serve script pins it; halogen also accepts `tools` and
|
||||
returns `tool_calls`. Check tool support before routing a role at an endpoint.
|
||||
- **Quantized KV on CUDA needs `GGML_CUDA_FA_ALL_QUANTS=ON`** — the same
|
||||
option and the same ~12x prefill collapse described in the ROCm note above.
|
||||
f16 KV needs nothing; frank-small ran f16 for that reason until
|
||||
f16 KV needs nothing; frank-zealot ran f16 for that reason until
|
||||
2026-09-17, when `build-cuda` was rebuilt with the option ON so the
|
||||
endpoint could take q8_0 KV.
|
||||
- **llama-bench separates a LIST with `/` and SWEEP values with `,`.** Both
|
||||
@@ -753,7 +1015,7 @@ linked runbooks.
|
||||
raises on anything else — HTTP 500, not a clamp — so the harness's levels
|
||||
must be folded onto that set in `models.yml` (`compat.reasoningEffortMap`).
|
||||
A role whose level is unmapped fails on its FIRST request, and the failure
|
||||
names Jinja, not the model: every subagent on `frank-small` died with
|
||||
names Jinja, not the model: every subagent on `frank-zealot` died with
|
||||
`Unexpected reasoning effort high` until `high: xhigh` was added on
|
||||
2026-09-16. Halogen accepts the wider set, so the two entries legitimately
|
||||
differ.
|
||||
@@ -773,7 +1035,7 @@ linked runbooks.
|
||||
times (`RuntimeError: No HIP GPUs are available`).
|
||||
- SearXNG outbound traffic must stay on the Gluetun tunnel; do not move its
|
||||
ports off the gluetun service.
|
||||
- `amd_iommu=off` on Dankclaw (bought 13–16 % prefill for frank-large,
|
||||
- `amd_iommu=off` on Dankclaw (bought 13–16 % prefill for frank-dragoon,
|
||||
2026-09-11) permanently retired the XDNA2 NPU: the driver's SVA/PASID bind
|
||||
needs an enabled IOMMU (`/dev/accel/accel0` fails `err=-19`). Revert path
|
||||
in `serverAppConfigs-docs/LLM/dankclaw-iommu-translation-off.md`.
|
||||
@@ -827,9 +1089,9 @@ Active:
|
||||
|
||||
| Service | Runbook |
|
||||
|---------|---------|
|
||||
| frank-large (gufo Qwen3.8-27B UD-Q8_K_XL, Dankclaw) | `serverAppConfigs-docs/LLM/frank-large-gufo-qwen38-27b.md` |
|
||||
| frank-small (SGLang MiMo-V2.6 INT8, Dankclaw) | `serverAppConfigs-docs/LLM/frank-small-sglang-mimo-v26.md` |
|
||||
| frank-primary (tcclaviger vLLM, Qwen3.8-Flash-Next FP8 + MTP4, Dankserver) | `serverAppConfigs-docs/LLM/dankserver-qwen38-fp8-expert-offload-tuning.md` |
|
||||
| frank-dragoon (flash-server 0.16.2, abliterated Swift1.5 v2 .hgn + MTP always-on, 2×262,144, vision ON, Dankclaw) | `serverAppConfigs-docs/LLM/frank-dragoon-flash-server-v2-swift15.md` |
|
||||
| frank-zealot (llama.cpp CUDA, Qwen3.5-9B-Uncensored Q6_K, 2×200,704 exact f16 KV MTP-off, vision ON, Dankclaw) | `serverAppConfigs-docs/LLM/frank-zealot-llama-qwen35-9b-q6k-2x200704.md` |
|
||||
| frank-archon (radiance StillDeadcode, Qwen3.8-Flash-Next-FP8 native, 2×R9700 TP=2, Dankserver) | `serverAppConfigs-docs/LLM/frank-archon-radiance-qwen38-native-fp8.md` |
|
||||
| Dankclaw kernel posture (`amd_iommu=off`) | `serverAppConfigs-docs/LLM/dankclaw-iommu-translation-off.md` |
|
||||
| Deployment flow + incidents | `serverAppConfigs-docs/LLM/deployment.md` |
|
||||
|
||||
@@ -837,28 +1099,41 @@ Retired (rollback targets and history):
|
||||
|
||||
| Runtime | Host | Window | Runbook |
|
||||
|---------|------|--------|---------|
|
||||
|tcclaviger vLLM GLM-5.3-Flash W4A16 int4, +/- DFlash2 drafter (`frank-primary`)|Dankserver|2026-09-30 → 10-01|`frank-primary-glm53-flash-w4a16.md`|
|
||||
| llama.cpp b11030 Qwen3.8-27B UD-Q8_K_XL + DFlash2 (`frank-large`) | Dankclaw | 2026-09-21 → 09-28 | `frank-large-unsloth-qwen38-27b-q8xl.md` |
|
||||
| R9V radiance-vLLM Qwen3.8-Flash-Next-Uncensored IQ4_XS (`frank-primary`) | Dankserver | 2026-09-27 (same-day swap to Q8_0) | `frank-primary-r9v-radiance-qwen38-uncensored.md` |
|
||||
| llama.cpp b11030 Qwen3.8-Flash-Next Q8_0 + shared MTP (`frank-primary`) | Dankserver | 2026-09-26 → 09-27 | `frank-primary-llamacpp-qwen38-flash-next-q8.md` |
|
||||
| ik_llama.cpp Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M (`frank-large`) | Dankclaw | 2026-09-15 → 09-16 | `frank-large-ik-uncensored.md` |
|
||||
| llama.cpp CUDA Gemma4-26B-A4B QAT Q4_K_M, 1×200,704 f16-K/q8_0-V MTP-off (`frank-zealot`) | Dankclaw | 2026-10-07 → 10-07 (same-day swap; rollback target — payload on disk; owns the `attention_k_eq_v` pair-billing KV law and the gemma4-assistant drafter-cost boots) | `frank-zealot-llama-gemma4-q4km-200704.md` |
|
||||
| ROCmFPX llama.cpp fork Qwen3.8-27B Uncensored Q6_0_ROCMFPX_AGENT + MTP (`frank-dragoon`) | Dankclaw | 2026-10-03 → 10-05 (rollback target, payload + fork build on disk) | `frank-dragoon-rocmfpx-q6agent-uncensored-mtp.md` |
|
||||
| tcclaviger vLLM fork Qwen3.8-Flash-Next-Uncensored FP8, TP=2, shape G (`frank-archon`) | Dankserver | 2026-09-29 → 10-05 (rollback target — tag `frank-archon-vllm-shapeG-20261003`; trio + env lore in the tag, image in rootless storage) | `dankserver-qwen38-fp8-expert-offload-tuning.md` |
|
||||
| FreeToken PR #587 ROCm, Qwen3.8-Flash-Next-Uncensored-FP8, ONE R9700 (`frank-archon`) | Dankserver | 2026-10-05 → 10-05 (rollback target — tag `frank-archon-freetoken-pr587-20261005`; trio + Dockerfile in the tag, image `localhost/frank-archon-freetoken:pr587-77a7201` and orcarouter payload on disk) | `frank-archon-freetoken-pr587.md` |
|
||||
| llama.cpp master MiMo-V2.6-Distill-Qwen-9B Q6_K 2×200,704 f16 KV (`frank-zealot`) | Dankclaw | 2026-10-03 → 10-05 (rollback target — payload on disk; owns the ubatch-2048 A/B; the Ornith swap of 10-05 was committed but never deployed, so this was the LIVE payload Swift replaced) | `frank-zealot-llama-mimo-q6k-2x200704.md` |
|
||||
| llama.cpp Ornith-1.5-9B Q8_0, ONE 262,144 f16 KV (`frank-zealot`) | Dankclaw | 2026-10-05 (committed 77decb5→380aafa, never deployed — superseded by Swift the same evening; payload on disk) | trail in `frank-zealot-llama-swift15-unc-iq3xxs-mtp-114688.md` |
|
||||
| llama.cpp CUDA Swift-1.5-27B Uncensored RCO IQ3_XXS-mtp, 2×114,688 draft-mtp q8_0 KV (`frank-zealot`) | Dankclaw | 2026-10-06 → 2026-10-07 (rollback target — payload on disk; owns the 3090 ubatch ladder A/B and the 34 KiB/token qwen35 KV law) | `frank-zealot-llama-swift15-unc-iq3xxs-mtp-114688.md` |
|
||||
| ROCmFPX llama.cpp fork Qwen3.8-27B Q8_0_ROCMFPX_AGENT + MTP (`frank-dragoon`) | Dankclaw | 2026-10-02 → 10-03 (rollback target, payload on disk) | `frank-dragoon-rocmfpx-q8agent-mtp.md` |
|
||||
| gufo v0.1.1 Qwen3.8-27B UD-Q8_K_XL/UD-Q4_K_XL + DFlash2 (`frank-dragoon`) | Dankclaw | 2026-09-28 → 10-01, again 10-01 → 10-02 (payloads removed from disk 2026-10-03; rollback = re-download) | `frank-dragoon-gufo-qwen38-27b.md` |
|
||||
| vLLM TheRock MiMo-V2.6-Distill-Qwen-9B BF16 + cross-model MTP (`frank-dragoon`) | Dankclaw | 2026-10-01 (same-day rollback to gufo) | `frank-dragoon-vllm-mimo-v26-bf16.md` |
|
||||
| llama.cpp master MiMo-V2.6-Distill-Qwen-9B Q6_K (`frank-zealot`) | Dankclaw | 2026-10-01 → 10-02 (the rung returned ACTIVE 2026-10-03 at the 2×200,704 shape — `frank-zealot-llama-mimo-q6k-2x200704.md`) | `frank-zealot-llama-mimo-q6k.md` |
|
||||
| llama.cpp master MiMo-V2.6-Distill-Qwen-9B Q5_K_M 2×200,704 f16 KV (`frank-zealot`) | Dankclaw | 2026-10-02 → 10-03 (rollback target — one `MODEL_FILE` line, buys back 875 MiB + the −10 % rung tax; owns the boot-pair table) | `frank-zealot-llama-mimo-q5km.md` |
|
||||
| SGLang MiMo-V2.6 INT8-W8A16-AutoRound (`frank-zealot`) | Dankclaw | 2026-10-01 (same-day swap to the GGUF path) | `frank-zealot-sglang-mimo-v26.md` |
|
||||
|tcclaviger vLLM GLM-5.3-Flash W4A16 int4, +/- DFlash2 drafter (`frank-archon`)|Dankserver|2026-09-30 → 10-01|`frank-archon-glm53-flash-w4a16.md`|
|
||||
| llama.cpp b11030 Qwen3.8-27B UD-Q8_K_XL + DFlash2 (`frank-dragoon`) | Dankclaw | 2026-09-21 → 09-28 | `frank-dragoon-unsloth-qwen38-27b-q8xl.md` |
|
||||
| R9V radiance-vLLM Qwen3.8-Flash-Next-Uncensored IQ4_XS (`frank-archon`) | Dankserver | 2026-09-27 (same-day swap to Q8_0) | `frank-archon-r9v-radiance-qwen38-uncensored.md` |
|
||||
| llama.cpp b11030 Qwen3.8-Flash-Next Q8_0 + shared MTP (`frank-archon`) | Dankserver | 2026-09-26 → 09-27 | `frank-archon-llamacpp-qwen38-flash-next-q8.md` |
|
||||
| ik_llama.cpp Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M (`frank-dragoon`) | Dankclaw | 2026-09-15 → 09-16 | `frank-dragoon-ik-uncensored.md` |
|
||||
| vllm-radiance TP=2 MXFP4 + DFlash2 (`frank-medium`) | Dankserver | 2026-09-20 → 09-26 | `frank-medium-radiance-mxfp4.md` |
|
||||
| Lucebox DS4.1 DeepSeek-V4.1-Flash ROCmFP23 (`frank-primary`) | Dankserver | 2026-09-26 (same-day swap) | `frank-primary-lucebox-ds4.md` |
|
||||
| Lucebox DS4.1 DeepSeek-V4.1-Flash ROCmFP23 (`frank-archon`) | Dankserver | 2026-09-26 (same-day swap) | `frank-archon-lucebox-ds4.md` |
|
||||
| kingjones STRIX Qwen3.8-Flash-Next ROCmFP4 | Dankclaw | 2026-08-30 → 09-11 | `kingjones-strix-rocmfp4.md` |
|
||||
| halogen-flash-server Qwen3.8-Flash-Next w4b (`frank-large`) | Dankclaw | 2026-09-16 → 09-19 | `halogen-flash-server.md` |
|
||||
| CIRU v4.4.1 Orca Qwen3.8-Flash (`frank-large`) | Dankclaw | 2026-09-19 → 09-21 | `frank-large-ciru-orca.md` |
|
||||
| unsloth llama.cpp b11030 Qwen3.8-Flash-Next UD-IQ4_XS (`frank-large`) | Dankclaw | 2026-09-21 (same-day swap) | `frank-large-unsloth-qwen38-flash-next.md` |
|
||||
| Kairic Edge Qwen3.8-27B IU4 W4A4 (`frank-large`) | Dankclaw | 2026-09-21 (same-day swap) | `frank-large-kairic-edge-qwen38-27b.md` |
|
||||
| ISTA-DASLab Qwen3.8-27B-GSQ-RCO IQ3_S (`frank-small`) | Dankclaw | 2026-09-16 → 09-17 | `frank-small-qwen38-gsq-rco.md` |
|
||||
| vLLM Gemma4-12B w4a16 (`frank-small`) | Dankclaw | 2026-09-19 (same-day revert) | `frank-small-vllm-gemma4.md` |
|
||||
| llama.cpp Gemma4-12B QAT Q4_K_M (`frank-small`) | Dankclaw | 2026-09-17 → 10-01 | `frank-small-gemma4-12b.md` |
|
||||
| halogen-flash-server Qwen3.8-Flash-Next w4b (`frank-dragoon`) | Dankclaw | 2026-09-16 → 09-19 | `halogen-flash-server.md` |
|
||||
| CIRU v4.4.1 Orca Qwen3.8-Flash (`frank-dragoon`) | Dankclaw | 2026-09-19 → 09-21 | `frank-dragoon-ciru-orca.md` |
|
||||
| unsloth llama.cpp b11030 Qwen3.8-Flash-Next UD-IQ4_XS (`frank-dragoon`) | Dankclaw | 2026-09-21 (same-day swap) | `frank-dragoon-unsloth-qwen38-flash-next.md` |
|
||||
| Kairic Edge Qwen3.8-27B IU4 W4A4 (`frank-dragoon`) | Dankclaw | 2026-09-21 (same-day swap) | `frank-dragoon-kairic-edge-qwen38-27b.md` |
|
||||
| ISTA-DASLab Qwen3.8-27B-GSQ-RCO IQ3_S (`frank-zealot`) | Dankclaw | 2026-09-16 → 09-17 | `frank-zealot-qwen38-gsq-rco.md` |
|
||||
| vLLM Gemma4-12B w4a16 (`frank-zealot`) | Dankclaw | 2026-09-19 (same-day revert) | `frank-zealot-vllm-gemma4.md` |
|
||||
| llama.cpp Gemma4-12B QAT Q4_K_M (`frank-zealot`) | Dankclaw | 2026-09-17 → 10-01 | `frank-zealot-gemma4-12b.md` |
|
||||
| agentionai ROCmFP4-FAST imatrix | Dankclaw | 2026-08-29 → 08-30 | `agentionai-fast-imatrix.md` |
|
||||
| eemin Qwen3.8-27B ROCmI4 W4A4 MTP | Dankclaw | 2026-08-18 → 08-25 | `eemin-rocmi4-mtp.md` |
|
||||
| Kairic Edge IU4 | Dankclaw | 2026-08-22 (same-day revert) | `kairic-edge.md` |
|
||||
| CIRU ActiveFPX v3 / PromptForge | Dankclaw | through 2026-08-22 | `ciru-activefpx-promptforge.md` |
|
||||
| DFlash2 Vulkan | Dankclaw | 2026-08-21 (same-day revert) | `dflash2-large-vulkan.md` |
|
||||
| FastFlowLM NPU (`frank-npu`) | Dankclaw | twice, retired 2026-09-11 | `frank-npu.md`, `FastFlowLM NPU Inference.md` |
|
||||
| FreeToken (official + Huihui FP8, `frank-small`) | Dankserver | 2026-09-02 → 09-15 | `freetoken-frank-small.md` |
|
||||
| FreeToken (official + Huihui FP8, `frank-zealot`) | Dankserver | 2026-09-02 → 09-15 | `freetoken-frank-small.md` |
|
||||
| FastMTP patched llama.cpp | Dankserver | retired 2026-08-18 | `qwen3.8-spec-matrix.md` |
|
||||
| ik_llama.cpp Ornith CRACK | Dankserver | retired 2026-09-02 | `ikllama-ornith-dankserver.md` |
|
||||
| llama.cpp layer split + DFlash2 (`frank-medium`) | Dankserver | 2026-09-16 → 09-20 | `frank-medium-llamacpp-dflash2.md` |
|
||||
|
||||
@@ -13,7 +13,7 @@ one deploy script.
|
||||
| Host | User | IP | Podman stack | LLM service | LLM port |
|
||||
|------|------|----|--------------|-------------|----------|
|
||||
| dankserver | dank | 192.168.0.235 | media and library | frank-medium (ROCmFPX llama.cpp, Qwen3.8-27B AGENT) | 7770 |
|
||||
| dankclaw | dank | 192.168.0.143 | AI, search, development, observability | frank-large (gufo, Qwen3.8-27B UD-Q8_K_XL) | 7770 |
|
||||
| dankclaw | dank | 192.168.0.143 | AI, search, development, observability | frank-dragoon (gufo, Qwen3.8-27B UD-Q8_K_XL) | 7770 |
|
||||
|
||||
## Active Services
|
||||
|
||||
@@ -30,7 +30,7 @@ one deploy script.
|
||||
| Gitea | dankclaw | gitea.dankclaw.local | 3001 |
|
||||
| Grafana | dankclaw | grafana.dankclaw.local | 3002 |
|
||||
| Prometheus | dankclaw | prometheus.dankclaw.local | 9090 |
|
||||
| frank-large | dankclaw | — (bare endpoint) | 7770 |
|
||||
| frank-dragoon | dankclaw | — (bare endpoint) | 7770 |
|
||||
|
||||
## Structure
|
||||
|
||||
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
# autoresearch.sh — canonical frank-archon benchmark entry point (dev box).
|
||||
#
|
||||
# The service runs on dankserver (192.168.0.235:7770), the two R9700s live
|
||||
# there, and the benchmark must own those cards, so the whole stop -> boot
|
||||
# scratch -> bench -> restore sequence runs ON dankserver (see
|
||||
# scripts/autoresearch-frank-archon.sh). This wrapper:
|
||||
# 1. copies the candidate config (the repo's scripts/frank-archon.env, the
|
||||
# file the autoresearch loop edits) and the bench script to dankserver,
|
||||
# 2. runs the bench over ssh, relaying its stdout (the METRIC lines) and
|
||||
# exit code back here verbatim.
|
||||
# It deliberately does NOT push to origin: the scp dirties dankserver's
|
||||
# working tree, and the loop's uncommitted candidate config must not be
|
||||
# pushed. Only this file and scripts/autoresearch-frank-archon.sh are
|
||||
# committed; the candidate env rides over ssh each run.
|
||||
set -uo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REMOTE=dankserver
|
||||
REMOTE_REPO=/home/dank/serverAppConfigs
|
||||
|
||||
scp -q "$DIR/scripts/frank-archon.env" \
|
||||
"$REMOTE:$REMOTE_REPO/scripts/frank-archon.env" || { echo "scp env failed" >&2; exit 1; }
|
||||
scp -q "$DIR/scripts/autoresearch-frank-archon.sh" \
|
||||
"$REMOTE:$REMOTE_REPO/scripts/autoresearch-frank-archon.sh" || { echo "scp script failed" >&2; exit 1; }
|
||||
|
||||
ssh "$REMOTE" "bash $REMOTE_REPO/scripts/autoresearch-frank-archon.sh $REMOTE_REPO/scripts/frank-archon.env"
|
||||
+10
-1
@@ -337,7 +337,16 @@ http://:8017 {
|
||||
try_files /docs.html
|
||||
file_server
|
||||
}
|
||||
@data path /services.json /status.json /app.js /style.css /deployed.js
|
||||
# /whoami echoes the outpost-injected identity headers so the page
|
||||
# can gate the admin button client-side (the /admin* 403 above stays
|
||||
# the real gate; this only decides what is shown). The outpost strips
|
||||
# client-supplied X-Authentik-* before injecting, so this is spoof-proof.
|
||||
handle /whoami {
|
||||
templates
|
||||
header Content-Type application/json
|
||||
respond "{\"user\":\"{header.X-Authentik-Username}\",\"groups\":\"{header.X-Authentik-Groups}\"}"
|
||||
}
|
||||
@data path /services.json /status.json /app.js /style.css /deployed.js /img/* /sfx/*
|
||||
handle @data {
|
||||
file_server
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"3": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"_meta": { "title": "Positive prompt" },
|
||||
"inputs": {
|
||||
"text": "A detailed illustration of a 1930s Soviet industrial design bureau at night: a well-measured engineer at a drafting table, a warm desk lamp, rolled blueprints, gears and calipers on the desk, deep red and cream palette, gouache painterly style, intricate mechanical detail, soft shadows",
|
||||
"clip": [ "6", 0 ]
|
||||
}
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"_meta": { "title": "Negative prompt" },
|
||||
"inputs": {
|
||||
"text": "lowres, blurry, deformed, extra limbs, watermark, signature, jpeg artifacts",
|
||||
"clip": [ "6", 0 ]
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"class_type": "UNETLoader",
|
||||
"_meta": { "title": "Qwen Image 2.1 (int8)" },
|
||||
"inputs": { "unet_name": "qwen_image_2.1_int8_convrot.safetensors", "weight_dtype": "default" }
|
||||
},
|
||||
"6": {
|
||||
"class_type": "CLIPLoader",
|
||||
"_meta": { "title": "Qwen3-VL 8B text encoder (w4a8)" },
|
||||
"inputs": { "clip_name": "qwen3vl_8b_w4a8.safetensors", "type": "qwen_image" }
|
||||
},
|
||||
"7": {
|
||||
"class_type": "VAELoader",
|
||||
"_meta": { "title": "VAE" },
|
||||
"inputs": { "vae_name": "qwen_image_2.1_vae_bf16.safetensors" }
|
||||
},
|
||||
"8": {
|
||||
"class_type": "EmptySD3LatentImage",
|
||||
"_meta": { "title": "Empty Latent 1024x1024" },
|
||||
"inputs": { "width": 1024, "height": 1024, "batch_size": 1 }
|
||||
},
|
||||
"9": {
|
||||
"class_type": "KSampler",
|
||||
"_meta": { "title": "KSampler" },
|
||||
"inputs": {
|
||||
"model": [ "5", 0 ],
|
||||
"seed": 1930,
|
||||
"steps": 20,
|
||||
"cfg": 1.0,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"positive": [ "3", 0 ],
|
||||
"negative": [ "4", 0 ],
|
||||
"latent_image": [ "8", 0 ],
|
||||
"denoise": 1.0
|
||||
}
|
||||
},
|
||||
"10": {
|
||||
"class_type": "VAEDecode",
|
||||
"_meta": { "title": "VAE Decode" },
|
||||
"inputs": { "samples": [ "9", 0 ], "vae": [ "7", 0 ] }
|
||||
},
|
||||
"11": {
|
||||
"class_type": "SaveImage",
|
||||
"_meta": { "title": "Save Image (art)" },
|
||||
"inputs": { "filename_prefix": "qwen21-art", "images": [ "10", 0 ] }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"3": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"_meta": { "title": "Positive prompt (legible text)" },
|
||||
"inputs": {
|
||||
"text": "Clean minimalist movie-poster design, dark charcoal background with a subtle vintage grain. Centered large bold white sans-serif text reading \"SOLIDARITY OF THE SOVIET COMPUTER SYSTEMS\", perfectly legible, crisp letterforms, even letter spacing. Below it, a smaller line of yellow text reading \"DANKCLAW DANKSERVER EST 2026\", equally legible. Flat graphic design, high contrast, generous margins, no other words, no watermark",
|
||||
"clip": [ "6", 0 ]
|
||||
}
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"_meta": { "title": "Negative prompt" },
|
||||
"inputs": {
|
||||
"text": "blurry, illegible, misspelled, warped letters, extra text, watermark, signature, lowres, jpeg artifacts",
|
||||
"clip": [ "6", 0 ]
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"class_type": "UNETLoader",
|
||||
"_meta": { "title": "Qwen Image 2.1 (int8)" },
|
||||
"inputs": { "unet_name": "qwen_image_2.1_int8_convrot.safetensors", "weight_dtype": "default" }
|
||||
},
|
||||
"6": {
|
||||
"class_type": "CLIPLoader",
|
||||
"_meta": { "title": "Qwen3-VL 8B text encoder (w4a8)" },
|
||||
"inputs": { "clip_name": "qwen3vl_8b_w4a8.safetensors", "type": "qwen_image" }
|
||||
},
|
||||
"7": {
|
||||
"class_type": "VAELoader",
|
||||
"_meta": { "title": "VAE" },
|
||||
"inputs": { "vae_name": "qwen_image_2.1_vae_bf16.safetensors" }
|
||||
},
|
||||
"8": {
|
||||
"class_type": "EmptySD3LatentImage",
|
||||
"_meta": { "title": "Empty Latent 832x1216 poster" },
|
||||
"inputs": { "width": 832, "height": 1216, "batch_size": 1 }
|
||||
},
|
||||
"9": {
|
||||
"class_type": "KSampler",
|
||||
"_meta": { "title": "KSampler" },
|
||||
"inputs": {
|
||||
"model": [ "5", 0 ],
|
||||
"seed": 2026,
|
||||
"steps": 25,
|
||||
"cfg": 1.0,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"positive": [ "3", 0 ],
|
||||
"negative": [ "4", 0 ],
|
||||
"latent_image": [ "8", 0 ],
|
||||
"denoise": 1.0
|
||||
}
|
||||
},
|
||||
"10": {
|
||||
"class_type": "VAEDecode",
|
||||
"_meta": { "title": "VAE Decode" },
|
||||
"inputs": { "samples": [ "9", 0 ], "vae": [ "7", 0 ] }
|
||||
},
|
||||
"11": {
|
||||
"class_type": "SaveImage",
|
||||
"_meta": { "title": "Save Image (text poster)" },
|
||||
"inputs": { "filename_prefix": "qwen21-text", "images": [ "10", 0 ] }
|
||||
}
|
||||
}
|
||||
@@ -10,15 +10,15 @@ model:
|
||||
provider: dankclaw-large
|
||||
base_url: http://192.168.0.143:7770/v1
|
||||
api_key: not-needed
|
||||
default: frank-large
|
||||
default: frank-dragoon
|
||||
|
||||
providers:
|
||||
dankclaw-large:
|
||||
api: http://192.168.0.143:7770/v1
|
||||
discover_models: false
|
||||
models:
|
||||
frank-large:
|
||||
context_length: 114688
|
||||
frank-dragoon:
|
||||
context_length: 262144
|
||||
dankserver-small:
|
||||
api: http://192.168.0.235:7770/v1
|
||||
discover_models: false
|
||||
@@ -29,7 +29,7 @@ providers:
|
||||
# Auxiliary routing from the live configuration. Named providers route via
|
||||
# their `api:` URL; `custom` entries route by their explicit `base_url`.
|
||||
auxiliary:
|
||||
vision: {provider: dankclaw-large, model: frank-large}
|
||||
vision: {provider: dankclaw-large, model: frank-dragoon}
|
||||
web_extract: {provider: dankserver-small, model: frank-medium}
|
||||
compression: {provider: dankserver-small, model: frank-medium}
|
||||
skills_hub: {provider: dankserver-small, model: frank-medium}
|
||||
@@ -50,8 +50,8 @@ auxiliary:
|
||||
effort: none
|
||||
memory_query_rewrite: {provider: custom, model: frank-medium, base_url: http://192.168.0.235:7770/v1}
|
||||
tts_audio_tags: {provider: custom, model: frank-medium, base_url: http://192.168.0.235:7770/v1}
|
||||
triage_specifier: {provider: dankclaw-large, model: frank-large}
|
||||
kanban_decomposer: {provider: dankclaw-large, model: frank-large}
|
||||
triage_specifier: {provider: dankclaw-large, model: frank-dragoon}
|
||||
kanban_decomposer: {provider: dankclaw-large, model: frank-dragoon}
|
||||
profile_describer: {provider: dankserver-small, model: frank-medium}
|
||||
curator: {provider: dankserver-small, model: frank-medium}
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1,9 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# download-frank-primary-fp8.sh — fetch/verify the Qwen3.8-Flash-Next-FP8
|
||||
# download-frank-archon-fp8.sh — fetch/verify the Qwen3.8-Flash-Next-FP8
|
||||
# payload (131 safetensors shards, 172.78 GiB total, plus aux files) for the
|
||||
# vLLM frank-primary deployment on dankserver.
|
||||
# vLLM frank-archon deployment on dankserver.
|
||||
#
|
||||
# bash scripts/download-frank-primary-fp8.sh [dest_dir]
|
||||
# bash scripts/download-frank-archon-fp8.sh [dest_dir]
|
||||
#
|
||||
# The manifest (scripts/manifests/qwen3.8-flash-next-fp8.manifest) carries
|
||||
# per-file sizes + LFS sha256s from the Hub; hf-fetch.sh is idempotent — an
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the z-lab Qwen3.5-9B-DFlash drafter for frank-dragoon's DFlash
|
||||
# speculative-decode arm (vLLM --speculative-config method=dflash). Public
|
||||
# repo — no auth header on the command line.
|
||||
#
|
||||
# The drafter is a 6-layer qwen3 draft whose dflash_config pins
|
||||
# target_layer_ids [1,5,9,13,17,21,25,29] and block_size 16; its hidden_size
|
||||
# 4096 / vocab_size 248320 match the MiMo-V2.6-Distill-Qwen-9B target exactly
|
||||
# (both preserve the Qwen3.5-9B geometry), which is what lets vLLM load it as
|
||||
# the draft for that target.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-dragoon-dflash.sh [DEST_DIR] [--verify-only]
|
||||
set -euo pipefail
|
||||
|
||||
REPO="z-lab/Qwen3.5-9B-DFlash"
|
||||
BASE="https://huggingface.co/${REPO}/resolve/main"
|
||||
|
||||
DEST=""
|
||||
VERIFY_ONLY=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--verify-only) VERIFY_ONLY=1 ;;
|
||||
*) DEST="$arg" ;;
|
||||
esac
|
||||
done
|
||||
DEST="${DEST:-/home/dank/models/Qwen3.5-9B-DFlash}"
|
||||
|
||||
# name bytes sha256(- = non-LFS, size-only check)
|
||||
FILES=(
|
||||
"model.safetensors 2583816465 0a42274b32554f48de1faa0d42824e9c2ceda649c30ae0a731cddf410dd698c7"
|
||||
"config.json 1222 -"
|
||||
)
|
||||
|
||||
mkdir -p "$DEST"
|
||||
status=0
|
||||
for entry in "${FILES[@]}"; do
|
||||
read -r name size sha <<<"$entry"
|
||||
f="$DEST/$name"
|
||||
have=0
|
||||
if [[ -f "$f" ]] && [[ "$(stat -c%s "$f")" == "$size" ]]; then
|
||||
if [[ "$sha" == "-" ]] || sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
have=1
|
||||
fi
|
||||
fi
|
||||
if (( have )); then
|
||||
echo "OK $name"
|
||||
continue
|
||||
fi
|
||||
if (( VERIFY_ONLY )); then
|
||||
echo "MISSING $name"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
echo "FETCHING $name"
|
||||
tmp="$f.part"
|
||||
curl -fsSL --fail-with-body -o "$tmp" "$BASE/$name"
|
||||
mv "$tmp" "$f"
|
||||
if [[ "$(stat -c%s "$f")" != "$size" ]]; then
|
||||
echo "SIZE FAIL $name ($(stat -c%s "$f") != $size)"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
if [[ "$sha" != "-" ]] && ! sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
echo "SHA FAIL $name"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
echo "VERIFIED $name"
|
||||
done
|
||||
if (( status == 0 )); then
|
||||
echo "DOWNLOAD_DONE $DEST"
|
||||
else
|
||||
echo "DOWNLOAD_FAILED $DEST" >&2
|
||||
fi
|
||||
exit "$status"
|
||||
Executable
+90
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the BF16 MiMo-V2.6-Distill-Qwen-9B payload for frank-dragoon
|
||||
# (vLLM/kyuz0 gfx1151, Strix Halo iGPU). Public repo — no auth header on the
|
||||
# command line (a Bearer token in argv is world-readable via /proc).
|
||||
#
|
||||
# Per-file FRESH downloads, no resume: the house rule (see
|
||||
# download-frank-zealot.sh); -f/--fail-with-body plus size+sha256 against the
|
||||
# Hub LFS oid is what makes an access error fail instead of "succeeding".
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-dragoon-mimo.sh [DEST_DIR] [--verify-only]
|
||||
#
|
||||
# The verified set below is exactly what the 2026-10-01 bench window served:
|
||||
# 17.53 GiB of BF16 weights (4 shards, 18,819,720,848 B total with the rest).
|
||||
set -euo pipefail
|
||||
|
||||
REPO="XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B"
|
||||
BASE="https://huggingface.co/${REPO}/resolve/main"
|
||||
|
||||
DEST=""
|
||||
VERIFY_ONLY=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--verify-only) VERIFY_ONLY=1 ;;
|
||||
*) DEST="$arg" ;;
|
||||
esac
|
||||
done
|
||||
DEST="${DEST:-/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-BF16}"
|
||||
|
||||
# name bytes sha256(- = non-LFS, size-only check)
|
||||
FILES=(
|
||||
"model-00001-of-00004.safetensors 5276436216 aab052180118aee34abc3029b54eaa49096aac606b97d703866b420dceb703c3"
|
||||
"model-00002-of-00004.safetensors 5033171280 7a0486565f06d25ac4628e9dba470dc3f604353471d240d5a0bf7128f64df396"
|
||||
"model-00003-of-00004.safetensors 5234499504 6c73207563d1879bfd6c143a028cc70be68edff56280458c71a43df4240300f4"
|
||||
"model-00004-of-00004.safetensors 3275613848 1379a7cf8c0b8555a39ab65a47e830e0eb45e776b46045734da3afd73c09eea2"
|
||||
"model.safetensors.index.json 69217 -"
|
||||
"config.json 2784 -"
|
||||
"generation_config.json 126 -"
|
||||
"chat_template.jinja 3916 -"
|
||||
"tokenizer.json 19989325 06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523"
|
||||
"tokenizer_config.json 1165 -"
|
||||
"vocab.json 6722759 -"
|
||||
"merges.txt 3353259 -"
|
||||
"preprocessor_config.json 443 -"
|
||||
"processor_config.json 1191 -"
|
||||
"video_preprocessor_config.json 385 -"
|
||||
)
|
||||
|
||||
mkdir -p "$DEST"
|
||||
status=0
|
||||
for entry in "${FILES[@]}"; do
|
||||
read -r name size sha <<<"$entry"
|
||||
f="$DEST/$name"
|
||||
have=0
|
||||
if [[ -f "$f" ]] && [[ "$(stat -c%s "$f")" == "$size" ]]; then
|
||||
if [[ "$sha" == "-" ]] || sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
have=1
|
||||
fi
|
||||
fi
|
||||
if (( have )); then
|
||||
echo "OK $name"
|
||||
continue
|
||||
fi
|
||||
if (( VERIFY_ONLY )); then
|
||||
echo "MISSING $name"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
echo "FETCHING $name"
|
||||
tmp="$f.part"
|
||||
curl -fsSL --fail-with-body -o "$tmp" "$BASE/$name"
|
||||
mv "$tmp" "$f"
|
||||
if [[ "$(stat -c%s "$f")" != "$size" ]]; then
|
||||
echo "SIZE FAIL $name ($(stat -c%s "$f") != $size)"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
if [[ "$sha" != "-" ]] && ! sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
echo "SHA FAIL $name"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
echo "VERIFIED $name"
|
||||
done
|
||||
if (( status == 0 )); then
|
||||
echo "DOWNLOAD_DONE $DEST"
|
||||
else
|
||||
echo "DOWNLOAD_FAILED $DEST" >&2
|
||||
fi
|
||||
exit "$status"
|
||||
Executable
+85
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the source Qwen3.5-9B checkpoint — the ONLY place this model
|
||||
# family's MTP head (15 mtp.* tensors) exists. frank-dragoon runs it as the
|
||||
# vLLM cross-model MTP draft (MiMo-V2.6-Distill-Qwen-9B itself ships zero
|
||||
# mtp tensors; that is why SGLang's NEXTN on frank-zealot draws 0.00
|
||||
# acceptance). Public repo — no auth header on the command line.
|
||||
#
|
||||
# NOTE the odd upstream shard names (model.safetensors-0000N-of-00004...).
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-dragoon-qwen35.sh [DEST_DIR] [--verify-only]
|
||||
set -euo pipefail
|
||||
|
||||
REPO="Qwen/Qwen3.5-9B"
|
||||
BASE="https://huggingface.co/${REPO}/resolve/main"
|
||||
|
||||
DEST=""
|
||||
VERIFY_ONLY=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--verify-only) VERIFY_ONLY=1 ;;
|
||||
*) DEST="$arg" ;;
|
||||
esac
|
||||
done
|
||||
DEST="${DEST:-/home/dank/models/Qwen3.5-9B}"
|
||||
|
||||
# name bytes sha256(- = non-LFS, size-only check)
|
||||
FILES=(
|
||||
"model.safetensors-00001-of-00004.safetensors 5276436216 db6f444b43d318c92f360a13a25561a6a65b10c0631b8ed305a426dbaa6c380e"
|
||||
"model.safetensors-00002-of-00004.safetensors 5335161512 31c7d7e2dd5d207840b31cc59083c8f4c4718959149e0358c0364052bb9a0330"
|
||||
"model.safetensors-00003-of-00004.safetensors 5368717440 7ec36ba3a4176a44c3c0876ad80c56a2f70c84bf008d82e9501df642f17dadec"
|
||||
"model.safetensors-00004-of-00004.safetensors 3325995712 b62b0c4cd7e44edee103ee8f4fe225f246d5e768e07bfd5f25b63a8aa1fdd0c6"
|
||||
"model.safetensors.index.json 79657 -"
|
||||
"config.json 3126 -"
|
||||
"chat_template.jinja 7756 -"
|
||||
"tokenizer.json 12807982 5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42"
|
||||
"tokenizer_config.json 16710 -"
|
||||
"vocab.json 6722759 -"
|
||||
"merges.txt 3353259 -"
|
||||
"preprocessor_config.json 390 -"
|
||||
"video_preprocessor_config.json 385 -"
|
||||
)
|
||||
|
||||
mkdir -p "$DEST"
|
||||
status=0
|
||||
for entry in "${FILES[@]}"; do
|
||||
read -r name size sha <<<"$entry"
|
||||
f="$DEST/$name"
|
||||
have=0
|
||||
if [[ -f "$f" ]] && [[ "$(stat -c%s "$f")" == "$size" ]]; then
|
||||
if [[ "$sha" == "-" ]] || sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
have=1
|
||||
fi
|
||||
fi
|
||||
if (( have )); then
|
||||
echo "OK $name"
|
||||
continue
|
||||
fi
|
||||
if (( VERIFY_ONLY )); then
|
||||
echo "MISSING $name"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
echo "FETCHING $name"
|
||||
tmp="$f.part"
|
||||
curl -fsSL --fail-with-body -o "$tmp" "$BASE/$name"
|
||||
mv "$tmp" "$f"
|
||||
if [[ "$(stat -c%s "$f")" != "$size" ]]; then
|
||||
echo "SIZE FAIL $name ($(stat -c%s "$f") != $size)"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
if [[ "$sha" != "-" ]] && ! sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
echo "SHA FAIL $name"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
echo "VERIFIED $name"
|
||||
done
|
||||
if (( status == 0 )); then
|
||||
echo "DOWNLOAD_DONE $DEST"
|
||||
else
|
||||
echo "DOWNLOAD_FAILED $DEST" >&2
|
||||
fi
|
||||
exit "$status"
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound for frank-small
|
||||
# Download MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound for frank-zealot
|
||||
# (SGLang, RTX 3090). Public repo — no auth header on the command line.
|
||||
#
|
||||
# Per-file FRESH downloads, no resume: `curl -C -` once produced a 213 MB
|
||||
@@ -7,7 +7,7 @@
|
||||
# the house rule.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-small.sh [DEST_DIR] [--verify-only]
|
||||
# scripts/download-frank-zealot.sh [DEST_DIR] [--verify-only]
|
||||
#
|
||||
# LFS shards are verified against the Hub LFS sha256; non-LFS text files
|
||||
# against their declared byte size. Idempotent: complete files are skipped.
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# bench-frank-large.sh — 64k-context benchmark for the frank-default-large stack.
|
||||
# bench-frank-dragoon.sh — 64k-context benchmark for the frank-default-large stack.
|
||||
#
|
||||
# Measures generation throughput of Qwen3.6-27B Q8_K_P + DFlash + ngram-mod on
|
||||
# a single backend at 64k context. It starts llama-server on a scratch port,
|
||||
@@ -7,15 +7,15 @@
|
||||
# prompt, and prints tokens/sec.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/bench-frank-large.sh <ROCm0|Vulkan0> [--generations N]
|
||||
# ./scripts/bench-frank-dragoon.sh <ROCm0|Vulkan0> [--generations N]
|
||||
#
|
||||
# Example:
|
||||
# ./scripts/bench-frank-large.sh ROCm0
|
||||
# ./scripts/bench-frank-large.sh Vulkan0 --generations 3
|
||||
# ./scripts/bench-frank-dragoon.sh ROCm0
|
||||
# ./scripts/bench-frank-dragoon.sh Vulkan0 --generations 3
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DEVICE="${1:?usage: bench-frank-large.sh <ROCm0|Vulkan0> [--generations N]}"
|
||||
DEVICE="${1:?usage: bench-frank-dragoon.sh <ROCm0|Vulkan0> [--generations N]}"
|
||||
GENERATIONS=1
|
||||
if [ "${2:-}" = "--generations" ]; then
|
||||
GENERATIONS="${3:?--generations needs a count}"
|
||||
@@ -41,7 +41,7 @@ SERVICE_HEALTH[dankclaw]="http://127.0.0.1:7770/health"
|
||||
# Drop page caches before restart: with the model's pages fully cached, the
|
||||
# KV/graph allocation stalls at load (VRAM stuck at 28 GiB). See AGENTS.md.
|
||||
DROP_CACHES[dankclaw]=1
|
||||
# Passwordless restart helper (root-owned, installed via /etc/sudoers.d/frank-large).
|
||||
# Passwordless restart helper (root-owned, installed via /etc/sudoers.d/frank-dragoon).
|
||||
# When present, deploy.sh uses it instead of inline sudo commands.
|
||||
HELPER[dankclaw]="/usr/local/sbin/frank-restart-large.sh"
|
||||
# Health wait in iterations of 2s. Cold loads after the cache drop measured
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Tunables for the Ling-3.0-flash TRAIL on dankclaw (frank-large-ling).
|
||||
# Tunables for the Ling-3.0-flash TRAIL on dankclaw (frank-dragoon-ling).
|
||||
# This REPLACES frank-default-large (Qwen3.6-27B) temporarily. To switch
|
||||
# back: systemctl stop frank-large-ling && systemctl start frank-default-large.
|
||||
# back: systemctl stop frank-dragoon-ling && systemctl start frank-default-large.
|
||||
# Runtime: llama.cpp FORK bailingmoe3-support; Vulkan + AMD GTT.
|
||||
#
|
||||
# Memory model (amdgpu Strix Halo gfx1151, 8x16GB LPDDR5 = 128 GiB physical):
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[Unit]
|
||||
Description=llama.cpp Ling-3.0-flash Q6_K Vulkan+MTP (frank-large-ling TRIAL)
|
||||
Description=llama.cpp Ling-3.0-flash Q6_K Vulkan+MTP (frank-dragoon-ling TRIAL)
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
@@ -8,7 +8,7 @@
|
||||
# Vulkan exposes as an ~111 GiB heap). MTP drafter is bundled in the same
|
||||
# GGUF (block 42) - no separate --spec-draft-model needed.
|
||||
#
|
||||
# Tie-back: to restore the Qwen3.6-27B service, stop frank-large-ling and
|
||||
# Tie-back: to restore the Qwen3.6-27B service, stop frank-dragoon-ling and
|
||||
# start frank-default-large (its env/unit are committed and unchanged).
|
||||
#
|
||||
# Tunables come from frank-default-large.ling.env in this directory unless
|
||||
|
||||
@@ -4,11 +4,11 @@
|
||||
# Runtime: llama.cpp llama-server (>= b10411).
|
||||
#
|
||||
# Tunables come from frank-default-large.env in this directory unless the
|
||||
# FRANK_LARGE_ENV_FILE environment variable overrides that path.
|
||||
# FRANK_DRAGOON_ENV_FILE environment variable overrides that path.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}"
|
||||
ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
@@ -0,0 +1,86 @@
|
||||
# Subsurface Aquifer Monitoring Network - System Design Document (Revision 7)
|
||||
|
||||
This document describes the deployment of a subsurface aquifer monitoring network for the Cascadia sedimentary basin. It is written for the operations crew and the regional water authority. Every sensor node, relay, aggregation service, calibration procedure, and failure mode below carries a stable alphanumeric identifier that appears in every table and in the field notes.
|
||||
|
||||
## Sensor nodes
|
||||
Node SN-000-08B9 sits in a discharge lens at 359.0 m below datum and reports a salinity of 2.38 ppt with the fractured interval reading 5.40 degC at dawn and the wellhead porous 1.72 m3/d while the electrode pair returns 3157 uS/cm.
|
||||
Node SN-001-0A65 sits in a percolate lens at 25.9 m below datum and steps down to a salinity of 2.13 ppt with the diapir interval reading 8.55 degC at dawn and the wellhead silty 0.67 m3/d while the electrode pair returns 979 uS/cm.
|
||||
Node SN-002-0571 sits in a diapir lens at 287.8 m below datum and peaks at a salinity of 7.63 ppt with the karst interval reading 4.26 degC at dawn and the wellhead spooned 11.50 m3/d while the electrode pair returns 2987 uS/cm.
|
||||
Node SN-003-0CE9 sits in a confining lens at 169.5 m below datum and logs a salinity of 0.96 ppt with the alluvial interval reading 5.03 degC at dawn and the wellhead salinized 12.08 m3/d while the electrode pair returns 2607 uS/cm.
|
||||
Node SN-004-0E2A sits in a freshened lens at 70.4 m below datum and settles near a salinity of 8.68 ppt with the spooned interval reading 4.88 degC at dawn and the wellhead siphon 4.38 m3/d while the electrode pair returns 2707 uS/cm.
|
||||
Node SN-005-0DDD sits in a percolate lens at 44.6 m below datum and reports a salinity of 6.25 ppt with the alluvial interval reading 10.85 degC at dawn and the wellhead brackish 13.99 m3/d while the electrode pair returns 3618 uS/cm.
|
||||
Node SN-006-0EFC sits in a till lens at 309.5 m below datum and reports a salinity of 3.49 ppt with the lacustrine interval reading 7.38 degC at dawn and the wellhead spooned 3.21 m3/d while the electrode pair returns 1253 uS/cm.
|
||||
Node SN-007-0B02 sits in a recharge lens at 262.0 m below datum and steps down to a salinity of 2.38 ppt with the discharge interval reading 8.18 degC at dawn and the wellhead perched 4.05 m3/d while the electrode pair returns 3900 uS/cm.
|
||||
Node SN-008-0448 sits in a karst lens at 406.5 m below datum and reports a salinity of 7.32 ppt with the upconing interval reading 6.17 degC at dawn and the wellhead fluvial 0.75 m3/d while the electrode pair returns 1448 uS/cm.
|
||||
Node SN-009-043C sits in a alluvial lens at 159.3 m below datum and flags a salinity of 6.20 ppt with the percolate interval reading 7.60 degC at dawn and the wellhead brackish 13.01 m3/d while the electrode pair returns 2025 uS/cm.
|
||||
Node SN-010-0FF5 sits in a karst lens at 264.2 m below datum and flags a salinity of 7.05 ppt with the freshened interval reading 7.88 degC at dawn and the wellhead siphon 8.41 m3/d while the electrode pair returns 1635 uS/cm.
|
||||
Node SN-011-098A sits in a drain lens at 365.7 m below datum and reports a salinity of 8.11 ppt with the wedge interval reading 5.51 degC at dawn and the wellhead glacial 2.52 m3/d while the electrode pair returns 2916 uS/cm.
|
||||
Node SN-012-0AE8 sits in a till lens at 259.6 m below datum and steps down to a salinity of 9.13 ppt with the recharge interval reading 11.60 degC at dawn and the wellhead spooned 0.46 m3/d while the electrode pair returns 3077 uS/cm.
|
||||
Node SN-013-0B83 sits in a salinized lens at 171.2 m below datum and peaks at a salinity of 2.83 ppt with the lacustrine interval reading 5.56 degC at dawn and the wellhead discharge 0.35 m3/d while the electrode pair returns 3083 uS/cm.
|
||||
Node SN-014-032E sits in a drain lens at 249.6 m below datum and flags a salinity of 1.09 ppt with the salinized interval reading 9.58 degC at dawn and the wellhead perched 12.00 m3/d while the electrode pair returns 2221 uS/cm.
|
||||
Node SN-015-004F sits in a aeolian lens at 264.4 m below datum and reports a salinity of 7.34 ppt with the salinized interval reading 8.76 degC at dawn and the wellhead perched 0.31 m3/d while the electrode pair returns 1483 uS/cm.
|
||||
Node SN-016-07C6 sits in a aeolian lens at 155.9 m below datum and reports a salinity of 0.64 ppt with the mineralized interval reading 11.75 degC at dawn and the wellhead diluted 13.46 m3/d while the electrode pair returns 524 uS/cm.
|
||||
Node SN-017-06C5 sits in a salinized lens at 72.1 m below datum and averages a salinity of 4.52 ppt with the siphon interval reading 8.93 degC at dawn and the wellhead confining 3.98 m3/d while the electrode pair returns 3687 uS/cm.
|
||||
Node SN-018-0E64 sits in a siphon lens at 334.9 m below datum and steps down to a salinity of 6.73 ppt with the salinized interval reading 7.63 degC at dawn and the wellhead freshened 9.64 m3/d while the electrode pair returns 1681 uS/cm.
|
||||
Node SN-019-0122 sits in a spooned lens at 117.2 m below datum and logs a salinity of 3.24 ppt with the fractured interval reading 9.26 degC at dawn and the wellhead karst 3.50 m3/d while the electrode pair returns 1065 uS/cm.
|
||||
Node SN-020-021D sits in a karst lens at 414.3 m below datum and recovers to a salinity of 0.76 ppt with the glacial interval reading 6.25 degC at dawn and the wellhead silty 9.60 m3/d while the electrode pair returns 1041 uS/cm.
|
||||
Node SN-021-06C6 sits in a percolate lens at 379.2 m below datum and drifts to a salinity of 7.61 ppt with the wedge interval reading 5.84 degC at dawn and the wellhead karst 1.65 m3/d while the electrode pair returns 1913 uS/cm.
|
||||
Node SN-022-0DC9 sits in a spooned lens at 327.1 m below datum and reports a salinity of 9.25 ppt with the freshened interval reading 5.05 degC at dawn and the wellhead silty 5.90 m3/d while the electrode pair returns 1544 uS/cm.
|
||||
Node SN-023-0717 sits in a karst lens at 222.0 m below datum and steps down to a salinity of 4.02 ppt with the alluvial interval reading 6.60 degC at dawn and the wellhead siphon 3.77 m3/d while the electrode pair returns 3892 uS/cm.
|
||||
Node SN-024-02A9 sits in a fractured lens at 479.7 m below datum and drifts to a salinity of 7.88 ppt with the silty interval reading 12.53 degC at dawn and the wellhead discharge 13.18 m3/d while the electrode pair returns 3592 uS/cm.
|
||||
Node SN-025-0C8F sits in a wedge lens at 434.3 m below datum and holds a salinity of 1.63 ppt with the alluvial interval reading 4.22 degC at dawn and the wellhead till 5.73 m3/d while the electrode pair returns 3905 uS/cm.
|
||||
Node SN-026-04BF sits in a colluvial lens at 460.0 m below datum and averages a salinity of 9.36 ppt with the diapir interval reading 8.98 degC at dawn and the wellhead brackish 10.29 m3/d while the electrode pair returns 802 uS/cm.
|
||||
Node SN-027-0DA4 sits in a silty lens at 265.7 m below datum and steps down to a salinity of 7.06 ppt with the percolate interval reading 4.69 degC at dawn and the wellhead freshened 8.42 m3/d while the electrode pair returns 2201 uS/cm.
|
||||
Node SN-028-01EB sits in a perched lens at 49.5 m below datum and recovers to a salinity of 1.83 ppt with the silty interval reading 9.32 degC at dawn and the wellhead drain 9.69 m3/d while the electrode pair returns 1126 uS/cm.
|
||||
Node SN-029-042C sits in a karst lens at 30.6 m below datum and averages a salinity of 0.86 ppt with the percolate interval reading 9.85 degC at dawn and the wellhead recharge 8.16 m3/d while the electrode pair returns 1452 uS/cm.
|
||||
|
||||
## Relays
|
||||
Relay RL-00-02181 reburses the upconing trunk from the brackish margin and peaks at a round-trip time of 131.8 ms with a frame loss of 1.04 percent across the night window.
|
||||
Relay RL-01-00262 reburses the spooned trunk from the fluvial margin and holds a round-trip time of 372.2 ms with a frame loss of 3.65 percent across the night window.
|
||||
Relay RL-02-03691 reburses the recharge trunk from the percolate margin and steps down to a round-trip time of 398.5 ms with a frame loss of 0.29 percent across the night window.
|
||||
Relay RL-03-0E117 reburses the lacustrine trunk from the confining margin and averages a round-trip time of 373.9 ms with a frame loss of 3.44 percent across the night window.
|
||||
Relay RL-04-0B41B reburses the aeolian trunk from the colluvial margin and peaks at a round-trip time of 70.7 ms with a frame loss of 3.25 percent across the night window.
|
||||
Relay RL-05-0D12D reburses the recharge trunk from the diluted margin and steps down to a round-trip time of 264.7 ms with a frame loss of 0.03 percent across the night window.
|
||||
Relay RL-06-043B4 reburses the colluvial trunk from the upconing margin and reports a round-trip time of 49.5 ms with a frame loss of 3.42 percent across the night window.
|
||||
Relay RL-07-09AD6 reburses the fractured trunk from the freshened margin and averages a round-trip time of 225.3 ms with a frame loss of 1.06 percent across the night window.
|
||||
Relay RL-08-04394 reburses the brackish trunk from the fluvial margin and steps down to a round-trip time of 88.6 ms with a frame loss of 2.47 percent across the night window.
|
||||
Relay RL-09-017A0 reburses the wedge trunk from the lacustrine margin and drifts to a round-trip time of 363.0 ms with a frame loss of 3.30 percent across the night window.
|
||||
Relay RL-10-0217D reburses the mineralized trunk from the lacustrine margin and peaks at a round-trip time of 26.2 ms with a frame loss of 1.30 percent across the night window.
|
||||
Relay RL-11-08F97 reburses the perched trunk from the freshened margin and logs a round-trip time of 181.8 ms with a frame loss of 2.75 percent across the night window.
|
||||
Relay RL-12-02628 reburses the fractured trunk from the glacial margin and steps down to a round-trip time of 378.5 ms with a frame loss of 2.69 percent across the night window.
|
||||
Relay RL-13-06E06 reburses the silty trunk from the mineralized margin and flags a round-trip time of 153.4 ms with a frame loss of 2.15 percent across the night window.
|
||||
|
||||
## Aggregation services
|
||||
Aggregator AG-00-3BB logs a queue depth of 5.2 messages and a histogram bucket of 6.82 seconds before the dedupe pass settles the stream for the next rotation.
|
||||
Aggregator AG-01-0FF reports a queue depth of 86.4 messages and a histogram bucket of 4.10 seconds before the dedupe pass settles the stream for the next rotation.
|
||||
Aggregator AG-02-37F drifts to a queue depth of 34.7 messages and a histogram bucket of 10.25 seconds before the dedupe pass settles the stream for the next rotation.
|
||||
Aggregator AG-03-3B3 averages a queue depth of 93.5 messages and a histogram bucket of 13.59 seconds before the dedupe pass settles the stream for the next rotation.
|
||||
Aggregator AG-04-33E flags a queue depth of 83.2 messages and a histogram bucket of 17.58 seconds before the dedupe pass settles the stream for the next rotation.
|
||||
Aggregator AG-05-2F2 settles near a queue depth of 84.8 messages and a histogram bucket of 0.84 seconds before the dedupe pass settles the stream for the next rotation.
|
||||
Aggregator AG-06-2AD averages a queue depth of 75.4 messages and a histogram bucket of 7.65 seconds before the dedupe pass settles the stream for the next rotation.
|
||||
Aggregator AG-07-06E drifts to a queue depth of 26.5 messages and a histogram bucket of 14.26 seconds before the dedupe pass settles the stream for the next rotation.
|
||||
Aggregator AG-08-0E3 averages a queue depth of 83.9 messages and a histogram bucket of 15.51 seconds before the dedupe pass settles the stream for the next rotation.
|
||||
Aggregator AG-09-138 averages a queue depth of 78.6 messages and a histogram bucket of 8.50 seconds before the dedupe pass settles the stream for the next rotation.
|
||||
|
||||
## Calibration procedures
|
||||
Procedure CP-00-047 steps the probe through a two-point rinse, then applies a trim of 0.2084 at the 66 milli-bar mark and records a residual of 0.02052 before the next field seal is crimped.
|
||||
Procedure CP-01-0A4 steps the probe through a two-point rinse, then applies a trim of 0.7793 at the 96 milli-bar mark and records a residual of 0.01468 before the next field seal is crimped.
|
||||
Procedure CP-02-054 steps the probe through a two-point rinse, then applies a trim of 0.4634 at the 68 milli-bar mark and records a residual of 0.04234 before the next field seal is crimped.
|
||||
Procedure CP-03-02D steps the probe through a two-point rinse, then applies a trim of 0.8459 at the 12 milli-bar mark and records a residual of 0.04855 before the next field seal is crimped.
|
||||
Procedure CP-04-098 steps the probe through a two-point rinse, then applies a trim of 0.5267 at the 97 milli-bar mark and records a residual of 0.00287 before the next field seal is crimped.
|
||||
Procedure CP-05-09B steps the probe through a two-point rinse, then applies a trim of 0.3968 at the 72 milli-bar mark and records a residual of 0.01637 before the next field seal is crimped.
|
||||
Procedure CP-06-030 steps the probe through a two-point rinse, then applies a trim of 0.8860 at the 12 milli-bar mark and records a residual of 0.04508 before the next field seal is crimped.
|
||||
Procedure CP-07-0EC steps the probe through a two-point rinse, then applies a trim of 0.2367 at the 70 milli-bar mark and records a residual of 0.00108 before the next field seal is crimped.
|
||||
Procedure CP-08-0BC steps the probe through a two-point rinse, then applies a trim of 0.7278 at the 68 milli-bar mark and records a residual of 0.04707 before the next field seal is crimped.
|
||||
Procedure CP-09-0AA steps the probe through a two-point rinse, then applies a trim of 0.6069 at the 37 milli-bar mark and records a residual of 0.00443 before the next field seal is crimped.
|
||||
|
||||
## Failure modes
|
||||
Mode FM-00-27 declares a fluvial excursion when two independent probes disagree beyond the band for three rotations; the fleet reports a mean time between failures of 36902 hours for this class of excursion.
|
||||
Mode FM-01-13 declares a drain excursion when two independent probes disagree beyond the band for three rotations; the fleet peaks at a mean time between failures of 29042 hours for this class of excursion.
|
||||
Mode FM-02-14 declares a till excursion when two independent probes disagree beyond the band for three rotations; the fleet peaks at a mean time between failures of 26976 hours for this class of excursion.
|
||||
Mode FM-03-0C declares a diapir excursion when two independent probes disagree beyond the band for three rotations; the fleet drifts to a mean time between failures of 22578 hours for this class of excursion.
|
||||
Mode FM-04-39 declares a perched excursion when two independent probes disagree beyond the band for three rotations; the fleet recovers to a mean time between failures of 27383 hours for this class of excursion.
|
||||
Mode FM-05-19 declares a siphon excursion when two independent probes disagree beyond the band for three rotations; the fleet logs a mean time between failures of 23152 hours for this class of excursion.
|
||||
|
||||
## End of document
|
||||
Summarize the network's dominant risk in one paragraph, then list the three node identifiers whose readings most limit the model this month.
|
||||
File diff suppressed because one or more lines are too long
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Decode throughput benchmark for frank-large.
|
||||
Decode throughput benchmark for frank-dragoon.
|
||||
Starts the server with a given --threads-batch value, sends an 8K+ token prompt,
|
||||
generates 256 tokens, measures generation tok/s.
|
||||
|
||||
@@ -13,12 +13,12 @@ SERVE = "/home/dank/GitPackages/ROCmFPX/build-qwen38/bin/llama-server"
|
||||
PORT = 7770
|
||||
N_GEN = 256
|
||||
PROMPT_FILE = "/tmp/bench-prompts/prompt_code.txt"
|
||||
ENV_FILE = os.path.join(os.path.dirname(__file__), "frank-large.env")
|
||||
ENV_FILE = os.path.join(os.path.dirname(__file__), "frank-dragoon.env")
|
||||
SERVER_DIR = os.path.dirname(SERVE)
|
||||
ROCM_PATH = "/opt/rocm"
|
||||
|
||||
def make_env():
|
||||
"""Build the same env as serve-frank-large.sh."""
|
||||
"""Build the same env as serve-frank-dragoon.sh."""
|
||||
env = {**os.environ, "HSA_OVERRIDE_GFX_VERSION": "11.5.1", "HIP_VISIBLE_DEVICES": "0"}
|
||||
|
||||
# Source env file for tunables
|
||||
@@ -59,7 +59,7 @@ def wait_ready():
|
||||
def send_request(prompt, max_tokens=N_GEN):
|
||||
"""Send chat completion, measure generation throughput."""
|
||||
body = json.dumps({
|
||||
"model": "frank-large",
|
||||
"model": "frank-dragoon",
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens,
|
||||
"stream": False,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Benchmark n-gram speculative decoding vs no-spec for frank-large.
|
||||
Benchmark n-gram speculative decoding vs no-spec for frank-dragoon.
|
||||
|
||||
Tests prompt-processing throughput with 16K token prompts (manageable request size).
|
||||
Code-heavy prompt (high n-gram redundancy) vs prose prompt (low redundancy).
|
||||
@@ -103,7 +103,7 @@ def start_server(port, spec_type):
|
||||
def run_test(prompt_text, spec_type, port):
|
||||
"""Send a request and extract prompt processing metrics."""
|
||||
data = json.dumps({
|
||||
"model": "frank-large",
|
||||
"model": "frank-dragoon",
|
||||
"messages": [{"role": "user", "content": prompt_text}],
|
||||
"max_tokens": TEST_TOKENS,
|
||||
"stream": False,
|
||||
@@ -189,7 +189,7 @@ def main():
|
||||
print(f" Prose: {prose_prompt_file.stat().st_size:,} bytes")
|
||||
|
||||
print("=" * 60)
|
||||
print(" N-gram Mod Benchmark — frank-large")
|
||||
print(" N-gram Mod Benchmark — frank-dragoon")
|
||||
print(f" STRIX ROCmFP4 · 64K context · 16K prompts")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
# Benchmark n-gram speculative decoding vs no-spec for frank-large.
|
||||
# Benchmark n-gram speculative decoding vs no-spec for frank-dragoon.
|
||||
#
|
||||
# This tests prompt-processing throughput at 64k context with two distinct
|
||||
# prompts: one code-heavy and one prose-heavy. Uses llama-server on scratch
|
||||
# ports with the OpenAI-compatible API to measure prompt_eval_duration_ms.
|
||||
#
|
||||
# Run on Dankclaw after stopping the production service:
|
||||
# sudo systemctl stop frank-large
|
||||
# sudo systemctl stop frank-dragoon
|
||||
# ./scripts/bench-ngram-mod.sh
|
||||
# sudo systemctl start frank-large
|
||||
# sudo systemctl start frank-dragoon
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a; source "$ENV_FILE"; set +a
|
||||
fi
|
||||
@@ -184,7 +184,7 @@ run_single_test() {
|
||||
response=$(curl -s -X POST "http://127.0.0.1:$port/v1/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"model\": \"frank-large\",
|
||||
\"model\": \"frank-dragoon\",
|
||||
\"prompt\": \"$prompt_text\",
|
||||
\"max_tokens\": 128,
|
||||
\"temperature\": 0,
|
||||
@@ -220,7 +220,7 @@ run_single_test() {
|
||||
}
|
||||
|
||||
echo "╔══════════════════════════════════════════════════════════╗"
|
||||
echo "║ N-gram Mod Benchmark — frank-large ║"
|
||||
echo "║ N-gram Mod Benchmark — frank-dragoon ║"
|
||||
echo "║ STRIX ROCmFP4 · 64k context · two prompts ║"
|
||||
echo "╚══════════════════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
|
||||
@@ -15,9 +15,9 @@ import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
SERVICE = "frank-large"
|
||||
ENV_FILE = "/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ALIAS = "frank-large"
|
||||
SERVICE = "frank-dragoon"
|
||||
ENV_FILE = "/home/dank/serverAppConfigs/scripts/frank-dragoon.env"
|
||||
ALIAS = "frank-dragoon"
|
||||
DEFAULT_HOST = "localhost"
|
||||
DEFAULT_PORT = 7770
|
||||
DEFAULT_MAX_TOKENS = 16384
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
# Tunables for frank-large (mradermacher i1-Q4_K_M of
|
||||
# Tunables for frank-dragoon (mradermacher i1-Q4_K_M of
|
||||
# orcarouter/Qwen3.8-Flash-Next-Uncensored on ik_llama.cpp, Dankclaw,
|
||||
# RTX 3090 CUDA + Strix Halo CPU path, since 2026-09-15).
|
||||
#
|
||||
@@ -65,7 +65,7 @@ MMPROJ_PATH=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K
|
||||
CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
|
||||
|
||||
# Alias + network.
|
||||
ALIAS=frank-large
|
||||
ALIAS=frank-dragoon
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7770
|
||||
|
||||
+3
-3
@@ -1,5 +1,5 @@
|
||||
[Unit]
|
||||
Description=ik_llama.cpp Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M 177B MoE (dankclaw, frank-large)
|
||||
Description=ik_llama.cpp Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M 177B MoE (dankclaw, frank-dragoon)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
@@ -12,8 +12,8 @@ Group=dank
|
||||
# budget still needs room. Parity with the halogen quadlet's
|
||||
# Ulimit=memlock=-1:-1.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
|
||||
Environment="FRANK_DRAGOON_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-dragoon.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-dragoon.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
+3
-3
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve mradermacher/Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M as frank-large
|
||||
# Serve mradermacher/Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M as frank-dragoon
|
||||
# via ik_llama.cpp on Dankclaw.
|
||||
#
|
||||
# Topology (ikawrakow's hybrid recipe; upstream issue #2402 for the PLE):
|
||||
@@ -26,7 +26,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
@@ -66,7 +66,7 @@ fi
|
||||
# CPU-overridden weights mmapped (src/llama-load-tensors.cpp); the env file
|
||||
# sets it. Refuse to run without it.
|
||||
if [[ "${GGML_CUDA_NO_PINNED_WEIGHTS:-}" != "1" ]]; then
|
||||
echo "frank-large: refusing to start — GGML_CUDA_NO_PINNED_WEIGHTS=1 is mandatory on this host" >&2
|
||||
echo "frank-dragoon: refusing to start — GGML_CUDA_NO_PINNED_WEIGHTS=1 is mandatory on this host" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -112,7 +112,7 @@ fi
|
||||
: "${SPEC_DRAFT_POLL_BATCH:=1}"
|
||||
|
||||
# Text only: the server must not try to autoload the vision tower. Vision
|
||||
# stays on frank-large (Dankclaw), which is where the fleet routes images.
|
||||
# stays on frank-dragoon (Dankclaw), which is where the fleet routes images.
|
||||
NO_MMPROJ="${NO_MMPROJ:-1}"
|
||||
|
||||
export HIP_VISIBLE_DEVICES
|
||||
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
# apply-protected-cast.sh — RUNS ON DANKSERVER. Rewrite the recipe's
|
||||
# PROTECTED-BEGIN/END section from Swift's OWN calibration grams: run
|
||||
# derive-protected-experts.py over the calib dir, splice its cast lines into
|
||||
# the marked section of the recipe in place, keep the pre-edit recipe in the
|
||||
# campaign trash. Refuses to run twice (marker text says derived).
|
||||
set -euo pipefail
|
||||
CALIB=${1:-/home/dank/corpora/calib}
|
||||
RECIPE=${2:-/home/dank/serverAppConfigs-swift15/scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe}
|
||||
PY=/home/dank/venvs/radcorpus/bin/python
|
||||
DERIVE=/home/dank/serverAppConfigs-swift15/scripts/derive-protected-experts.py
|
||||
TS=$(date +%Y%m%d-%H%M%S)
|
||||
|
||||
grep -q "derive-protected-experts.py rewrote" "$RECIPE" && {
|
||||
echo "recipe already carries derived cast lines; refusing to re-derive" >&2; exit 1; }
|
||||
grep -q "PROTECTED-BEGIN" "$RECIPE" || { echo "no PROTECTED markers in $RECIPE" >&2; exit 1; }
|
||||
|
||||
"$PY" "$DERIVE" "$CALIB" > "/tmp/protected-cast-$TS.txt"
|
||||
N=$(grep -c 'cast dtype=bf16' "/tmp/protected-cast-$TS.txt")
|
||||
[ "$N" -ge 2 ] || { echo "derive produced $N cast lines; refusing" >&2; exit 1; }
|
||||
echo "derived $N protected experts -> /tmp/protected-cast-$TS.txt"
|
||||
|
||||
cp -f "$RECIPE" "/home/dank/.AGENTTRASH/$TS-qwen4exp-swift15-w4dn8-i8-hc8m.recipe"
|
||||
"$PY" - "$RECIPE" "/tmp/protected-cast-$TS.txt" <<'PYEOF'
|
||||
import sys
|
||||
recipe, castfile = sys.argv[1], sys.argv[2]
|
||||
lines = open(recipe).read().splitlines()
|
||||
cast = [l for l in open(castfile).read().splitlines() if 'cast dtype=bf16' in l]
|
||||
out, insec = [], False
|
||||
for l in lines:
|
||||
if 'PROTECTED-BEGIN' in l:
|
||||
insec = True
|
||||
out.append(l)
|
||||
out.extend(cast)
|
||||
elif 'PROTECTED-END' in l:
|
||||
insec = False
|
||||
out.append(l)
|
||||
out.append('# (derive-protected-experts.py rewrote this section from Swift\'s own grams)')
|
||||
elif not insec:
|
||||
out.append(l)
|
||||
open(recipe, 'w').write('\n'.join(out) + '\n')
|
||||
print(f'spliced {len(cast)} cast lines into {recipe}')
|
||||
PYEOF
|
||||
Executable
+141
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""autoresearch-bench.py — deterministic greedy battery for frank-dragoon.
|
||||
|
||||
Runs ON Dankclaw (no LAN jitter) against the candidate container on :7770.
|
||||
Fixed embedded corpus, temperature 0, streaming so decode speed is measured
|
||||
between real token arrivals (not request totals). Median of REPS samples per
|
||||
metric is the reported value; medians are far more stable than means under
|
||||
GPU thermal drift.
|
||||
|
||||
Metrics emitted (stdout, one per line):
|
||||
METRIC decode_tokens_per_sec=<float> primary, higher better
|
||||
METRIC prefill_tokens_per_sec=<float> secondary
|
||||
METRIC ttft_ms=<float> secondary, lower better
|
||||
Exit non-zero if the endpoint is unhealthy or the served alias is wrong.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
# Fixed corpus: deterministic, ~28k tokens after tokenization (same scale as
|
||||
# the 2026-10-01 bench window). Built by repeating a fixed paragraph block a
|
||||
# fixed number of times - NO random, NO clock, NO network.
|
||||
_BLOCK = (
|
||||
"The rendering pipeline walks the scene graph once per frame, resolving "
|
||||
"material bindings into descriptor sets before any draw call is recorded. "
|
||||
"Because the allocator recycles command buffers only after their fence "
|
||||
"signals, a submission that outlives its frame must pin the pool entry; "
|
||||
"otherwise the next frame overwrites recorded commands in flight. The "
|
||||
"validation layer flags exactly this class of lifetime bug, but only for "
|
||||
"buffers it can see - GPU-side timestamps and queue-family ownership "
|
||||
"transfers hide in the driver's own bookkeeping, which is why release "
|
||||
"builds must lean on explicit barriers and why a frame-time regression "
|
||||
"often shows up as corruption one frame later than its cause.\n"
|
||||
)
|
||||
LONG_CTX = _BLOCK * 220 # ~28k-token prefill, matching the deployed workload
|
||||
_LONG = LONG_CTX + "\nNow, in one focused paragraph (80-150 words), explain " \
|
||||
"which lifetime bugs this pipeline can still suffer and what explicit " \
|
||||
"mitigations the text implies."
|
||||
_SHORT = "Design a rate limiter for a homelab reverse proxy that must not " \
|
||||
"lock out the LAN. Think it through, then give the algorithm."
|
||||
|
||||
REPS = 3
|
||||
MAX_TOKENS = 256
|
||||
REQ_TIMEOUT = 900
|
||||
|
||||
|
||||
def stream_completion(url: str, model: str, prompt: str):
|
||||
"""Greedy streaming completion.
|
||||
|
||||
Returns (ttft_s, decode_tps, prefill_tps, prompt_tokens, n_tokens).
|
||||
ttft is RELATIVE to request send: time.perf_counter()'s epoch is host
|
||||
boot, and returning the raw stamp read as an 11-day first token.
|
||||
"""
|
||||
body = json.dumps({
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": MAX_TOKENS,
|
||||
"temperature": 0,
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
}).encode()
|
||||
req = urllib.request.Request(
|
||||
url.rstrip("/") + "/v1/chat/completions", data=body,
|
||||
headers={"Content-Type": "application/json"})
|
||||
t0 = time.perf_counter()
|
||||
first = None
|
||||
stamps = []
|
||||
usage = {}
|
||||
with urllib.request.urlopen(req, timeout=REQ_TIMEOUT) as resp:
|
||||
for raw in resp:
|
||||
line = raw.decode(errors="replace").strip()
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
payload = line[5:].strip()
|
||||
if payload == "[DONE]":
|
||||
break
|
||||
chunk = json.loads(payload)
|
||||
if chunk.get("usage"):
|
||||
usage = chunk["usage"]
|
||||
for ch in chunk.get("choices", []):
|
||||
d = ch.get("delta") or {}
|
||||
# reasoning AND content deltas both arrive token by token;
|
||||
# count either so greedy text length drives the rate.
|
||||
# (llama.cpp names the thinking field reasoning_content.)
|
||||
if d.get("content") or d.get("reasoning") or d.get("reasoning_content"):
|
||||
now = time.perf_counter()
|
||||
if first is None:
|
||||
first = now
|
||||
stamps.append(now)
|
||||
if first is None or len(stamps) < 2:
|
||||
# A zero-argmax degenerate answer would poison the median; raise.
|
||||
raise RuntimeError(f"stream produced {len(stamps)} tokens")
|
||||
ttft_s = first - t0
|
||||
decode_s = stamps[-1] - stamps[0]
|
||||
decode_tps = (len(stamps) - 1) / decode_s if decode_s > 0 else 0.0
|
||||
prompt_tokens = usage.get("prompt_tokens") or 0
|
||||
prefill_tps = prompt_tokens / ttft_s if prompt_tokens else 0.0
|
||||
return ttft_s, decode_tps, prefill_tps, prompt_tokens, len(stamps)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--url", default="http://127.0.0.1:7770")
|
||||
ap.add_argument("--model", default="frank-dragoon")
|
||||
args = ap.parse_args()
|
||||
|
||||
with urllib.request.urlopen(args.url.rstrip("/") + "/v1/models",
|
||||
timeout=10) as r:
|
||||
ids = [m["id"] for m in json.load(r)["data"]]
|
||||
if args.model not in ids:
|
||||
print(f"FATAL: served ids {ids}, expected {args.model}",
|
||||
file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
# Warmup (compile/graph warm + allocator settle), NOT measured.
|
||||
stream_completion(args.url, args.model, _SHORT)
|
||||
|
||||
decodes, prefills, ttfts = [], [], []
|
||||
for rep in range(REPS):
|
||||
for name, prompt in (("long", _LONG), ("short", _SHORT)):
|
||||
ttft, dt, pt, ptk, ntok = stream_completion(
|
||||
args.url, args.model, prompt)
|
||||
decodes.append(dt)
|
||||
ttfts.append(ttft * 1000.0)
|
||||
if name == "long":
|
||||
prefills.append(pt)
|
||||
print(f" rep{rep} {name:5s} ttft={ttft*1000:8.1f}ms "
|
||||
f"prefill={pt:8.1f} t/s decode={dt:6.2f} t/s "
|
||||
f"({ntok} tok, prompt {ptk})", file=sys.stderr)
|
||||
|
||||
med = lambda xs: statistics.median(xs)
|
||||
print(f"METRIC decode_tokens_per_sec={med(decodes):.2f}")
|
||||
print(f"METRIC prefill_tokens_per_sec={med(prefills):.1f}")
|
||||
print(f"METRIC ttft_ms={med(ttfts):.1f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+187
@@ -0,0 +1,187 @@
|
||||
#!/usr/bin/env bash
|
||||
# autoresearch-frank-archon.sh — RUNS ON DANKSERVER, driven over ssh by the
|
||||
# dev-box autoresearch.sh. It is the canonical frank-archon benchmark entry
|
||||
# point for the autoresearch loop.
|
||||
#
|
||||
# Method (mandated by the Benchmarking Policy): the production service serves
|
||||
# real users and shares the two R9700s, so a bench beside it starves the GPUs
|
||||
# and corrupts results. This script therefore:
|
||||
# 1. stops the production frank-archon unit (root, relink symlink: stop,
|
||||
# NOT mask — mask refuses to replace a symlinked unit file),
|
||||
# 2. boots a scratch radiance container named `arbench` on the SAME two
|
||||
# cards with the candidate engine argv (the env file the loop edits),
|
||||
# 3. polls /health until the listener is live (it binds AFTER the full
|
||||
# load, so a bare curl is an honest gate),
|
||||
# 4. runs ONE self-contained completion request locally (127.0.0.1, no LAN
|
||||
# jitter): a fixed ~140k-char real-text prompt (~32k tokens) decoded for
|
||||
# 256 tokens, greedy. The engine's `timings` object reports the honest
|
||||
# prefill and decode rates for that single request.
|
||||
# 5. prints METRIC lines and tears the scratch down,
|
||||
# 6. restores production (systemctl start), via a trap so a failed bench
|
||||
# never leaves frank-archon down (the stale-probe-tail lesson).
|
||||
#
|
||||
# Determinism: the prompt is built from a fixed, sorted corpus of repo files
|
||||
# with a fixed random seed, so prompt_n is stable across runs. Sampling is
|
||||
# greedy (temperature 0), so the decode length is fixed; the only residual
|
||||
# nondeterminism is the wht6 lossy TP all-reduce, which is a constant across
|
||||
# runs and cancels in A/B comparisons.
|
||||
#
|
||||
# The scratch mirrors the production container contract exactly (serve-frank-
|
||||
# primary.sh): --network host, --init, seccomp=unconfined (io_uring for the
|
||||
# n-gram table), memlock unbounded (the ~132 GiB pinned ledger), /dev/kfd +
|
||||
# /dev/dri, the .rad ro, the state dir rw, the tune cache mount. The ONLY
|
||||
# differences: the container name (arbench, so prod's identity is untouched)
|
||||
# and a FRESH prefix-cache disk tier (arbench/, wiped per run) so the cold
|
||||
# prefill is cold — the prod tier at kvcache/frank-archon is left intact.
|
||||
set -uo pipefail
|
||||
|
||||
ENV_FILE="${1:-/home/dank/serverAppConfigs/scripts/frank-archon.env}"
|
||||
SCRATCH=arbench
|
||||
PORT=7770
|
||||
HEALTH="http://127.0.0.1:$PORT/health"
|
||||
COMPLETE="http://127.0.0.1:$PORT/v1/completions"
|
||||
BODY=/tmp/arbench-body.json
|
||||
LOG=/home/dank/logs/arbench.log
|
||||
|
||||
restore() {
|
||||
podman rm -f "$SCRATCH" >/dev/null 2>&1 || true
|
||||
sudo systemctl start frank-archon >/dev/null 2>&1 || true
|
||||
}
|
||||
trap restore EXIT
|
||||
|
||||
# --- source the candidate config (same file the production unit reads) ------
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
echo "no env file at $ENV_FILE" >&2; exit 1
|
||||
fi
|
||||
# shellcheck disable=SC1090
|
||||
set -a; . "$ENV_FILE"; set +a
|
||||
|
||||
# --- stop production; the scratch needs both cards to itself ----------------
|
||||
sudo systemctl stop frank-archon >/dev/null 2>&1 || true
|
||||
podman rm -f "$SCRATCH" >/dev/null 2>&1 || true
|
||||
|
||||
# --- fresh prefix-cache tier for a cold prefill (prod's tier untouched) ------
|
||||
FRESH_KV="$FRANK_ARCHON_STATE_DIR/kvcache/$SCRATCH"
|
||||
rm -rf "$FRESH_KV"; mkdir -p "$FRESH_KV"
|
||||
mkdir -p "$FRANK_ARCHON_STATE_DIR/tune"
|
||||
|
||||
# --- boot the scratch: the production run line, scratch name + fresh KV ------
|
||||
podman run -d --name "$SCRATCH" \
|
||||
--network host \
|
||||
--init \
|
||||
--security-opt seccomp=unconfined \
|
||||
--ulimit memlock=-1:-1 \
|
||||
--device /dev/kfd --device /dev/dri \
|
||||
-v "$FRANK_ARCHON_MODEL_DIR":/models:ro \
|
||||
-v "$FRANK_ARCHON_STATE_DIR":/data \
|
||||
-v "$FRANK_ARCHON_STATE_DIR/tune":/opt/radiance/share/radiance/tune \
|
||||
-v "$FRANK_ARCHON_TEMPLATE":/templates/template.jinja:ro \
|
||||
"$FRANK_ARCHON_IMAGE" \
|
||||
--model "/models/$FRANK_ARCHON_MODEL_FILE" \
|
||||
--host :: --port "$PORT" \
|
||||
--served-model-name "$FRANK_ARCHON_SERVED_NAME" \
|
||||
--override-chat-template /templates/template.jinja \
|
||||
--tp "$FRANK_ARCHON_TP" --tp-wire "$FRANK_ARCHON_TP_WIRE" \
|
||||
--max-num-seqs "$FRANK_ARCHON_MAX_SEQS" \
|
||||
--max-model-len "$FRANK_ARCHON_MAX_MODEL_LEN" \
|
||||
--placement "$FRANK_ARCHON_PLACEMENT" \
|
||||
--host-pool-mib "$FRANK_ARCHON_HOST_POOL_MIB" \
|
||||
--gpu-headroom-mib "$FRANK_ARCHON_GPU_HEADROOM_MIB" \
|
||||
--vram-weights-mib "$FRANK_ARCHON_VRAM_WEIGHTS_MIB" \
|
||||
--vram-kv-mib "$FRANK_ARCHON_VRAM_KV_MIB" \
|
||||
--kv-cache-dtype "$FRANK_ARCHON_KV_DTYPE" \
|
||||
--prefix-cache-host-mib "$FRANK_ARCHON_PREFIX_HOST_MIB" \
|
||||
--prefix-cache-dir "/data/kvcache/$SCRATCH" \
|
||||
--prefix-cache-disk-mib "$FRANK_ARCHON_PREFIX_DISK_MIB" \
|
||||
--ngram-placement "$FRANK_ARCHON_NGRAM_PLACEMENT" \
|
||||
--num-speculative-tokens "$FRANK_ARCHON_SPEC_TOKENS" \
|
||||
--max-num-batched-tokens "$FRANK_ARCHON_MAX_BATCHED_TOKENS" \
|
||||
${FRANK_ARCHON_EXTRA_ARGS:+$FRANK_ARCHON_EXTRA_ARGS} \
|
||||
> "$LOG" 2>&1
|
||||
if [ $? -ne 0 ]; then echo "scratch boot failed; see $LOG" >&2; exit 1; fi
|
||||
|
||||
# --- poll health (listener binds after the full load; cold boot ~110 s) ------
|
||||
ok=0
|
||||
for _ in $(seq 1 360); do
|
||||
code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 3 "$HEALTH" 2>/dev/null || echo 000)
|
||||
if [ "$code" = "200" ]; then ok=1; break; fi
|
||||
sleep 2
|
||||
done
|
||||
if [ "$ok" -ne 1 ]; then
|
||||
echo "scratch never reached health 200; tail of $LOG:" >&2
|
||||
tail -30 "$LOG" >&2; exit 1
|
||||
fi
|
||||
|
||||
# --- build the workload: a fixed ~140k-char real-text prompt (repo corpus) ---
|
||||
python3 - "$BODY" <<'PY'
|
||||
import json, sys, random, glob
|
||||
random.seed(1234)
|
||||
root = "/home/dank/serverAppConfigs"
|
||||
files = sorted(
|
||||
glob.glob(f"{root}/serverAppConfigs-docs/**/*.md", recursive=True)
|
||||
+ glob.glob(f"{root}/scripts/*.sh")
|
||||
+ glob.glob(f"{root}/scripts/*.py")
|
||||
+ glob.glob(f"{root}/AGENTS.md")
|
||||
)
|
||||
corpus = []
|
||||
total = 0
|
||||
for f in files:
|
||||
try:
|
||||
t = open(f, encoding="utf-8", errors="ignore").read()
|
||||
except Exception:
|
||||
continue
|
||||
if not t.strip():
|
||||
continue
|
||||
corpus.append(t)
|
||||
total += len(t)
|
||||
if total > 240000:
|
||||
break
|
||||
corpus = "\n".join(corpus)
|
||||
TGT = 140000
|
||||
parts, n = [], 0
|
||||
while n < TGT:
|
||||
start = random.randint(0, max(0, len(corpus) - 500))
|
||||
seg = corpus[start:start + random.randint(200, 500)]
|
||||
parts.append(seg)
|
||||
n += len(seg) + 1
|
||||
prompt = " ".join(parts)
|
||||
json.dump(
|
||||
{"model": "frank-archon", "prompt": prompt,
|
||||
"max_tokens": 256, "temperature": 0.0, "stream": False,
|
||||
"stop": ["<|im_end|>"]},
|
||||
open(sys.argv[1], "w"),
|
||||
)
|
||||
PY
|
||||
if [ ! -f "$BODY" ]; then echo "prompt build failed" >&2; exit 1; fi
|
||||
|
||||
# --- the bench request (prefill ~32k + decode 256, greedy) -------------------
|
||||
RESP=$(curl -s --max-time 600 -X POST "$COMPLETE" \
|
||||
-H 'Content-Type: application/json' -d "@$BODY")
|
||||
echo "$RESP" > /tmp/arbench-resp.json
|
||||
|
||||
# --- parse the timings into METRIC lines -------------------------------------
|
||||
python3 - <<'PY'
|
||||
import json, sys
|
||||
try:
|
||||
r = json.load(open("/tmp/arbench-resp.json"))
|
||||
except Exception as e:
|
||||
sys.stderr.write(f"unparseable response: {e}\n"); sys.exit(1)
|
||||
if "timings" not in r:
|
||||
sys.stderr.write(f"no timings in response: {json.dumps(r)[:500]}\n"); sys.exit(1)
|
||||
t = r["timings"]
|
||||
dtps = t.get("predicted_per_second")
|
||||
ptps = t.get("prompt_per_second")
|
||||
pms = t.get("prompt_ms")
|
||||
dn = t.get("draft_n", 0)
|
||||
da = t.get("draft_n_accepted", 0)
|
||||
if dtps is None:
|
||||
sys.stderr.write("predicted_per_second null (no decode produced)\n"); sys.exit(1)
|
||||
print(f"METRIC decode_tps={dtps:.4f}")
|
||||
if ptps is not None:
|
||||
print(f"METRIC prefill_tps={ptps:.4f}")
|
||||
if pms is not None:
|
||||
print(f"METRIC ttft_ms={pms:.4f}")
|
||||
if dn:
|
||||
print(f"METRIC draft_accept={da/dn:.4f}")
|
||||
print(f"INFO prompt_n={t.get('prompt_n')} cache_n={t.get('cache_n')} predicted_n={t.get('predicted_n')}")
|
||||
PY
|
||||
Executable
+176
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env bash
|
||||
# frank-archon-arms.sh — run a SEQUENCE of frank-archon bench arms and keep
|
||||
# every arm's evidence. Runs on the operator box; each arm is one full window
|
||||
# of ./autoresearch.sh (stop+mask -> boot the arm as the service -> measure ->
|
||||
# restore production and verify), so the fleet is only ever down for one arm at
|
||||
# a time and the production unit is back up between arms.
|
||||
#
|
||||
# An arm is the repo's own frank-archon.env plus an OVERLAY of engine knobs
|
||||
# appended after it (later assignment wins when the serve script sources it),
|
||||
# staged through FRANK_ARCHON_AR_ENV_FILE so the checkout is never mutated and
|
||||
# each arm gets its own fingerprint (the memo can never answer one arm with
|
||||
# another arm's numbers).
|
||||
#
|
||||
# Usage: scripts/autoresearch/frank-archon-arms.sh [arm ...]
|
||||
# scripts/autoresearch/frank-archon-arms.sh --list
|
||||
# Output, under /tmp/frank-archon-arms-<UTC stamp>/ (override with ARMS_DIR):
|
||||
#
|
||||
# <arm>.out full autoresearch output (metrics + engine evidence)
|
||||
# <arm>.env the exact staged env
|
||||
# <arm>.rc window exit code
|
||||
# summary.tsv arm, rc, decode_tps, decode_tps_t0.7, kv_tokens, boot_s
|
||||
#
|
||||
# Exit 0 when every arm produced metrics; 1 if any arm failed (a failed boot is
|
||||
# a RESULT — the window still restored production, and the rc says which arm).
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO="$(cd -- "$SCRIPT_DIR/../.." && pwd)"
|
||||
BASE_ENV="$REPO/scripts/frank-archon.env"
|
||||
ARMS_DIR="${ARMS_DIR:-/tmp/frank-archon-arms-$(date -u +%Y%m%dT%H%M%SZ)}"
|
||||
|
||||
# --- arms ------------------------------------------------------------------
|
||||
# baseline: the shipped shape E, measured with THIS harness so the comparison is
|
||||
# matched (the ~29.6 t/s figure in the docs is an older shape, and no shape-E
|
||||
# decode number exists yet).
|
||||
OVER_baseline=()
|
||||
# B: MTP3 with the vendor's probabilistic draft sampling, a capture ladder whose
|
||||
# step is num_speculative_tokens + 1 = 4, and the sampling override. Precise
|
||||
# memory stays ON — this arm is simultaneously the MTP test and the test of
|
||||
# whether MTP's drafter workspace and the fitted 5.25 GiB overhead can coexist.
|
||||
SPEC3='{"method":"mtp","num_speculative_tokens":3,"draft_sample_method":"probabilistic"}'
|
||||
LADDER3='{"cudagraph_capture_sizes":[4,8,12],"max_cudagraph_capture_size":12,"inductor_compile_config":{"combo_kernels":false,"benchmark_combo_kernel":false}}'
|
||||
# The value must arrive in the staged env file SINGLE-QUOTED: sourcing an env
|
||||
# file performs quote removal, so an unquoted {"temperature":0.7} would reach
|
||||
# the engine as {temperature:0.7}.
|
||||
GEN07='{"temperature":0.7}'
|
||||
OVER_B=(
|
||||
"FRANK_ARCHON_SPEC='$SPEC3'"
|
||||
"FRANK_ARCHON_COMPCONFIG='$LADDER3'"
|
||||
'FRANK_ARCHON_GEN_CONFIG='"'$GEN07'"
|
||||
)
|
||||
# B2: identical to B with the default (7.60 GiB) overhead, i.e. precise memory
|
||||
# OFF. It is the fallback if B dies at graph capture, and it prices the arena
|
||||
# that precise memory buys (7,522 slots -> ~6,496).
|
||||
OVER_B2=(
|
||||
"FRANK_ARCHON_SPEC='$SPEC3'"
|
||||
"FRANK_ARCHON_COMPCONFIG='$LADDER3'"
|
||||
'FRANK_ARCHON_GEN_CONFIG='"'$GEN07'"
|
||||
'FRANK_ARCHON_PRECISE_MEMORY='
|
||||
)
|
||||
# C: the vendor's concurrency shape — SEQS 16 with the [4..32] ladder and util
|
||||
# 0.95. This is also the direct test of the theory that pinning the capture
|
||||
# ladder unlocks SEQS > 3 (the SEQS=2 failure was query_start_loc specialised to
|
||||
# the constant 3, which is NOT in the default ladder; 4 is).
|
||||
LADDER16='{"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32],"max_cudagraph_capture_size":32,"inductor_compile_config":{"combo_kernels":false,"benchmark_combo_kernel":false}}'
|
||||
OVER_C=(
|
||||
"FRANK_ARCHON_SPEC='$SPEC3'"
|
||||
"FRANK_ARCHON_COMPCONFIG='$LADDER16'"
|
||||
'FRANK_ARCHON_GEN_CONFIG='"'$GEN07'"
|
||||
'FRANK_ARCHON_SEQS=16'
|
||||
'FRANK_ARCHON_MNBT=4096'
|
||||
'FRANK_ARCHON_UTIL=0.95'
|
||||
)
|
||||
|
||||
# D/E: the CONTEXT arms. Measured 2026-10-02 (arms run 20261002T233855Z): the
|
||||
# KV pool is a hard byte request divided by a PER-TOKEN PRICE, and MTP raises
|
||||
# that price because the drafter keeps its own KV per position:
|
||||
# no spec: 3.15e9 / 398,658 tokens = 7,901 B/token (1.99x of 200,704)
|
||||
# MTP3: 3.15e9 / 313,002 tokens = 10,064 B/token (1.56x)
|
||||
# So arm C lost a fifth of the context to the drafter, not to util. Frank's
|
||||
# contract is >=401,400 tokens over >=2 seqs = 2.00x of 200,704, which at
|
||||
# 10,064 B/token costs 4.04e9 -> ask 4.05e9 (+0.83 GiB/rank).
|
||||
# D is arm B with EXACTLY ONE variable changed (the KV bytes), so its result
|
||||
# says whether the 4k-depth collapse (B 23.09 vs baseline 33.99) was KV or the
|
||||
# batch shape.
|
||||
KV401K='FRANK_ARCHON_KV_BYTES=4050000000'
|
||||
OVER_D=(
|
||||
"FRANK_ARCHON_SPEC='$SPEC3'"
|
||||
"FRANK_ARCHON_COMPCONFIG='$LADDER3'"
|
||||
'FRANK_ARCHON_GEN_CONFIG='"'$GEN07'"
|
||||
"$KV401K"
|
||||
)
|
||||
# E: D with C's max_num_batched_tokens (4096 — C is the only arm that did NOT
|
||||
# collapse at 4k depth: 33.15 where B gave 23.09) and util 0.99, the value
|
||||
# Frank asked about. 0.99 is affordable on paper — production at util 0.98
|
||||
# measures 30.37/34.21 GiB per card, 3.58 GiB physically unused — but it leaves
|
||||
# only 0.32 GiB above the allowance while the CUDA-graph cost is an ANALYTIC
|
||||
# estimate (C's was 1.47 GiB over 24 graphs), so it is measured, not assumed.
|
||||
OVER_E=(
|
||||
"FRANK_ARCHON_SPEC='$SPEC3'"
|
||||
"FRANK_ARCHON_COMPCONFIG='$LADDER3'"
|
||||
'FRANK_ARCHON_GEN_CONFIG='"'$GEN07'"
|
||||
"$KV401K"
|
||||
'FRANK_ARCHON_MNBT=4096'
|
||||
'FRANK_ARCHON_UTIL=0.99'
|
||||
)
|
||||
|
||||
arm_overlay() {
|
||||
case "$1" in
|
||||
baseline) ;; # the shipped env, no overlay
|
||||
B) printf '%s\n' "${OVER_B[@]}" ;;
|
||||
B2) printf '%s\n' "${OVER_B2[@]}" ;;
|
||||
C) printf '%s\n' "${OVER_C[@]}" ;;
|
||||
D) printf '%s\n' "${OVER_D[@]}" ;;
|
||||
E) printf '%s\n' "${OVER_E[@]}" ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
if [ "${1:-}" = "--list" ]; then
|
||||
printf 'baseline B B2 C D E\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[ -f "$BASE_ENV" ] || { echo "[arms] missing $BASE_ENV" >&2; exit 1; }
|
||||
if [ "$#" -gt 0 ]; then ARMS=("$@"); else ARMS=(baseline B B2 C); fi
|
||||
mkdir -p "$ARMS_DIR"
|
||||
echo "[arms] dir $ARMS_DIR arms: ${ARMS[*]}" >&2
|
||||
|
||||
: >"$ARMS_DIR/summary.tsv"
|
||||
printf 'arm\trc\tdecode_tps\tdecode_tps_t0.7\tkv_tokens\tboot_s\n' >>"$ARMS_DIR/summary.tsv"
|
||||
overall=0
|
||||
|
||||
for arm in "${ARMS[@]}"; do
|
||||
if ! arm_overlay "$arm" >"$ARMS_DIR/$arm.overlay"; then
|
||||
echo "[arms] unknown arm: $arm" >&2
|
||||
overall=1
|
||||
continue
|
||||
fi
|
||||
{ cat "$BASE_ENV"; echo; echo "# --- arm $arm overlay ($(date -u +%FT%TZ)) ---"
|
||||
arm_overlay "$arm"; } >"$ARMS_DIR/$arm.env"
|
||||
# Gate the staged env BEFORE burning a ~20-minute window: every knob that
|
||||
# holds JSON must still be one valid JSON object after the file is SOURCED
|
||||
# (that is the step that silently eats unquoted braces).
|
||||
if ! bash -c 'set -a; . "$1" >/dev/null; set +a
|
||||
for k in FRANK_ARCHON_SPEC FRANK_ARCHON_COMPCONFIG FRANK_ARCHON_GEN_CONFIG; do
|
||||
v="${!k:-}"
|
||||
[ -n "$v" ] || continue
|
||||
case "$v" in
|
||||
"{"*) python3 -c "import json,sys; json.loads(sys.argv[1])" "$v" \
|
||||
|| { echo "[arms] $k is not valid JSON: $v" >&2; exit 1; } ;;
|
||||
*) echo "[arms] $k is not a JSON object: $v" >&2; exit 1 ;;
|
||||
esac
|
||||
done' _ "$ARMS_DIR/$arm.env"; then
|
||||
echo "[arms] $arm: staged env rejected, window not opened" >&2
|
||||
echo 99 >"$ARMS_DIR/$arm.rc"
|
||||
overall=1
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "[arms] === $arm: opening window $(date -u +%H:%M:%S)Z ===" >&2
|
||||
FRANK_ARCHON_AR_ENV_FILE="$ARMS_DIR/$arm.env" \
|
||||
"$REPO/autoresearch.sh" >"$ARMS_DIR/$arm.out" 2>&1
|
||||
rc=$?
|
||||
echo "$rc" >"$ARMS_DIR/$arm.rc"
|
||||
echo "[arms] === $arm: rc=$rc $(date -u +%H:%M:%S)Z ===" >&2
|
||||
grep -E '^METRIC ' "$ARMS_DIR/$arm.out" | sed 's/^METRIC //' >"$ARMS_DIR/$arm.metrics" || true
|
||||
metric() { local v; v="$(sed -n "s/^$1=//p" "$ARMS_DIR/$arm.metrics" | tail -1)"; printf '%s' "${v:-$2}"; }
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$arm" "$rc" \
|
||||
"$(metric decode_tps FAIL)" "$(metric decode_tps_t0.7 FAIL)" \
|
||||
"$(metric kv_tokens ?)" "$(metric boot_seconds ?)" >>"$ARMS_DIR/summary.tsv"
|
||||
[ "$rc" -ne 0 ] && overall=1
|
||||
done
|
||||
|
||||
echo "[arms] summary:" >&2
|
||||
cat "$ARMS_DIR/summary.tsv" >&2
|
||||
exit "$overall"
|
||||
Executable
+216
@@ -0,0 +1,216 @@
|
||||
#!/usr/bin/env python3
|
||||
"""frank-archon decode measurement — runs INSIDE the engine container.
|
||||
|
||||
The method is the fleet's established one (the 2026-09-29 / 2026-10-01
|
||||
tuning campaigns, /home/dank/mtp_measure.py): SSE chunk counting is invalid
|
||||
under speculative decoding because rejected drafts change the accounting, so
|
||||
decode is measured from the server's own usage counters:
|
||||
|
||||
decode_tok_s = completion_tokens / (wall - ttft)
|
||||
|
||||
with ttft = wall time to the FIRST content-bearing chunk (thinking arrives in
|
||||
`reasoning`/`delta.reasoning` on this stack and is excluded from the decode
|
||||
window — and is deterministic per prompt at temperature 0, so the exclusion
|
||||
cancels in any A/B on the same workload).
|
||||
|
||||
Workload (fixed, deterministic, offline):
|
||||
* prompts built from a large fixed-seed shuffled word list so tokens rarely
|
||||
repeat (no n-gram/prefix predictability of the text itself), sized with
|
||||
the server's own /v1/messages/count_tokens counter
|
||||
* depths 115 / 1056 / 4096 target prompt tokens
|
||||
* one 64-token warm pass per depth (warms the expert arena), then
|
||||
max_tokens=256, 3 runs, median — repeated for every temperature in AR_TEMPS
|
||||
(default 0 and 0.7). The prompt workload is deterministic (fixed seed, no
|
||||
network, no time-of-day dependence); at temperature > 0 the generated text
|
||||
is not, but the measured window is bounded by max_tokens, so the rate stays
|
||||
comparable and the non-determinism is exactly the sampling that a
|
||||
speculative-decode arm must survive.
|
||||
|
||||
Env:
|
||||
AR_BASE endpoint (default http://127.0.0.1:7770)
|
||||
AR_MODEL served name (default frank-archon)
|
||||
AR_PROMPT_CEILING hard prompt ceiling in tokens (default 100000)
|
||||
AR_TEMPS comma-separated sampling temperatures (default "0,0.7")
|
||||
|
||||
Output: progress on stderr, ONE json object on stdout (the last line).
|
||||
Exit codes: 0 ok, 2 endpoint unhealthy / no usable completion, 3 bad usage.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
BASE = os.environ.get("AR_BASE", "http://127.0.0.1:7770")
|
||||
MODEL = os.environ.get("AR_MODEL", "frank-archon")
|
||||
PROMPT_CEILING = int(os.environ.get("AR_PROMPT_CEILING", "100000"))
|
||||
DEPTHS = (115, 1056, 4096)
|
||||
WARM_TOKENS = 64
|
||||
MAX_TOKENS = 256
|
||||
RUNS = 3
|
||||
# Temperature sweep. The unsuffixed keys are temperature 0 — byte-comparable to
|
||||
# every historical measurement — and AR_TEMPS adds more. The second one matters
|
||||
# for speculative decode: draft ACCEPTANCE is sampling-dependent (a greedy draft
|
||||
# matches a greedy target exactly and is rejected more under a non-greedy
|
||||
# target), so an MTP arm judged only at temp 0 would be judged on half the
|
||||
# contract. Production traffic is non-zero temperature (the payload's
|
||||
# generation_config says 1.0; an override to 0.7 is under test).
|
||||
TEMPS = tuple(float(x) for x in os.environ.get("AR_TEMPS", "0,0.7").split(","))
|
||||
|
||||
# 43-word x 12 = 516-word vocabulary, seed 1234 — the campaign's exact list
|
||||
# and seed, so prompts are byte-identical to the historical measurements.
|
||||
_VOCAB = ("alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima "
|
||||
"mike november oscar papa quebec romeo sierra tango uniform victor whiskey "
|
||||
"xray yankee zulu cardinal ordinal topology kernel scheduler resident "
|
||||
"unified aperture expert arena slab offload prefetch gemm triton quant "
|
||||
"tensor shard checkpoint manifest embed router cache page table window "
|
||||
"gradient sparse dense moe attention head dim block stream decode prefill "
|
||||
"bandwidth latency throughput residency eviction tier swap pin memlock").split()
|
||||
_R = random.Random(1234)
|
||||
_ORDER = list(range(len(_VOCAB)))
|
||||
_R.shuffle(_ORDER)
|
||||
|
||||
|
||||
def log(msg):
|
||||
print("[measure] %s" % msg, file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def post_json(path, body, timeout):
|
||||
req = urllib.request.Request(
|
||||
BASE + path, data=json.dumps(body).encode(),
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return json.load(r)
|
||||
|
||||
|
||||
def count_tokens(text):
|
||||
"""Token count without a prefill, from the server's own counter."""
|
||||
body = {"model": MODEL,
|
||||
"messages": [{"role": "user", "content": text}]}
|
||||
r = post_json("/v1/messages/count_tokens", body, timeout=300)
|
||||
n = r.get("input_tokens") or r.get("count") or r.get("token_count")
|
||||
if not n:
|
||||
raise RuntimeError("count_tokens returned no count: %r" % (r,))
|
||||
return int(n)
|
||||
|
||||
|
||||
def words(n):
|
||||
return " ".join(_VOCAB[_ORDER[i % len(_VOCAB)]] for i in range(n))
|
||||
|
||||
|
||||
def calibrate():
|
||||
"""Measure words->tokens from the server's own counter."""
|
||||
return 2000.0 / count_tokens(words(2000))
|
||||
|
||||
|
||||
def prompt_for(target_tokens, wpt):
|
||||
n = max(1, int(target_tokens * wpt))
|
||||
text = words(n)
|
||||
pt = count_tokens(text)
|
||||
while pt > PROMPT_CEILING:
|
||||
n = max(1, int(n * (PROMPT_CEILING / pt) * 0.98))
|
||||
text = words(n)
|
||||
pt = count_tokens(text)
|
||||
return text, pt
|
||||
|
||||
|
||||
def measured(prompt, max_tokens, temperature=0):
|
||||
"""Return (ttft_s, decode_tok_s, completion_tokens) via usage counters."""
|
||||
body = {"model": MODEL,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens, "temperature": temperature,
|
||||
"stream": True, "stream_options": {"include_usage": True}}
|
||||
req = urllib.request.Request(
|
||||
BASE + "/v1/chat/completions", data=json.dumps(body).encode(),
|
||||
headers={"Content-Type": "application/json"})
|
||||
t0 = time.time()
|
||||
ttft = None
|
||||
usage = {}
|
||||
with urllib.request.urlopen(req, timeout=1800) as r:
|
||||
for raw in r:
|
||||
line = raw.decode("utf-8", "replace").strip()
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
s = line[5:].strip()
|
||||
if s == "[DONE]":
|
||||
break
|
||||
try:
|
||||
ev = json.loads(s)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if ev.get("usage"):
|
||||
usage = ev["usage"]
|
||||
for ch in ev.get("choices", []):
|
||||
if (ch.get("delta") or {}).get("content"):
|
||||
if ttft is None:
|
||||
ttft = time.time() - t0
|
||||
wall = time.time() - t0
|
||||
ct = int(usage.get("completion_tokens") or 0)
|
||||
if ct <= 0:
|
||||
# No usage on the stream: the endpoint is unhealthy or the model
|
||||
# returned nothing (e.g. HTTP 200 with an error payload).
|
||||
raise RuntimeError("no completion_tokens in usage: %r" % (usage,))
|
||||
dec_window = max(wall - (ttft or 0.0), 1e-9)
|
||||
return (ttft or wall), (ct / dec_window), ct
|
||||
|
||||
|
||||
def fmt_temp(t):
|
||||
return ("%.1f" % t).rstrip("0").rstrip(".") if t else "0"
|
||||
|
||||
|
||||
def main():
|
||||
log("base=%s model=%s ceiling=%d temps=%s"
|
||||
% (BASE, MODEL, PROMPT_CEILING, ",".join(fmt_temp(t) for t in TEMPS)))
|
||||
try:
|
||||
wpt = calibrate()
|
||||
except Exception as e:
|
||||
log("endpoint unhealthy during calibration: %s" % e)
|
||||
return 2
|
||||
log("words/token: %.3f" % wpt)
|
||||
|
||||
result = {"words_per_token": round(wpt, 5)}
|
||||
for tgt in DEPTHS:
|
||||
try:
|
||||
p, pt = prompt_for(tgt, wpt)
|
||||
measured(p, WARM_TOKENS) # warm the arena and the prefix cache
|
||||
except Exception as e:
|
||||
log("warm pass failed at depth %d: %s" % (tgt, e))
|
||||
return 2
|
||||
for temp in TEMPS:
|
||||
suffix = "" if temp == 0 else "_t%s" % fmt_temp(temp)
|
||||
rates, tts, cts = [], [], []
|
||||
for i in range(RUNS):
|
||||
try:
|
||||
tt, r, ct = measured(p, MAX_TOKENS, temp)
|
||||
except Exception as e:
|
||||
log("run %d/%d failed at depth %d temp %s: %s"
|
||||
% (i + 1, RUNS, tgt, fmt_temp(temp), e))
|
||||
continue
|
||||
rates.append(r)
|
||||
tts.append(tt)
|
||||
cts.append(ct)
|
||||
if len(rates) < 2:
|
||||
log("only %d usable run(s) at depth %d temp %s — refusing to median"
|
||||
% (len(rates), tgt, fmt_temp(temp)))
|
||||
return 2
|
||||
result["decode_tps_%d%s" % (tgt, suffix)] = round(statistics.median(rates), 2)
|
||||
result["ttft_ms_%d%s" % (tgt, suffix)] = round(1000.0 * statistics.median(tts), 1)
|
||||
result["completion_tokens_%d%s" % (tgt, suffix)] = int(statistics.median(cts))
|
||||
result["prompt_tokens_%d%s" % (tgt, suffix)] = pt
|
||||
log("depth %5d tok temp %-4s decode %7.2f t/s ttft %.3f s (runs=%d)"
|
||||
% (pt, fmt_temp(temp), result["decode_tps_%d%s" % (tgt, suffix)],
|
||||
result["ttft_ms_%d%s" % (tgt, suffix)] / 1000.0, len(rates)))
|
||||
|
||||
for temp in TEMPS:
|
||||
suffix = "" if temp == 0 else "_t%s" % fmt_temp(temp)
|
||||
meds = [result["decode_tps_%d%s" % (t, suffix)] for t in DEPTHS]
|
||||
result["decode_tps%s" % suffix] = round(statistics.median(meds), 2)
|
||||
print(json.dumps(result))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+218
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env bash
|
||||
# frank-archon-window.sh — one full bench window for frank-archon, run ON
|
||||
# DANKSERVER as the deploy user (invoked over ssh by ./autoresearch.sh).
|
||||
#
|
||||
# Discipline (AGENTS.md Benchmarking Policy): the production unit is
|
||||
# STOPPED + MASKED first, the candidate config boots AS THE SERVICE itself
|
||||
# (same serve script, same container name, same port 7770 — no scratch
|
||||
# ports, no co-tenant GPU), the in-container measure runs, and the window
|
||||
# ALWAYS ends by restoring the production unit and verifying its health.
|
||||
# The EXIT trap makes the restore unconditional: a boot failure, a measure
|
||||
# failure, or a dead ssh session all end with production back up.
|
||||
#
|
||||
# Staging layout (created by the orchestrator, ephemeral — never a config
|
||||
# source; the repo checkout is):
|
||||
# /home/dank/.autoresearch-frank-archon/
|
||||
# serve.sh staged candidate serve script
|
||||
# frank-archon.env staged candidate env (the candidate knobs)
|
||||
# measure.py the in-container measure client
|
||||
# hash candidate fingerprint
|
||||
# run.json result (metrics + boot + kv + fingerprint)
|
||||
# serve.out/measure.out/fail.log
|
||||
# last/ memo: run.json + hash of the last FULLY
|
||||
# successful window (bench AND verified restore)
|
||||
#
|
||||
# This script logs to stderr and exits 0 only when the measure produced a
|
||||
# usable result (the restore happens in the trap regardless of the bench
|
||||
# outcome; a failed restore forces exit 1).
|
||||
set -uo pipefail
|
||||
|
||||
STAGE="${FRANK_ARCHON_AR_STAGE:-/home/dank/.autoresearch-frank-archon}"
|
||||
UNIT="${FRANK_ARCHON_AR_UNIT:-frank-archon}"
|
||||
CTR="${FRANK_ARCHON_AR_CTR:-frank-archon-qwen}"
|
||||
HEALTH_URL="http://127.0.0.1:7770/health"
|
||||
MODELS_URL="http://127.0.0.1:7770/v1/models"
|
||||
BOOT_TIMEOUT="${FRANK_ARCHON_AR_BOOT_TIMEOUT:-1200}"
|
||||
RESTORE_TIMEOUT="${FRANK_ARCHON_AR_RESTORE_TIMEOUT:-1200}"
|
||||
MEASURE_TIMEOUT="${FRANK_ARCHON_AR_MEASURE_TIMEOUT:-1800}"
|
||||
SERVED="$(sed -n 's/^FRANK_ARCHON_SERVED_NAME=//p' "$STAGE/frank-archon.env" 2>/dev/null | head -1)"
|
||||
SERVED="${SERVED:-frank-archon}"
|
||||
|
||||
RESTORED=0
|
||||
|
||||
log() { printf '[window %s] %s\n' "$(date '+%H:%M:%S')" "$*" >&2; }
|
||||
|
||||
container_state() { podman inspect -f '{{.State.Status}}' "$CTR" 2>/dev/null || echo absent; }
|
||||
|
||||
wait_port_free() {
|
||||
local i
|
||||
for i in $(seq 1 90); do
|
||||
[ "$(container_state)" != "running" ] && return 0
|
||||
sleep 2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_health() { # $1 = timeout seconds
|
||||
local t0 code
|
||||
t0=$(date +%s)
|
||||
while :; do
|
||||
code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$HEALTH_URL" 2>/dev/null || echo 000)"
|
||||
[ "$code" = "200" ] && return 0
|
||||
[ "$(( $(date +%s) - t0 ))" -ge "$1" ] && return 1
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
capture_bench_failure() {
|
||||
log "capturing bench-boot failure evidence"
|
||||
{
|
||||
echo "=== podman logs (tail 400) ==="
|
||||
podman logs --tail 400 "$CTR" 2>&1 || true
|
||||
echo "=== staged serve.out (tail 60) ==="
|
||||
tail -60 "$STAGE/serve.out" 2>/dev/null || true
|
||||
echo "=== engine log (tail 60) ==="
|
||||
tail -60 /home/dank/logs/frank_archon.log 2>/dev/null || true
|
||||
} >"$STAGE/fail.log" 2>&1 || true
|
||||
grep -E 'OutOfMemory|Tried to allocate|Engine core initialization failed|Error|error' \
|
||||
"$STAGE/fail.log" 2>/dev/null | tail -25 >&2 || true
|
||||
}
|
||||
|
||||
restore() {
|
||||
[ "$RESTORED" = 1 ] && return 0
|
||||
RESTORED=1
|
||||
log "restoring production unit $UNIT"
|
||||
if [ -f "$STAGE/serve.pid" ]; then
|
||||
kill "$(cat "$STAGE/serve.pid" 2>/dev/null)" 2>/dev/null || true
|
||||
fi
|
||||
podman stop -t 120 "$CTR" >/dev/null 2>&1 || true
|
||||
wait_port_free || log "WARNING: bench container still draining after 180s"
|
||||
sudo systemctl unmask "$UNIT" 2>/dev/null || true
|
||||
if ! sudo systemctl start "$UNIT"; then
|
||||
log "ERROR: systemctl start $UNIT failed — operator intervention required"
|
||||
return 1
|
||||
fi
|
||||
if wait_health "$RESTORE_TIMEOUT"; then
|
||||
if curl -s --max-time 10 "$MODELS_URL" 2>/dev/null | grep -q "$SERVED"; then
|
||||
log "production restored and verified ($SERVED answering on :7770)"
|
||||
# Memo promotion lives HERE: a cached result is only ever the
|
||||
# product of a fully successful window (bench AND verified
|
||||
# restore), never of a bare bench success.
|
||||
cp "$STAGE/run.json" "$STAGE/last/" 2>/dev/null || true
|
||||
cp "$STAGE/hash" "$STAGE/last/" 2>/dev/null || true
|
||||
else
|
||||
log "ERROR: health 200 but /v1/models does not list $SERVED"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
log "ERROR: production not healthy within ${RESTORE_TIMEOUT}s — unit left to systemd (Restart=always)"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
trap 'restore; rc=$?; [ "$rc" -ne 0 ] && exit 1' EXIT
|
||||
|
||||
main() {
|
||||
local f
|
||||
for f in serve.sh frank-archon.env measure.py hash; do
|
||||
[ -f "$STAGE/$f" ] || { log "staged file missing: $STAGE/$f"; exit 1; }
|
||||
done
|
||||
mkdir -p "$STAGE/last"
|
||||
rm -f "$STAGE/run.json" "$STAGE/fail.log" "$STAGE/measure.out" "$STAGE/evidence.log"
|
||||
|
||||
local t_start t0 i kvline kv_tokens kv_conc measure_json boot_seconds
|
||||
t_start="$(date +%s)"
|
||||
|
||||
log "stopping and masking production unit $UNIT"
|
||||
sudo systemctl stop "$UNIT" 2>/dev/null || true
|
||||
# Explicit stop is not auto-restarted even with Restart=always; the mask
|
||||
# is belt-and-braces against a stray start during the window.
|
||||
sudo systemctl mask "$UNIT" 2>/dev/null || \
|
||||
log "WARNING: mask failed — relying on explicit-stop semantics"
|
||||
wait_port_free || log "note: production container still draining"
|
||||
|
||||
t0="$(date +%s)"
|
||||
log "starting bench instance from staged config (fingerprint $(tr -d '[:space:]' <"$STAGE/hash"))"
|
||||
FRANK_ARCHON_ENV_FILE="$STAGE/frank-archon.env" \
|
||||
setsid nohup bash "$STAGE/serve.sh" >"$STAGE/serve.out" 2>&1 </dev/null &
|
||||
echo "$!" >"$STAGE/serve.pid"
|
||||
|
||||
for i in $(seq 1 45); do
|
||||
[ "$(container_state)" = "running" ] && break
|
||||
sleep 2
|
||||
done
|
||||
if [ "$(container_state)" != "running" ]; then
|
||||
log "bench container never reached running state"
|
||||
tail -40 "$STAGE/serve.out" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if wait_health "$BOOT_TIMEOUT"; then
|
||||
boot_seconds=$(( $(date +%s) - t0 ))
|
||||
log "bench healthy after ${boot_seconds}s"
|
||||
else
|
||||
if [ "$(container_state)" = "running" ]; then
|
||||
log "bench boot timed out after ${BOOT_TIMEOUT}s (still loading)"
|
||||
else
|
||||
log "bench container died during boot"
|
||||
capture_bench_failure
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "running measure in container (timeout ${MEASURE_TIMEOUT}s)"
|
||||
if ! podman cp "$STAGE/measure.py" "$CTR":/tmp/ar-measure.py; then
|
||||
log "podman cp of measure client failed"
|
||||
exit 1
|
||||
fi
|
||||
if ! timeout "$MEASURE_TIMEOUT" podman exec "$CTR" python3 /tmp/ar-measure.py \
|
||||
>"$STAGE/measure.out" 2>&1; then
|
||||
log "measure failed or timed out"
|
||||
tail -40 "$STAGE/measure.out" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Capacity facts for the run (context-size experiments change these).
|
||||
kvline="$(podman logs "$CTR" 2>&1 | grep -E 'GPU KV cache size' | tail -1 || true)"
|
||||
kv_tokens="$(printf '%s' "$kvline" | sed -E 's/.*GPU KV cache size: ([0-9,]+) tokens.*/\1/' | tr -d ',' || true)"
|
||||
kv_conc="$(printf '%s' "$kvline" | sed -E 's/.*per request: ([0-9.]+x).*/\1/' || true)"
|
||||
|
||||
# Human-facing evidence, in a sidecar file because it is multi-line prose,
|
||||
# not metrics: the expert-plan lines (the ONLY place the arena / runtime
|
||||
# overhead / KV-reserve numbers exist — they are never in /metrics), the
|
||||
# compile path (cold build vs cache hit), the graph-capture sizes, the
|
||||
# speculative-decode counters, and any OOM signature. Without this an arm
|
||||
# that loses decode speed is unexplainable afterwards.
|
||||
{
|
||||
echo "=== engine plan / capacity / spec evidence ==="
|
||||
podman logs "$CTR" 2>&1 | grep -E \
|
||||
'KV cache size|Maximum concurrency|layers split|expert cache|runtime overhead|non-default args|r4d all-reduce|Using \[|cudagraph|CUDA graph|Capturing|graph for compile range|torch.compile took|Directly load the compiled|speculative|Speculative|acceptance|Acceptance|offload|precise|OutOfMemory|Tried to allocate|Engine core initialization failed' \
|
||||
| tail -150 || true
|
||||
echo "=== /metrics counters (spec decode, caches, success) ==="
|
||||
curl -s --max-time 15 http://127.0.0.1:7770/metrics 2>/dev/null \
|
||||
| grep -E '^vllm:(spec_decode|gpu_cache|prefix_cache|request_success|generation_tokens|iteration)' \
|
||||
| head -80 || true
|
||||
} >"$STAGE/evidence.log" 2>&1 || true
|
||||
log "engine evidence -> $STAGE/evidence.log ($(wc -l <"$STAGE/evidence.log") lines)"
|
||||
|
||||
measure_json="$(grep -E '^\{' "$STAGE/measure.out" | tail -1 || true)"
|
||||
if [ -z "$measure_json" ]; then
|
||||
log "measure produced no JSON result line"
|
||||
tail -40 "$STAGE/measure.out" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The measure line is already JSON, so the run object can be assembled
|
||||
# without any host-side interpreter.
|
||||
printf '{"hash":"%s","boot_seconds":%s,"kv_tokens":%s,"kv_concurrency":"%s","total_seconds":%s,"measure":%s}\n' \
|
||||
"$(tr -d '[:space:]' <"$STAGE/hash")" \
|
||||
"$boot_seconds" "${kv_tokens:-0}" "${kv_conc:-unknown}" \
|
||||
"$(( $(date +%s) - t_start ))" "$measure_json" >"$STAGE/run.json"
|
||||
|
||||
log "bench window complete in $(( $(date +%s) - t_start ))s"
|
||||
# Memo only after a FULL success; the restore below still has to verify.
|
||||
exit 0
|
||||
}
|
||||
|
||||
main "$@"
|
||||
exit "$?"
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Decode throughput benchmark for frank-large.
|
||||
# Decode throughput benchmark for frank-dragoon.
|
||||
# Measures generation tok/s with 8K+ token prompts and 256 generated tokens.
|
||||
set -euo pipefail
|
||||
|
||||
@@ -57,7 +57,7 @@ run_bench() {
|
||||
# Warmup (small request)
|
||||
curl -s -m 60 http://127.0.0.1:$PORT/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model":"frank-large","messages":[{"role":"user","content":"Hi"}],"max_tokens":16,"stream":false}' > /dev/null 2>&1 || true
|
||||
-d '{"model":"frank-dragoon","messages":[{"role":"user","content":"Hi"}],"max_tokens":16,"stream":false}' > /dev/null 2>&1 || true
|
||||
|
||||
# Build 8K prompt
|
||||
local prompt
|
||||
@@ -75,7 +75,7 @@ print(json.dumps(' '.join(words)))
|
||||
t0=$(date +%s%N)
|
||||
resp=$(curl -s -m 600 http://127.0.0.1:$PORT/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"model\":\"frank-large\",\"messages\":[{\"role\":\"user\",\"content\":$prompt}],\"max_tokens\":$N_GEN,\"stream\":false}" 2>&1)
|
||||
-d "{\"model\":\"frank-dragoon\",\"messages\":[{\"role\":\"user\",\"content\":$prompt}],\"max_tokens\":$N_GEN,\"stream\":false}" 2>&1)
|
||||
t1=$(date +%s%N)
|
||||
|
||||
if ! echo "$resp" | python3 -c "import sys,json; json.load(sys.stdin)" 2>/dev/null; then
|
||||
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""bench-probe.py — RUNS ON DANKSERVER. The campaign's decode probe: one
|
||||
/completion request, a prompt assembled from calib.jsonl to ~TARGET tokens,
|
||||
temperature 0, max_tokens 512. Reports the engine's own timings: decode rate,
|
||||
prefill rate, draft accept. Run 3x; the median is the arm's number."""
|
||||
import json, sys, urllib.request
|
||||
|
||||
TARGET = 43600 # tokens
|
||||
BYTES_PER_TOK = 3.8
|
||||
URL = "http://localhost:7770/v1/completions"
|
||||
|
||||
def build_prompt():
|
||||
want = int(TARGET * BYTES_PER_TOK)
|
||||
parts = []
|
||||
n = 0
|
||||
with open("/home/dank/corpora/out/calib.jsonl") as f:
|
||||
for line in f:
|
||||
d = json.loads(line)
|
||||
p = d.get("prompt") or d.get("messages", [{}])[0].get("content", "")
|
||||
parts.append(p)
|
||||
n += len(p.encode())
|
||||
if n >= want:
|
||||
break
|
||||
return "".join(parts)
|
||||
|
||||
def probe(salt):
|
||||
prompt = f"[probe {salt}]\n" + build_prompt()
|
||||
body = json.dumps({
|
||||
"model": "frank-archon",
|
||||
"prompt": prompt, "temperature": 0.0, "max_tokens": 512,
|
||||
}).encode()
|
||||
req = urllib.request.Request(URL, data=body,
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=1800) as r:
|
||||
d = json.load(r)
|
||||
t = d.get("timings", {})
|
||||
return {
|
||||
"prompt_n": t.get("prompt_n"),
|
||||
"prefill_tps": round(t.get("prompt_per_second", 0), 1),
|
||||
"pred_n": t.get("predicted_n"),
|
||||
"decode_tps": round(t.get("predicted_per_second", 0), 2),
|
||||
"draft_n": t.get("draft_n"),
|
||||
"accept": (round(t["draft_n_accepted"] / t["draft_n"], 4)
|
||||
if t.get("draft_n") else None),
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(json.dumps(probe(sys.argv[1] if len(sys.argv) > 1 else "a")))
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
HTTP-based prefill-throughput sweep for frank-large.
|
||||
HTTP-based prefill-throughput sweep for frank-dragoon.
|
||||
Starts the server with a given --threads-batch value, sends a 64K-prompt via
|
||||
the chat API (max_tokens=0), measures prompt_tokens/wall_time.
|
||||
|
||||
@@ -39,7 +39,7 @@ def send_prefill_request():
|
||||
with open(PROMPT_FILE, "r") as f:
|
||||
prompt = f.read()
|
||||
body = json.dumps({
|
||||
"model": "frank-large",
|
||||
"model": "frank-dragoon",
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": 1,
|
||||
"stream": False
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Threads-batch sweep for frank-large on dankclaw
|
||||
# Threads-batch sweep for frank-dragoon on dankclaw
|
||||
# Benches prefill throughput at 65536 prompt tokens across threads-batch values.
|
||||
set -uo pipefail
|
||||
|
||||
@@ -10,7 +10,7 @@ TB_LIST=(16 32 48 64 80 96 128)
|
||||
ENV_FILE="/home/dank/GitPackages/ROCmFPX/build-qwen38/bin"
|
||||
|
||||
echo "========================================"
|
||||
echo "THREADS-BATCH SWEEP — frank-large"
|
||||
echo "THREADS-BATCH SWEEP — frank-dragoon"
|
||||
echo "Model: $(basename "$MODEL") ($(du -h "$MODEL" | cut -f1))"
|
||||
echo "Threads-batch values: ${TB_LIST[*]}"
|
||||
echo "Prompt: 65536 tokens, n-gen: 0 (prefill-only)"
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# Benchmark ubatch with llama-bench for frank-large.
|
||||
# Benchmark ubatch with llama-bench for frank-dragoon.
|
||||
#
|
||||
# This is intentionally a ubatch-only benchmark. It uses one llama-bench
|
||||
# process, an 8K prompt workload, and equal batch/ubatch values. No server
|
||||
# process, HTTP requests, or log scraping are used.
|
||||
#
|
||||
# Run on Dankclaw after stopping frank-large:
|
||||
# sudo systemctl stop frank-large
|
||||
# Run on Dankclaw after stopping frank-dragoon:
|
||||
# sudo systemctl stop frank-dragoon
|
||||
# ./scripts/bench-ubatch.sh
|
||||
# sudo systemctl start frank-large
|
||||
# sudo systemctl start frank-dragoon
|
||||
set -euo pipefail
|
||||
|
||||
: "${LLAMA_BENCH:=/home/dank/GitPackages/ROCmFPX/build-strix-rocmfp4/bin/llama-bench}"
|
||||
|
||||
Executable
+166
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env bash
|
||||
# build-frank-archon-rad.sh — build frank-archon's .rad container from the
|
||||
# OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 checkpoint, at NATIVE precision.
|
||||
#
|
||||
# bash scripts/build-frank-archon-rad.sh # convert
|
||||
# bash scripts/build-frank-archon-rad.sh --plan-only # declare, no I/O
|
||||
# bash scripts/build-frank-archon-rad.sh --restage # redo staging
|
||||
# bash scripts/build-frank-archon-rad.sh --force # rebuild existing
|
||||
#
|
||||
# WHY A STAGING DIR (the one non-obvious step): rad-convert selects its
|
||||
# architecture plugin by the checkpoint's quantisation DESCRIPTOR —
|
||||
# derived from config.json's quantization_config (core/format/checkpoint.cpp:
|
||||
# "fp8" here). No shipped plugin claims (qwen4exp, "fp8"): the shipped
|
||||
# qwen4exp plugin claims (qwen4exp, ""), the descriptor recipe-built
|
||||
# containers carry (they are built from the bf16 upstream, which has no
|
||||
# quantization_config at all). The match is exact with no fallback, so an
|
||||
# unmodified convert refuses before declare. The descriptor is only a
|
||||
# selection key; per-weight encodings are data — the checkpoint reader
|
||||
# derives them from the tensors themselves (X.weight_scale_inv is the
|
||||
# scale plane; checkpoint.h). Staging drops the quantization_config key
|
||||
# (everything else is a relative symlink into the pristine download) so
|
||||
# the convert selects the shipped plugin. EVERY weight is then copied through
|
||||
# in its own encoding — block-FP8 codes + bf16 scale planes stay block-FP8 —
|
||||
# except TWO value-preserving reshapes that stack-qwen38-official-fp8.py
|
||||
# performs on the staging dir first, and ONE recipe rule that finishes the
|
||||
# PLE table:
|
||||
# * experts: the plugin maps the STACKED layout (mlp.experts.gate_up_proj
|
||||
# [E,2F,H] with the _scale_inv plane at the same index) while the official
|
||||
# repo ships per-expert unfused tensors — staging byte-stacks them
|
||||
# (gate rows first, no requantisation).
|
||||
# * PLE: the plugin's fp8 branch wants the table codes beside a ONE-ELEMENT
|
||||
# bf16 scale plane; the official repo ships 128 SCALELESS fp8 shards plus
|
||||
# one bf16 scalar. Staging decodes to bf16 with the checkpoint's own
|
||||
# scale, and the convert's rtn rule re-encodes at that same scale —
|
||||
# every original code is restored exactly and the scale plane lands in
|
||||
# the form the plugin declares.
|
||||
#
|
||||
# WHY A CONTAINER AT ALL: a checkpoint dir has no mapped/file tiers, and a
|
||||
# weight only a host kernel reads (the n-gram / PLE table, 51B params)
|
||||
# cannot be served from one (GUIDE §3.4). rad-convert also bakes the
|
||||
# tokenizer and the checkpoint's chat template; the runtime replaces the
|
||||
# template with froggeric's (--override-chat-template).
|
||||
#
|
||||
# The convert runs rad-convert inside the digest-pinned radiance image
|
||||
# (the host has no ROCm and needs none): /home/dank/models is mounted at
|
||||
# /models so the staging dir's RELATIVE symlinks resolve inside the
|
||||
# container. Output lands beside the raw download, owned by dank
|
||||
# (rootless podman maps container-root to the invoking user).
|
||||
set -euo pipefail
|
||||
|
||||
SRC="/home/dank/models/Qwen/Qwen3.8-Flash-Next-FP8"
|
||||
STAGE="/home/dank/models/stage-qwen4exp-fp8"
|
||||
OUT_DIR="/home/dank/models/rad"
|
||||
OUT="$OUT_DIR/qwen4exp-fp8-native.rad"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/frank-archon.env"
|
||||
IMAGE="$FRANK_ARCHON_IMAGE"
|
||||
|
||||
plan_only=0 restage=0 force=0
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
--plan-only) plan_only=1 ;;
|
||||
--restage) restage=1 ;;
|
||||
--force) force=1 ;;
|
||||
*) echo "unknown flag: $a" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ ! -f "$SRC/config.json" ]; then
|
||||
echo "ERROR: raw checkpoint missing: $SRC (run scripts/download-frank-archon.sh --fetch)" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "verifying raw checkpoint first..."
|
||||
bash "$SCRIPT_DIR/download-frank-archon.sh"
|
||||
|
||||
if [ -f "$OUT" ] && [ "$plan_only" -eq 0 ] && [ "$force" -eq 0 ]; then
|
||||
echo "container already exists: $OUT"
|
||||
echo "rebuild it with --force (the old file moves to ~/.AGENTTRASH)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ---- staging ----------------------------------------------------------------
|
||||
if [ "$restage" -eq 1 ] && [ -e "$STAGE" ]; then
|
||||
TRASH="$HOME/.AGENTTRASH/$(date +%Y%m%d)-frank-archon-stage"
|
||||
mkdir -p "$TRASH"
|
||||
mv "$STAGE" "$TRASH/stage-$(date +%H%M%S)"
|
||||
echo "old staging dir moved to $TRASH"
|
||||
fi
|
||||
if [ ! -f "$STAGE/config.json" ]; then
|
||||
mkdir -p "$STAGE"
|
||||
python3 - "$SRC/config.json" "$STAGE/config.json" <<'PY'
|
||||
import json, sys
|
||||
src, dst = sys.argv[1], sys.argv[2]
|
||||
cfg = json.load(open(src))
|
||||
dropped = cfg.pop("quantization_config", None)
|
||||
json.dump(cfg, open(dst, "w"), indent=2)
|
||||
print(f"staging config.json written (quantization_config dropped: {dropped is not None})")
|
||||
PY
|
||||
# The index is COPIED like config.json, never symlinked: the reshape step
|
||||
# rewrites the stage's index, and a symlink would write through to the
|
||||
# raw checkpoint (measured 2026-10-05: the raw index was restamped with
|
||||
# the stacked layout and the raw verify failed 143/144). Shards stay
|
||||
# relative symlinks — relative so the targets resolve inside the convert
|
||||
# container (the stage's parent is mounted at /models).
|
||||
for f in "$SRC"/*; do
|
||||
name="$(basename "$f")"
|
||||
case "$name" in config.json|model.safetensors.index.json) continue ;; esac
|
||||
ln -sfn "../Qwen/Qwen3.8-Flash-Next-FP8/$name" "$STAGE/$name"
|
||||
done
|
||||
cp "$SRC/model.safetensors.index.json" "$STAGE/"
|
||||
echo "staging dir ready: $STAGE ($(ls "$STAGE" | wc -l) entries)"
|
||||
fi
|
||||
|
||||
# ---- checkpoint reshape (stacked experts + bf16 PLE table) -------------------
|
||||
# stack-qwen38-official-fp8.py reshapes the staged checkpoint in place
|
||||
# (idempotent) and writes the PLE scale to $STAGE/ple_scale.txt — see the
|
||||
# script's docstring for why each transform is value-preserving.
|
||||
echo "stacking experts + merging the PLE table (value-preserving)..."
|
||||
python3 "$SCRIPT_DIR/stack-qwen38-official-fp8.py" --stage "$STAGE"
|
||||
PLE_SCALE="$(tr -d '\n' < "$STAGE/ple_scale.txt")"
|
||||
echo "PLE fixed scale: $PLE_SCALE"
|
||||
|
||||
# ---- convert ----------------------------------------------------------------
|
||||
# rad-convert needs no GPU: declare resolves kernels and asks each about the
|
||||
# encoding it will read — CPU-side layout questions only. (If a plugin load
|
||||
# ever demands a card, add --device /dev/kfd --device /dev/dri to this run.)
|
||||
# --plan-only runs the same declare phase and exits before a byte is written.
|
||||
echo "rad-convert: $SRC (staged) -> $OUT"
|
||||
mkdir -p "$OUT_DIR"
|
||||
CONVERT_ARGS=(/models/stage-qwen4exp-fp8 -o /models/rad/qwen4exp-fp8-native.rad
|
||||
--home /opt/radiance/share/radiance)
|
||||
[ "$plan_only" -eq 1 ] && CONVERT_ARGS+=(--plan-only -v)
|
||||
# THE ONE RECIPE RULE (see header): the PLE table arrives from staging as a
|
||||
# bf16 merge of the checkpoint's own fp8 codes decoded with the checkpoint's
|
||||
# own scale; rtn re-encodes it at that same scale — the codes come back
|
||||
# exactly and the plugin's one-element scale plane is produced.
|
||||
CONVERT_ARGS+=(--quant "blk.*.ple_ngram.weight=rtn:codes=fp8_e4m3,block=*x*,scale=bf16,rule=fixed,scale_value=$PLE_SCALE")
|
||||
[ "$force" -eq 1 ] && [ -f "$OUT" ] && {
|
||||
TRASH="$HOME/.AGENTTRASH/$(date +%Y%m%d)-frank-archon-rad"
|
||||
mkdir -p "$TRASH"
|
||||
mv "$OUT" "$TRASH/qwen4exp-fp8-native.rad.$(date +%H%M%S)"
|
||||
echo "previous container moved to $TRASH"
|
||||
}
|
||||
podman run --rm \
|
||||
--entrypoint rad-convert \
|
||||
-v /home/dank/models:/models \
|
||||
"$IMAGE" "${CONVERT_ARGS[@]}"
|
||||
|
||||
if [ "$plan_only" -eq 1 ]; then
|
||||
echo "plan-only run complete (no container written)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ---- record -----------------------------------------------------------------
|
||||
echo "rad-info summary:"
|
||||
podman run --rm --entrypoint rad-info -v /home/dank/models:/models:ro \
|
||||
"$IMAGE" "/models/rad/qwen4exp-fp8-native.rad" | head -20
|
||||
MIB=$(( $(stat -c %s "$OUT") / 1048576 ))
|
||||
echo "container: $OUT (${MIB} MiB; env floor $FRANK_ARCHON_RAD_MIN_MIB MiB)"
|
||||
if [ "$MIB" -lt "$FRANK_ARCHON_RAD_MIN_MIB" ]; then
|
||||
echo "ERROR: container below the env floor — the convert was truncated" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "build complete"
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the gufo runtime that serves frank-large on Dankclaw.
|
||||
# Build the gufo runtime that serves frank-dragoon on Dankclaw.
|
||||
#
|
||||
# bash scripts/build-gufo.sh # build + install the pinned tag
|
||||
# bash scripts/build-gufo.sh --verify # only check the installed tree
|
||||
@@ -93,4 +93,4 @@ cmake --install build/release
|
||||
|
||||
echo "== verify =="
|
||||
"$PREFIX/bin/gufo" diagnose
|
||||
echo "frank-large runtime ready: $PREFIX/bin/gufo ($GUFO_TAG @ $commit)"
|
||||
echo "frank-dragoon runtime ready: $PREFIX/bin/gufo ($GUFO_TAG @ $commit)"
|
||||
|
||||
Executable
+89
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the ROCmFPX runtime that serves frank-dragoon on Dankclaw.
|
||||
#
|
||||
# bash scripts/build-rocmfpx.sh # clone/fetch + build the pin
|
||||
# bash scripts/build-rocmfpx.sh --verify # only check the installed tree
|
||||
#
|
||||
# Runtime: kingjones30/ROCmFPX — charlie12345/ROCmFPX plus the model-card
|
||||
# patches, i.e. the fork that owns the ROCmFPx tensor formats. frank-dragoon's
|
||||
# payload is ftype 115 (`Q8_0_ROCMFPX_AGENT`): 311 tensors of
|
||||
# GGML_TYPE_Q8_0_ROCMFPX (103), a type that exists ONLY in this fork, so
|
||||
# stock llama.cpp, gufo and every other loader on the host reject the file.
|
||||
# This is the same tree that served the Qwen3.8-Flash-Next ROCmFP4 STRIX
|
||||
# endpoint 2026-08-30 → 09-11 (see the kingjones runbook), rebuilt here so a
|
||||
# fresh host can be brought up from the repo alone.
|
||||
#
|
||||
# Why this commit and not a newer one: 36e9acd (2026-08-28) is what was
|
||||
# qualified against this payload on 2026-10-02 — it loads ftype 115, survives
|
||||
# warmup, and drafts with the MTP head at acceptance 0.65. The five legacy
|
||||
# builds of 2026-08-25 crashed this same file in ggml_compute_forward_set_rows
|
||||
# (see kingjones-q8agent-rocmfpx.md), so the pin is load-bearing: do not bump
|
||||
# it without re-running that crash test.
|
||||
#
|
||||
# Host notes:
|
||||
# * -DLLAMA_BUILD_WEBUI=OFF is REQUIRED on this box. The tree provisions its
|
||||
# embedded UI assets via npm or the HF llama-ui bucket; npm is absent here
|
||||
# and the bucket download half-failed, aborting the link at 98%. The API
|
||||
# server is unaffected.
|
||||
# * The build links against the host ROCm (/opt/rocm); no bundled toolchain
|
||||
# like the gfx1201 R9700 build needed.
|
||||
set -euo pipefail
|
||||
|
||||
ROCMFPX_REPO="${ROCMFPX_REPO:-https://github.com/kingjones30/ROCmFPX.git}"
|
||||
# The commit frank-dragoon was qualified with (measured 2026-10-02).
|
||||
ROCMFPX_COMMIT="${ROCMFPX_COMMIT:-36e9acd40e10a87cd3c3ef8ec734668757dc8520}"
|
||||
SRC_DIR="${ROCMFPX_SRC_DIR:-/home/dank/GitPackages/kingjones-ROCmFPX}"
|
||||
BUILD_DIR="${ROCMFPX_BUILD_DIR:-$SRC_DIR/build}"
|
||||
GPU_TARGET="${ROCMFPX_GPU_TARGET:-gfx1151}"
|
||||
JOBS="${JOBS:-$(nproc)}"
|
||||
|
||||
# check_pin — refuse a source tree that is not the qualified commit. A moved
|
||||
# fork means different kernels under the same path, and on this model line a
|
||||
# wrong build is a segfault at first request, not a clean error.
|
||||
check_pin() {
|
||||
local commit
|
||||
commit="$(git -C "$SRC_DIR" rev-parse HEAD 2>/dev/null || true)"
|
||||
if [[ -n "$commit" && "$commit" != "$ROCMFPX_COMMIT" ]]; then
|
||||
echo "ROCMFPX_COMMIT mismatch: $SRC_DIR is at $commit, pinned $ROCMFPX_COMMIT" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ "${1:-}" == "--verify" ]]; then
|
||||
check_pin
|
||||
[[ -x "$BUILD_DIR/bin/llama-server" ]] || {
|
||||
echo "no ROCmFPX llama-server at $BUILD_DIR/bin/llama-server" >&2
|
||||
exit 1
|
||||
}
|
||||
# The format the endpoint exists for: the fork must carry the ROCmFPx
|
||||
# tensor type, or the payload cannot load at all.
|
||||
"$BUILD_DIR/bin/llama-server" --help 2>&1 | grep -q -- '--spec-type' || {
|
||||
echo "$BUILD_DIR/bin/llama-server has no --spec-type: MTP drafting is impossible" >&2
|
||||
exit 1
|
||||
}
|
||||
"$BUILD_DIR/bin/llama-server" --version | head -1
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "== source: $ROCMFPX_REPO @ $ROCMFPX_COMMIT =="
|
||||
if [[ ! -d "$SRC_DIR/.git" ]]; then
|
||||
mkdir -p "$(dirname "$SRC_DIR")"
|
||||
git clone "$ROCMFPX_REPO" "$SRC_DIR"
|
||||
fi
|
||||
git -C "$SRC_DIR" fetch --quiet origin "$ROCMFPX_COMMIT" || true
|
||||
git -C "$SRC_DIR" checkout -q --detach "$ROCMFPX_COMMIT"
|
||||
check_pin
|
||||
echo " $(git -C "$SRC_DIR" rev-parse HEAD)"
|
||||
|
||||
echo "== configure + build (release, HIP $GPU_TARGET) =="
|
||||
cmake -S "$SRC_DIR" -B "$BUILD_DIR" \
|
||||
-DGGML_HIP=ON \
|
||||
-DGPU_TARGETS="$GPU_TARGET" \
|
||||
-DGGML_NATIVE=ON \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DLLAMA_BUILD_WEBUI=OFF
|
||||
cmake --build "$BUILD_DIR" --target llama-server llama-quantize -j "$JOBS"
|
||||
|
||||
echo "== verify =="
|
||||
"$BUILD_DIR/bin/llama-server" --version | head -1
|
||||
echo "frank-dragoon runtime ready: $BUILD_DIR/bin/llama-server ($ROCMFPX_COMMIT)"
|
||||
Executable
+110
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env python3
|
||||
"""calib-push.py — drive a calibration corpus through a radiance scratch
|
||||
server's prefill so its tap writes layer Hessians (docs/MOE-W4.md).
|
||||
|
||||
Runs ON DANKSERVER next to the scratch server (the one autoresearch- /
|
||||
serve-style scratch container booted with RADIANCE_CALIB_DIR set).
|
||||
Reads the corpus JSONL produced by make-calib-corpus.py
|
||||
({"prompt": "..."} a line), POSTs each prompt to /v1/completions with
|
||||
max_tokens=1 (prefill-only cost, one token out), at N-way concurrency
|
||||
matching the scratch server's --max-num-seqs budget.
|
||||
|
||||
Corpus rounds repeat until the engine prints the tap's completion line
|
||||
("calibration wrote layer Hessians") in `podman logs <container>` — that
|
||||
is the ground truth that RADIANCE_CALIB_TOKENS was reached. After that
|
||||
line, further prompts are pure waste, so the driver exits.
|
||||
|
||||
Mechanics notes:
|
||||
- requests carry no sampling params that matter (max_tokens=1, greedy).
|
||||
- a per-request timeout of 30 min: at ~1k tok/s prefill a 12k-token
|
||||
prompt is a few seconds; the timeout only catches a wedged engine.
|
||||
- failures print and continue: the tap pools over tokens, a few lost
|
||||
prompts do not bias the Hessians (the corpus is drawn round-robin over
|
||||
source classes, so loss is class-neutral as long as failures are rare;
|
||||
a systematic 4xx is loud in the log and must be fixed, not ignored).
|
||||
|
||||
Usage: calib-push.py [--port 7770] [--parallel 64] [--container arbench]
|
||||
[--corpus /home/dank/corpora/out/calib.jsonl] [--max-rounds 4]
|
||||
"""
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
DONE_MARK = "calibration wrote"
|
||||
|
||||
|
||||
def tap_done(container):
|
||||
"""True if the engine logged the tap's completion line."""
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["podman", "logs", "--tail", "50", container],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
return DONE_MARK in (out.stdout + out.stderr)
|
||||
|
||||
|
||||
def push_one(url, prompt):
|
||||
body = json.dumps({
|
||||
"model": "frank-archon",
|
||||
"prompt": prompt,
|
||||
"max_tokens": 1,
|
||||
"temperature": 0,
|
||||
"stream": False,
|
||||
}).encode()
|
||||
req = urllib.request.Request(
|
||||
url, data=body, headers={"Content-Type": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=1800) as r:
|
||||
r.read()
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"FAIL {str(e)[:120]}", flush=True)
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--port", type=int, default=7770)
|
||||
ap.add_argument("--parallel", type=int, default=64)
|
||||
ap.add_argument("--container", default="arbench")
|
||||
ap.add_argument("--corpus", default="/home/dank/corpora/out/calib.jsonl")
|
||||
ap.add_argument("--max-rounds", type=int, default=4)
|
||||
a = ap.parse_args()
|
||||
url = f"http://127.0.0.1:{a.port}/v1/completions"
|
||||
|
||||
prompts = []
|
||||
with open(a.corpus) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
prompts.append(json.loads(line)["prompt"])
|
||||
print(f"corpus: {len(prompts)} prompts", flush=True)
|
||||
|
||||
for rnd in range(1, a.max_rounds + 1):
|
||||
if tap_done(a.container):
|
||||
break
|
||||
t0 = time.time()
|
||||
ok = 0
|
||||
with concurrent.futures.ThreadPoolExecutor(a.parallel) as ex:
|
||||
for good in ex.map(lambda p: push_one(url, p), prompts):
|
||||
ok += 1 if good else 0
|
||||
dt = time.time() - t0
|
||||
print(f"round {rnd}: {ok}/{len(prompts)} ok in {dt:.0f}s "
|
||||
f"({sum(len(p) for p in prompts)/4/dt:.0f} chars/s est)",
|
||||
flush=True)
|
||||
if tap_done(a.container):
|
||||
break
|
||||
else:
|
||||
print("max rounds exhausted, tap not done", flush=True)
|
||||
sys.exit(1)
|
||||
print("TAP_DONE: calibration wrote layer Hessians", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+74
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env bash
|
||||
# convert-swift15-w4dn8.sh — RUNS ON DANKSERVER. The final recipe convert of
|
||||
# the Swift 1.5 checkpoint: gate_up experts through GPTQ against the 10M-token
|
||||
# calibration Grams, down projections block-fp8, protected experts bf16,
|
||||
# trunk int8, PLE fixed-scale fp8 (scripts/recipes/
|
||||
# qwen4exp-swift15-w4dn8-i8-hc8m.recipe).
|
||||
#
|
||||
# The convert engine is 1.0.13 — the SAME pin that will serve the result, so
|
||||
# the .rad crosses no version boundary. Only the Gram files cross versions
|
||||
# (1.2.0 wrote them; the RADGRAM2 magic, gram.r%d.%s.bin naming and the
|
||||
# calib_gram/calib_resolve/calib_stem symbols are byte-identical between the
|
||||
# two libquant.so). rad-convert runs on the host backend (no GPU visible in
|
||||
# the container) — production can stay up through this script.
|
||||
#
|
||||
# Prereqs (campaign order): calibration complete (calib-push printed TAP_DONE),
|
||||
# the protected-expert cast lines rewritten into the recipe by
|
||||
# derive-protected-experts.py, and the 335 GiB bf16 container deleted —
|
||||
# the peak-disk arithmetic needs that space (checkpoint 335.3 + output ~132
|
||||
# GiB on the volume; --reuse is NOT used, see the recipe header).
|
||||
set -euo pipefail
|
||||
|
||||
PIN_1013=sha256:4c3f484564bb2872784e46b0a59cbd71becc9c1e746e301a3596ffae253c322e
|
||||
IMAGE="stilldeadcode/radiance@$PIN_1013"
|
||||
CKPT=/home/dank/models/ukisai/Swift1.5-Qwen3.8-Flash-Next
|
||||
RECIPE=/home/dank/serverAppConfigs-swift15/scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe
|
||||
CALIB=/home/dank/corpora/calib
|
||||
OUT=/home/dank/models/rad/qwen4exp-swift15-w4dn8-i8-hc8m.rad
|
||||
LOG=/home/dank/logs/swift15_convert.log
|
||||
# gate_up ~37.6 + down ~18.8 + PLE 47.7 + trunk/embeddings/head ~10 GiB (all-w4 expert table;
|
||||
# the format assigns ONE encoding per expert table, so a per-tensor block-fp8
|
||||
# down tier cannot be served beside a w4 gate_up tier -- see the recipe header).
|
||||
MIN_MIB=${FRANK_SWIFT15_RAD_MIN_MIB:-108000}
|
||||
|
||||
# Gate 1: inputs. A missing Gram directory converts UNCALIBRATED with a
|
||||
# per-tensor stderr note, which is silently worse than failing here.
|
||||
[ -d "$CKPT" ] || { echo "no checkpoint at $CKPT" >&2; exit 1; }
|
||||
[ -f "$RECIPE" ] || { echo "no recipe at $RECIPE" >&2; exit 1; }
|
||||
GRAMS=$(find "$CALIB" -maxdepth 1 -name 'gram.r*.bin' 2>/dev/null | wc -l)
|
||||
[ "$GRAMS" -ge 48 ] || { echo "calib dir $CALIB holds $GRAMS gram files (<48)" >&2; exit 1; }
|
||||
if [ -e "$OUT" ]; then echo "$OUT already exists; move it to trash first" >&2; exit 1; fi
|
||||
if [ -e /home/dank/models/rad/qwen4exp-swift15-bf16.rad ]; then
|
||||
echo "the bf16 container is still on disk; delete it first (peak disk)" >&2; exit 1
|
||||
fi
|
||||
|
||||
# Gate 2: disk. Output ~132 GiB plus convert staging, 20 GiB of slack.
|
||||
FREE_MIB=$(df --output=avail -m /home/dank/models | tail -1 | tr -d ' ')
|
||||
[ "$FREE_MIB" -ge $((MIN_MIB + 20480)) ] || {
|
||||
echo "only ${FREE_MIB} MiB free on /home/dank/models" >&2; exit 1; }
|
||||
|
||||
# The recipe names $CALIB; rad-convert expands it from the environment
|
||||
# (MOE-W4.md:275). The recipe's cast section must name THIS checkpoint's
|
||||
# protected experts — derive-protected-experts.py rewrote it.
|
||||
echo "convert: $CKPT -> $OUT (gptq calib=$CALIB, $(date -Is))" | tee "$LOG"
|
||||
podman run --rm --name swift15-convert \
|
||||
-v /home/dank/models:/models \
|
||||
-v /home/dank/serverAppConfigs-swift15:/campaign:ro \
|
||||
-v "$CALIB:/data/calib:ro" \
|
||||
-e CALIB=/data/calib \
|
||||
--entrypoint /opt/radiance/bin/rad-convert \
|
||||
"$IMAGE" "/models/${CKPT#/home/dank/models/}" \
|
||||
--recipe /campaign/"${RECIPE#/home/dank/serverAppConfigs-swift15/}" \
|
||||
-o /models/rad/"$(basename "$OUT")" 2>&1 | tee -a "$LOG"
|
||||
rc=${PIPESTATUS[0]}
|
||||
[ "$rc" -eq 0 ] || { echo "rad-convert rc=$rc; log: $LOG" >&2; exit "$rc"; }
|
||||
|
||||
# Gate 3: the artifact. rad-info's recipe echo is the receipt that GPTQ ran.
|
||||
SIZE_MIB=$(( $(stat -c %s "$OUT") / 1048576 ))
|
||||
[ "$SIZE_MIB" -ge "$MIN_MIB" ] || {
|
||||
echo "$OUT is ${SIZE_MIB} MiB < floor ${MIN_MIB}" >&2; exit 1; }
|
||||
podman run --rm --entrypoint /opt/radiance/bin/rad-info \
|
||||
-v /home/dank/models:/models:ro "$IMAGE" /models/rad/"$(basename "$OUT")" \
|
||||
2>&1 | tee -a "$LOG" | grep -c 'gptq' >/dev/null \
|
||||
|| echo "WARNING: rad-info shows no gptq rule in the recorded recipe" >&2
|
||||
echo "CONVERT_DONE ${SIZE_MIB} MiB -> $OUT"
|
||||
@@ -57,9 +57,9 @@ else
|
||||
echo " none"
|
||||
fi
|
||||
|
||||
section "frank-primary unit"
|
||||
printf ' active: %s\n' "$(systemctl is-active frank-primary 2>/dev/null | head -1)"
|
||||
printf ' enabled: %s\n' "$(systemctl is-enabled frank-primary 2>/dev/null | head -1)"
|
||||
section "frank-archon unit"
|
||||
printf ' active: %s\n' "$(systemctl is-active frank-archon 2>/dev/null | head -1)"
|
||||
printf ' enabled: %s\n' "$(systemctl is-enabled frank-archon 2>/dev/null | head -1)"
|
||||
|
||||
section "glm serve tail"
|
||||
if podman ps -a --format '{{.Names}}' 2>/dev/null | grep -qx "$GLM_CTR"; then
|
||||
|
||||
Executable
+55
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python3
|
||||
"""deep-probe.py — one chat completion at ~43.6k prompt tokens, temp 0, 512
|
||||
generated, streamed; decode = completion_tokens / (total - ttft)."""
|
||||
import json, time, urllib.request
|
||||
|
||||
TARGET_BYTES = int(43600 * 3.8)
|
||||
|
||||
def build():
|
||||
parts, n = [], 0
|
||||
with open("/home/dank/corpora/out/calib.jsonl") as f:
|
||||
for line in f:
|
||||
p = json.loads(line)["prompt"]
|
||||
parts.append(p)
|
||||
n += len(p.encode())
|
||||
if n >= TARGET_BYTES:
|
||||
break
|
||||
return "".join(parts)
|
||||
|
||||
def main(salt):
|
||||
body = json.dumps({
|
||||
"model": "frank-archon",
|
||||
"messages": [{"role": "user", "content": f"[probe {salt}] Continue and summarize this material in English, 400 words.\n\n" + build()}],
|
||||
"temperature": 0.0, "max_tokens": 512,
|
||||
"stream": True, "stream_options": {"include_usage": True},
|
||||
}).encode()
|
||||
req = urllib.request.Request("http://localhost:7770/v1/chat/completions",
|
||||
data=body, headers={"Content-Type": "application/json"})
|
||||
t0 = time.time(); ttft = None; ctok = 0; usage = {}
|
||||
with urllib.request.urlopen(req, timeout=2400) as r:
|
||||
for raw in r:
|
||||
line = raw.decode().strip()
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
d = line[5:].strip()
|
||||
if d == "[DONE]":
|
||||
break
|
||||
j = json.loads(d)
|
||||
if j.get("usage"):
|
||||
usage = j["usage"]
|
||||
for ch in j.get("choices", []):
|
||||
delta = (ch.get("delta") or {})
|
||||
piece = delta.get("content") or delta.get("reasoning_content") or ""
|
||||
if piece and ttft is None:
|
||||
ttft = time.time() - t0
|
||||
ctok += 1
|
||||
total = time.time() - t0
|
||||
gen = usage.get("completion_tokens") or ctok
|
||||
dec = gen / (total - (ttft or 0)) if total > (ttft or 0) else 0
|
||||
print(json.dumps({"salt": salt, "prompt_tokens": usage.get("prompt_tokens"),
|
||||
"completion_tokens": gen, "ttft_s": round(ttft or 0, 2),
|
||||
"total_s": round(total, 1), "decode_tps": round(dec, 2)}))
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
main(sys.argv[1] if len(sys.argv) > 1 else "x")
|
||||
+23
-23
@@ -61,9 +61,9 @@ SERVICE_ORDER = (
|
||||
"sonarr",
|
||||
"lidarr",
|
||||
"readarr",
|
||||
"frank-large",
|
||||
"frank-small",
|
||||
"frank-primary",
|
||||
"frank-dragoon",
|
||||
"frank-zealot",
|
||||
"frank-archon",
|
||||
)
|
||||
|
||||
DANKCLAW_SERVICES = {
|
||||
@@ -93,8 +93,8 @@ DANKCLAW_SERVICES = {
|
||||
"authentik-outpost-ldap",
|
||||
"caddy",
|
||||
"homepage",
|
||||
"frank-large",
|
||||
"frank-small",
|
||||
"frank-dragoon",
|
||||
"frank-zealot",
|
||||
}
|
||||
|
||||
DANKSERVER_SERVICES = {
|
||||
@@ -113,7 +113,7 @@ DANKSERVER_SERVICES = {
|
||||
"sonarr",
|
||||
"lidarr",
|
||||
"readarr",
|
||||
"frank-primary",
|
||||
"frank-archon",
|
||||
"node-exporter",
|
||||
}
|
||||
|
||||
@@ -316,7 +316,7 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
if len(parts) >= 2 and parts[1] == "open-webui":
|
||||
_add(services, "open-webui")
|
||||
return services
|
||||
# frank-large left the quadlet shape on 2026-09-19 (CIRU v4.4.1 host
|
||||
# frank-dragoon left the quadlet shape on 2026-09-19 (CIRU v4.4.1 host
|
||||
# build, root unit): its files live in scripts/ and map there.
|
||||
if len(parts) >= 2 and parts[1] == "observability":
|
||||
# The five observability containers became Quadlet units on
|
||||
@@ -371,10 +371,10 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
if name == "node-exporter":
|
||||
_add(services, "node-exporter")
|
||||
return services
|
||||
# frank-primary's quadlet-era directory left the tree on 2026-09-26
|
||||
# frank-archon's quadlet-era directory left the tree on 2026-09-26
|
||||
# (the Lucebox DS4.1 unit was retired for the llama.cpp root unit —
|
||||
# the trio in scripts/ now maps it). Nothing lives under
|
||||
# dankserver/frank-primary/ except dated trash, and trash plans
|
||||
# dankserver/frank-archon/ except dated trash, and trash plans
|
||||
# nothing (the retire rule above the service blocks).
|
||||
return services
|
||||
|
||||
@@ -400,32 +400,32 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
# change to it redeploying alertmanager is the only path that
|
||||
# reaches it on the host.
|
||||
_add(services, "alertmanager")
|
||||
if name in {"frank-small.env", "frank-small.service", "serve-frank-small.sh"}:
|
||||
# llama.cpp frank-small (2026-09-16) keeps its unit, env and serve
|
||||
# files here; each one selects frank-small only. The ik_llama.cpp
|
||||
if name in {"frank-zealot.env", "frank-zealot.service", "serve-frank-zealot.sh"}:
|
||||
# llama.cpp frank-zealot (2026-09-16) keeps its unit, env and serve
|
||||
# files here; each one selects frank-zealot only. The ik_llama.cpp
|
||||
# trio that lived here for two days is in scripts/.AGENTTRASH/,
|
||||
# which is archive material and plans nothing.
|
||||
_add(services, "frank-small")
|
||||
if name in {"frank-large.env", "frank-large.service", "serve-frank-large.sh"}:
|
||||
# CIRU v4.4.1 frank-large (2026-09-19) keeps its unit, env and
|
||||
_add(services, "frank-zealot")
|
||||
if name in {"frank-dragoon.env", "frank-dragoon.service", "serve-frank-dragoon.sh"}:
|
||||
# CIRU v4.4.1 frank-dragoon (2026-09-19) keeps its unit, env and
|
||||
# serve files here the same way: a root unit on a host-built
|
||||
# runtime. The download script fetches the payload, which is
|
||||
# host-side state, so it selects nothing.
|
||||
_add(services, "frank-large")
|
||||
_add(services, "frank-dragoon")
|
||||
if name == "qwen-sharp-v22.4.0.jinja":
|
||||
# The Qwen-Sharp chat template frank-large loads via
|
||||
# The Qwen-Sharp chat template frank-dragoon loads via
|
||||
# --chat-template-file. It lived under dankserver/frank-medium/ until
|
||||
# the 2026-09-26 cutover retired that directory; editing it changes
|
||||
# what the live frank-large endpoint renders, so it plans
|
||||
# frank-large.
|
||||
_add(services, "frank-large")
|
||||
if name in {"frank-primary.env", "frank-primary.service", "serve-frank-primary.sh"}:
|
||||
# llama.cpp frank-primary (2026-09-26, replacing the Lucebox
|
||||
# what the live frank-dragoon endpoint renders, so it plans
|
||||
# frank-dragoon.
|
||||
_add(services, "frank-dragoon")
|
||||
if name in {"frank-archon.env", "frank-archon.service", "serve-frank-archon.sh"}:
|
||||
# llama.cpp frank-archon (2026-09-26, replacing the Lucebox
|
||||
# DS4.1 quadlet) keeps its unit, env and serve files here the
|
||||
# same way: a root unit on the unsloth b11030-mix prebuilt
|
||||
# serving Qwen3.8-Flash-Next Q8_0. The download script fetches
|
||||
# the payload, which is host-side state, so it selects nothing.
|
||||
_add(services, "frank-primary")
|
||||
_add(services, "frank-archon")
|
||||
# Deployment/test/benchmark scripts and other repo tooling do not
|
||||
# restart a runtime service.
|
||||
return services
|
||||
|
||||
+233
-161
@@ -90,12 +90,12 @@ service_is_allowed() {
|
||||
dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
||||
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
|
||||
dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
|
||||
dankclaw:frank-large|dankclaw:frank-small|\
|
||||
dankclaw:frank-dragoon|dankclaw:frank-zealot|\
|
||||
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
|
||||
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
|
||||
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
|
||||
dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
|
||||
dankserver:frank-primary|dankserver:node-exporter|\
|
||||
dankserver:frank-archon|dankserver:node-exporter|\
|
||||
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
|
||||
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
|
||||
frankbook-pro:readarr)
|
||||
@@ -192,7 +192,7 @@ add_pending_services() {
|
||||
continue
|
||||
fi
|
||||
[[ -n "$recorded" && "$recorded" != "$REVISION" ]] || continue
|
||||
if [[ "$TARGET" == dankserver && "$service" == frank-large ]]; then continue; fi
|
||||
if [[ "$TARGET" == dankserver && "$service" == frank-dragoon ]]; then continue; fi
|
||||
if ! changed="$(python3 scripts/deploy-plan.py --target "$TARGET" \
|
||||
--repo "$REMOTE_DIR" --base "$recorded" --head "$REVISION")"; then
|
||||
echo "WARN: unable to compare state for $service; retaining it as pending" >&2
|
||||
@@ -322,6 +322,12 @@ deploy_homepage() {
|
||||
if [[ -d shared/homepage/fonts ]]; then
|
||||
run_privileged cp -r shared/homepage/fonts /srv/homepage/
|
||||
fi
|
||||
if [[ -d shared/homepage/img ]]; then
|
||||
run_privileged cp -r shared/homepage/img /srv/homepage/
|
||||
fi
|
||||
if [[ -d shared/homepage/sfx ]]; then
|
||||
run_privileged cp -r shared/homepage/sfx /srv/homepage/
|
||||
fi
|
||||
local deployed_at
|
||||
deployed_at="$(git log -1 --format=%cI)"
|
||||
printf 'window.DEPLOYED_AT = "%s";\n' "$deployed_at" > /tmp/deployed.js
|
||||
@@ -935,15 +941,24 @@ deploy_plex() {
|
||||
fi
|
||||
systemctl --user restart plex.service
|
||||
# Readiness gate: /identity answers MediaContainer XML once PMS is up.
|
||||
#
|
||||
# HTTPS, NOT HTTP. PMS drops plain HTTP off-loopback: the probe's
|
||||
# connection arrives on the container's eth0, so an http:// request is
|
||||
# accepted and then answered with nothing (curl 52, Empty reply) against a
|
||||
# perfectly healthy server. Measured 2026-10-02 — the gate burned its full
|
||||
# 600s and failed a plex whose https://127.0.0.1:32400/identity answered
|
||||
# 200 in 44 ms the whole time. The same requirement is why the Caddyfile dials the
|
||||
# plex upstream over TLS (tests/caddyfile.bats pins it).
|
||||
# -k: PMS presents its own self-signed cert on 32400.
|
||||
#
|
||||
# Reality check from the 2026-09 cutover: the FIRST container boot of a
|
||||
# years-old library ran ~10 min (PMS pre-binds 32400 during boot and
|
||||
# empty-replies until the HTTP layer is ready), so the gate allows 600s.
|
||||
# years-old library ran ~10 min, so the gate still allows 600s.
|
||||
# Fail the deploy (pending state preserved) rather than mark it deployed blind.
|
||||
for ((i = 1; i <= 120; i++)); do
|
||||
curl -fsS --max-time 3 http://127.0.0.1:32400/identity >/dev/null && return 0
|
||||
curl -fsSk --max-time 3 https://127.0.0.1:32400/identity >/dev/null && return 0
|
||||
sleep 5
|
||||
done
|
||||
echo "FAIL: plex did not answer /identity within 600s" >&2
|
||||
echo "FAIL: plex did not answer /identity over HTTPS within 600s" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -1077,35 +1092,45 @@ deploy_anubis() {
|
||||
wait_container_healthy anubis 60 || return 1
|
||||
}
|
||||
|
||||
# frank-large is gufo (gufo-org/gufo), the Strix Halo inference engine built
|
||||
# from the pinned v0.1.1 tag by scripts/build-gufo.sh, serving
|
||||
# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL with the DFlash2 drafter
|
||||
# (z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M) on the Strix Halo iGPU. It is a
|
||||
# ROOT unit: the runtime is a native binary linked against the host ROCm,
|
||||
# so there is no container image to install and no generated unit to dodge.
|
||||
# relink_unit points /etc/systemd/system/frank-large.service at the repo
|
||||
# file and boot persistence is that unit's [Install] section.
|
||||
# frank-dragoon is the halogen-flash-server CONTAINER (0.16.2, digest-pinned
|
||||
# through scripts/frank-dragoon.env) serving the Quat3rnion ABLITERATED
|
||||
# Swift1.5 finetune of peonist's Qwen3.8-Flash-Next v2 checkpoint —
|
||||
# qwen38-flash-next-v2-swift15-abliterated.hgn (66,687,678,464 B, ~62 GiB
|
||||
# resident) + the REQUIRED n-gram lookup table + the vision tower —
|
||||
# MTP spec decode (HALOGEN_SPEC_ADAPT=0), 2 slots x 200192, image input —
|
||||
# on the Strix Halo iGPU. It stays a ROOT unit because the serve script
|
||||
# drives `podman run` directly (the frank-archon pattern): relink_unit
|
||||
# points /etc/systemd/system/frank-dragoon.service at the repo file, boot
|
||||
# persistence is that unit's [Install], and LimitMEMLOCK=infinity is
|
||||
# fleet parity. 2026-10-06 cutover from the halogen 27B-dense shape —
|
||||
# rollback = revert the swap commit's trio; the 27B payload stays on
|
||||
# disk (~/models/halogen-qwen3.8-27b).
|
||||
#
|
||||
# The runtime gate matters as much as the payload gate: gufo is built from
|
||||
# source on this host (there is no prebuilt release), so a machine whose
|
||||
# /home/dank/GitPackages tree was never built has a unit that can never
|
||||
# become healthy, and a green deploy would leave the endpoint dark.
|
||||
#
|
||||
# It replaces the stock llama.cpp prebuilt (unsloth b11030), which had
|
||||
# replaced the Kairic Edge runtime (ciru-ai/ROCmFPX). The port gate is the
|
||||
# same as every cutover since 2026-09-16: nothing starts until 7770 is
|
||||
# released.
|
||||
LLM_LARGE_RUNTIME="${LLM_LARGE_RUNTIME:-/home/dank/GitPackages/gufo-v0.1.1/bin/gufo}"
|
||||
LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Swift-1.5-Qwen3.8-27B-Uncensored-MTP-UD-Q8_K_XL/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf}"
|
||||
LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}"
|
||||
# The gates mirror serve-frank-dragoon.sh's predicates EXACTLY (the
|
||||
# 2026-10-01 crash-loop lesson): image digest in dank's podman storage,
|
||||
# each .hgn payload (checkpoint, n-gram table, vision tower) at the exact
|
||||
# byte count, the tokenizer directory holding every named file as a
|
||||
# regular (not symlinked) file — a green deploy against a missing
|
||||
# artifact would leave a restart-looping unit and a dark endpoint. The
|
||||
# port gate is the same as every cutover since 2026-09-16: nothing
|
||||
# starts until 7770 is released.
|
||||
LLM_DRAGOON_IMAGE="${LLM_DRAGOON_IMAGE:-ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84ac22308a53f5d1ca6b86806702d7039e5ebc51cae4c66621b92fe04a}"
|
||||
LLM_DRAGOON_CKPT="${LLM_DRAGOON_CKPT:-/home/dank/models/halogen-flashnext-v2/qwen38-flash-next-v2-swift15-abliterated.hgn}"
|
||||
LLM_DRAGOON_CKPT_BYTES="${LLM_DRAGOON_CKPT_BYTES:-66687678464}"
|
||||
LLM_DRAGOON_NGRAM="${LLM_DRAGOON_NGRAM:-/home/dank/models/halogen-flashnext-v2/qwen38-flash-next-ngram.hgn}"
|
||||
LLM_DRAGOON_NGRAM_BYTES="${LLM_DRAGOON_NGRAM_BYTES:-51200246144}"
|
||||
LLM_DRAGOON_VISION="${LLM_DRAGOON_VISION:-/home/dank/models/halogen-flashnext-v2/qwen38-flash-next-vision.hgn}"
|
||||
LLM_DRAGOON_VISION_BYTES="${LLM_DRAGOON_VISION_BYTES:-897916416}"
|
||||
LLM_DRAGOON_TOK_DIR="${LLM_DRAGOON_TOK_DIR:-/home/dank/models/halogen-flashnext-v2/tokenizer}"
|
||||
LLM_DRAGOON_TOK_FILES="${LLM_DRAGOON_TOK_FILES:-tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json}"
|
||||
|
||||
retire_halogen_large_quadlet() {
|
||||
retire_halogen_dragoon_quadlet() {
|
||||
local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
|
||||
systemctl --user stop frank-large.service 2>/dev/null || true
|
||||
if [[ -e "$quadlet_unit_dir/frank-large.container" ]]; then
|
||||
systemctl --user stop frank-dragoon.service 2>/dev/null || true
|
||||
if [[ -e "$quadlet_unit_dir/frank-dragoon.container" ]]; then
|
||||
mkdir -p "$trash"
|
||||
mv -- "$quadlet_unit_dir/frank-large.container" "$trash/frank-large.container"
|
||||
echo "retired quadlet frank-large.container -> $trash"
|
||||
mv -- "$quadlet_unit_dir/frank-dragoon.container" "$trash/frank-dragoon.container"
|
||||
echo "retired quadlet frank-dragoon.container -> $trash"
|
||||
fi
|
||||
systemctl --user daemon-reload
|
||||
}
|
||||
@@ -1129,155 +1154,185 @@ wait_port_free() {
|
||||
return 1
|
||||
}
|
||||
|
||||
deploy_llm_large() {
|
||||
# Refuse rather than restart-loop. A unit whose runtime or payload is
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the deploy log to say why.
|
||||
if [[ ! -x "$LLM_LARGE_RUNTIME" ]]; then
|
||||
echo "ERROR: gufo runtime absent at $LLM_LARGE_RUNTIME" >&2
|
||||
echo "frank-large: build it: bash scripts/build-gufo.sh" >&2
|
||||
deploy_llm_dragoon() {
|
||||
# Refuse rather than restart-loop; every gate below has a twin in
|
||||
# serve-frank-dragoon.sh and they must agree (2026-10-01 lesson).
|
||||
# Capture podman output before grepping: `podman images | grep -q`
|
||||
# races under pipefail (SIGPIPE 141 becomes "no match", 2026-09-30).
|
||||
if ! grep -qF "${LLM_DRAGOON_IMAGE#*@}" <<<"$(podman images --format '{{.RepoDigests}}' 2>/dev/null)"; then
|
||||
echo "ERROR: halogen image absent: $LLM_DRAGOON_IMAGE" >&2
|
||||
echo "frank-dragoon: pull it: podman pull ${LLM_DRAGOON_IMAGE%@*}" >&2
|
||||
return 1
|
||||
fi
|
||||
local missing=0 f
|
||||
for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "ERROR: payload absent at $f" >&2
|
||||
echo "frank-large: fetch it: bash scripts/download-frank-large.sh" >&2
|
||||
missing=1
|
||||
if [[ ! -f "$LLM_DRAGOON_CKPT" ]]; then
|
||||
echo "ERROR: halogen checkpoint absent at $LLM_DRAGOON_CKPT" >&2
|
||||
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "$(stat -c %s "$LLM_DRAGOON_CKPT")" != "$LLM_DRAGOON_CKPT_BYTES" ]]; then
|
||||
echo "ERROR: halogen checkpoint is not $LLM_DRAGOON_CKPT_BYTES bytes (partial download?)" >&2
|
||||
echo "frank-dragoon: re-check: bash scripts/download-frank-dragoon-flashnext.sh --verify-only" >&2
|
||||
return 1
|
||||
fi
|
||||
for gate in "$LLM_DRAGOON_NGRAM:$LLM_DRAGOON_NGRAM_BYTES:n-gram table" \
|
||||
"$LLM_DRAGOON_VISION:$LLM_DRAGOON_VISION_BYTES:vision tower"; do
|
||||
IFS=: read -r gpath gbytes gname <<<"$gate"
|
||||
if [[ ! -f "$gpath" ]]; then
|
||||
echo "ERROR: halogen $gname absent at $gpath" >&2
|
||||
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "$(stat -c %s "$gpath")" != "$gbytes" ]]; then
|
||||
echo "ERROR: halogen $gname is not $gbytes bytes (partial download?)" >&2
|
||||
echo "frank-dragoon: re-check: bash scripts/download-frank-dragoon-flashnext.sh --verify-only" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
if [[ -L "$LLM_DRAGOON_TOK_DIR" || ! -d "$LLM_DRAGOON_TOK_DIR" ]]; then
|
||||
echo "ERROR: tokenizer dir missing or a symlink at $LLM_DRAGOON_TOK_DIR" >&2
|
||||
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
for tf in $LLM_DRAGOON_TOK_FILES; do
|
||||
if [[ ! -f "$LLM_DRAGOON_TOK_DIR/$tf" || -L "$LLM_DRAGOON_TOK_DIR/$tf" ]]; then
|
||||
echo "ERROR: tokenizer file missing or a symlink: $LLM_DRAGOON_TOK_DIR/$tf" >&2
|
||||
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
[[ "$missing" -eq 0 ]] || return 1
|
||||
|
||||
retire_halogen_large_quadlet
|
||||
# On a re-deploy the predecessor is THIS unit: stop it before waiting on
|
||||
# the port, or the wait deadlocks on the server it just stopped serving.
|
||||
# Then sweep a stray manual run -- that is how a bench harness leaves the
|
||||
# port busy. Match the DRAFTER path, which every frank-large command
|
||||
# line carries via --dflash-model and which is UNCHANGED by the 2026-09-28
|
||||
# payload swap: the target moved to a different repository while the
|
||||
# drafter stayed put, so keying on the target filename would have quietly
|
||||
# stopped matching. It is also the right key in principle — it names this
|
||||
# endpoint's speculative drafter, not a binary. Match that, not a binary
|
||||
# name: frank-small on this host serves a different GGUF over llama-server
|
||||
# with no DFlash2 drafter and must not be killed. The pattern never
|
||||
# appears in the deploying shell's own command line, so -f is safe here
|
||||
# (it is not in `bash deploy-remote.sh`).
|
||||
run_privileged systemctl stop frank-large.service 2>/dev/null || true
|
||||
pkill -f 'Qwen3.8-27B-DFlash2-Q4_K_M' 2>/dev/null || true
|
||||
# Nothing may start until the predecessor has released the port. On
|
||||
# 2026-09-16 the cutover failed exactly here: the old server logged
|
||||
# "Deactivated successfully", the next engine tried to bind 7770 a second
|
||||
# later and reported "Address already in use". Gate on the port, not on
|
||||
# hope.
|
||||
retire_halogen_dragoon_quadlet
|
||||
# The predecessor on a re-deploy is THIS unit: stop it (ExecStop stops
|
||||
# the container) before waiting on the port. The stray sweep is keyed
|
||||
# on the FULL container name, which appears in the argv of a manual
|
||||
# `podman run --name frank-dragoon-halogen` / `podman wait` but NOT in
|
||||
# this deployer's own command line (`bash deploy-remote.sh
|
||||
# frank-dragoon`) — so the sweep can never match the deployer's own argv
|
||||
# (`pkill -f` kills the invoker when the pattern appears in the invoker's
|
||||
# own command line), and it can never match frank-zealot's llama-server.
|
||||
run_privileged systemctl stop frank-dragoon.service 2>/dev/null || true
|
||||
pkill -f 'frank-dragoon-halogen' 2>/dev/null || true
|
||||
podman rm -f frank-dragoon frank-dragoon-halogen >/dev/null 2>&1 || true
|
||||
# Nothing may start until the predecessor has released the port
|
||||
# (2026-09-16: "Deactivated successfully" was not "port free").
|
||||
wait_port_free 7770 || return 1
|
||||
relink_unit frank-large
|
||||
relink_unit frank-dragoon
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-large.service
|
||||
run_privileged systemctl restart frank-large.service
|
||||
wait_for_health frank-large http://127.0.0.1:7770/health 1200
|
||||
run_privileged systemctl enable frank-dragoon.service
|
||||
run_privileged systemctl restart frank-dragoon.service
|
||||
wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200
|
||||
}
|
||||
|
||||
# deploy_llm_small — frank-small is SGLang (python3.12 venv) on the RTX 3090,
|
||||
# as a ROOT unit: the runtime is a self-contained venv, so there is no
|
||||
# container image to install and no generated unit to avoid. relink_unit
|
||||
# points /etc/systemd/system/frank-small.service at the repo file and boot
|
||||
# deploy_llm_zealot — frank-zealot is llama.cpp (master build, CUDA sm_86) on
|
||||
# the RTX 3090 as a ROOT unit: the runtime is a native binary tree under
|
||||
# ~/GitPackages/llama-master/build-cuda, so there is no container image and
|
||||
# no generated unit to avoid. relink_unit points
|
||||
# /etc/systemd/system/frank-zealot.service at the repo file and boot
|
||||
# persistence is that unit's [Install] section.
|
||||
#
|
||||
# It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
|
||||
# not shared: frank-large's engine runs on the gfx1151 iGPU.
|
||||
LLM_SMALL_MODEL_DIR="${LLM_SMALL_MODEL_DIR:-/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound}"
|
||||
LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/sglang-cu128/bin/sglang}"
|
||||
# It is a SEPARATE endpoint on port 7771; frank-dragoon keeps 7770. The 3090 is
|
||||
# not shared: frank-dragoon's engine runs on the gfx1151 iGPU.
|
||||
LLM_ZEALOT_MODEL_FILE="${LLM_ZEALOT_MODEL_FILE:-/home/dank/models/Qwen35-9B-HauhauCS-Q6K/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K.gguf}"
|
||||
LLM_ZEALOT_RUNTIME="${LLM_ZEALOT_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
||||
# The projector IS the vision capability here: the model is natively
|
||||
# multimodal and the Q6_K file carries no vision tensors — the publisher ships
|
||||
# its own BF16 projector in the same repo, so a deploy that starts without
|
||||
# this file silently ships a text-only endpoint. Checked unconditionally, like
|
||||
# frank-dragoon's.
|
||||
LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Qwen35-9B-HauhauCS-Q6K/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf}"
|
||||
|
||||
deploy_llm_small() {
|
||||
deploy_llm_zealot() {
|
||||
# Refuse rather than restart-loop: a unit whose runtime or payload is
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the log to say why.
|
||||
if [[ ! -x "$LLM_SMALL_RUNTIME" ]]; then
|
||||
echo "ERROR: SGLang runtime absent at $LLM_SMALL_RUNTIME" >&2
|
||||
echo "frank-small: build the venv (python3.12 -m venv /home/dank/GitPackages/sglang-cu128" >&2
|
||||
echo "frank-small: and pip install 'sglang[all]' into it) first." >&2
|
||||
if [[ ! -x "$LLM_ZEALOT_RUNTIME" ]]; then
|
||||
echo "ERROR: llama.cpp runtime absent at $LLM_ZEALOT_RUNTIME" >&2
|
||||
echo "frank-zealot: build it (cmake -B build-cuda -DGGML_CUDA=ON" >&2
|
||||
echo "frank-zealot: -DCMAKE_CUDA_ARCHITECTURES=86 -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/gcc-13" >&2
|
||||
echo "frank-zealot: && cmake --build build-cuda --target llama-server) first." >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$LLM_SMALL_MODEL_DIR/model.safetensors.index.json" ]]; then
|
||||
echo "ERROR: payload index absent at $LLM_SMALL_MODEL_DIR" >&2
|
||||
echo "frank-small: run scripts/download-frank-small.sh first;" >&2
|
||||
echo "frank-small: refusing to start a unit that cannot load." >&2
|
||||
return 1
|
||||
fi
|
||||
local count
|
||||
count="$(find "$LLM_SMALL_MODEL_DIR" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l)"
|
||||
if [[ "$count" -ne 8 ]]; then
|
||||
echo "ERROR: payload shard count wrong: $LLM_SMALL_MODEL_DIR (found $count, want 8)" >&2
|
||||
return 1
|
||||
fi
|
||||
relink_unit frank-small
|
||||
for f in "$LLM_ZEALOT_MODEL_FILE" "$LLM_ZEALOT_MMPROJ"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "ERROR: frank-zealot payload file absent at $f" >&2
|
||||
echo "frank-zealot: run scripts/download-frank-zealot.sh first;" >&2
|
||||
echo "frank-zealot: refusing to start a unit that cannot load." >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
relink_unit frank-zealot
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-small.service
|
||||
run_privileged systemctl restart frank-small.service
|
||||
wait_for_health frank-small http://127.0.0.1:7771/health 900
|
||||
run_privileged systemctl enable frank-zealot.service
|
||||
run_privileged systemctl restart frank-zealot.service
|
||||
wait_for_health frank-zealot http://127.0.0.1:7771/health 900 || return 1
|
||||
# False-green hardening (frank-dragoon's lesson): the port answering
|
||||
# proves nothing about WHO answers — the unit must be active with a
|
||||
# live MainPID, or a stray SGLang holdover is masquerading as deployed.
|
||||
local main_pid
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-zealot.service)"
|
||||
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
|
||||
echo "FAIL: :7771 answered but frank-zealot.service has no MainPID — a stray process holds the port" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "frank-zealot: health answered by unit MainPID $main_pid"
|
||||
}
|
||||
|
||||
# deploy_llm_primary — frank-primary is the tcclaviger RDNA4 vLLM fork
|
||||
# (digest-pinned image, gfx1201) serving the orcarouter
|
||||
# Qwen3.8-Flash-Next-Uncensored-FP8 payload (131 safetensors shards) with
|
||||
# MTP4 spec decode and q8/q8 KV on the two R9700s as a ROOT unit whose serve
|
||||
# script drives `podman run` directly (the image entrypoint execs
|
||||
# `vllm serve`). 2026-09-29 cutover from the R9V radiance fork, whose
|
||||
# launcher/shim/setup-state machinery is retired (state dirs stay on disk as
|
||||
# rollback; pre-cutover tag frank-primary-r9v-q8-20260929). Same port and
|
||||
# alias as the endpoints it replaced; the harness models.yml id is unchanged.
|
||||
# deploy_llm_archon — frank-archon is the radiance engine (StillDeadcode,
|
||||
# digest-pinned stilldeadcode/radiance image) serving the OFFICIAL
|
||||
# Qwen/Qwen3.8-Flash-Next-FP8 at native block-FP8 precision from the
|
||||
# rad-convert container qwen4exp-fp8-native.rad, on BOTH R9700s (TP=2) as a
|
||||
# ROOT unit whose serve script drives `podman run` directly (the engine is
|
||||
# the image entrypoint; the image lives in dank's rootless podman storage).
|
||||
# 2026-10-05 cutover from FreeToken (pre-cutover tag
|
||||
# frank-archon-freetoken-pr587-20261005); before that, the tcclaviger vLLM
|
||||
# fork (tag frank-archon-vllm-shapeG-20261003). Same port and alias; the
|
||||
# harness models.yml id is unchanged.
|
||||
#
|
||||
# Gates: the pinned runtime image must be loadable from dank's rootless
|
||||
# storage by its OCI manifest digest (the env file's FRANK_PRIMARY_IMAGE),
|
||||
# the payload dir must carry config.json and the expected shard count
|
||||
# (FRANK_PRIMARY_MODEL_SHARDS), and the unit env must be complete. Per-file
|
||||
# sha256 verification was the fetch script's job at download time, never
|
||||
# Gates: the image must exist under its digest ref in dank's rootless
|
||||
# storage (FRANK_ARCHON_IMAGE), the .rad container file must exist at the
|
||||
# env's size floor (FRANK_ARCHON_MODEL_FILE >= FRANK_ARCHON_RAD_MIN_MIB —
|
||||
# a truncated convert is a guaranteed long-timeout boot), and the unit env
|
||||
# must be complete. Per-file sha256 verification is the fetch script's job
|
||||
# at download time and rad-convert/rad-info's job at build time, never
|
||||
# re-hashed here.
|
||||
LLM_PRIMARY_CONTAINER=frank-primary-qwen
|
||||
LLM_PRIMARY_LEGACY_CONTAINERS="frank-primary-r9v qwen-fp8 glm-probe"
|
||||
LLM_ARCHON_CONTAINER=frank-archon
|
||||
LLM_ARCHON_LEGACY_CONTAINERS="frank-archon-qwen frank-archon-r9v qwen-fp8 glm-probe"
|
||||
|
||||
deploy_llm_primary() {
|
||||
deploy_llm_archon() {
|
||||
# Refuse rather than restart-loop: a unit whose image or payload is
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the log to say why.
|
||||
local env_file="/home/dank/serverAppConfigs/scripts/frank-primary.env"
|
||||
local image model_dir shards count
|
||||
local env_file="/home/dank/serverAppConfigs/scripts/frank-archon.env"
|
||||
local image model_dir model_file rad_min mib
|
||||
if [[ ! -f "$env_file" ]]; then
|
||||
echo "ERROR: frank-primary env file absent: $env_file" >&2
|
||||
echo "ERROR: frank-archon env file absent: $env_file" >&2
|
||||
return 1
|
||||
fi
|
||||
image="$(sed -n 's/^FRANK_PRIMARY_IMAGE=//p' "$env_file" | tail -1)"
|
||||
model_dir="$(sed -n 's/^FRANK_PRIMARY_MODEL_DIR=//p' "$env_file" | tail -1)"
|
||||
shards="$(sed -n 's/^FRANK_PRIMARY_MODEL_SHARDS=//p' "$env_file" | tail -1)"
|
||||
if [[ -z "$image" || -z "$model_dir" || -z "$shards" ]]; then
|
||||
echo "ERROR: frank-primary.env missing FRANK_PRIMARY_IMAGE/MODEL_DIR/MODEL_SHARDS" >&2
|
||||
image="$(sed -n 's/^FRANK_ARCHON_IMAGE=//p' "$env_file" | tail -1)"
|
||||
model_dir="$(sed -n 's/^FRANK_ARCHON_MODEL_DIR=//p' "$env_file" | tail -1)"
|
||||
model_file="$(sed -n 's/^FRANK_ARCHON_MODEL_FILE=//p' "$env_file" | tail -1)"
|
||||
rad_min="$(sed -n 's/^FRANK_ARCHON_RAD_MIN_MIB=//p' "$env_file" | tail -1)"
|
||||
if [[ -z "$image" || -z "$model_dir" || -z "$model_file" || -z "$rad_min" ]]; then
|
||||
echo "ERROR: frank-archon.env missing FRANK_ARCHON_IMAGE/MODEL_DIR/MODEL_FILE/RAD_MIN_MIB" >&2
|
||||
return 1
|
||||
fi
|
||||
# Capture podman's output BEFORE grepping: under set -o pipefail, the
|
||||
# classic `podman images | grep -q` races (grep exits on match, podman
|
||||
# dies SIGPIPE 141 writing the rest, pipefail calls it no-match — the
|
||||
# 2026-09-30 deploy refusal).
|
||||
local digests
|
||||
digests="$(podman images --format '{{.RepoDigests}}')"
|
||||
if ! grep -qF "${image#*@}" <<<"$digests"; then
|
||||
echo "ERROR: pinned runtime image absent from rootless storage: $image" >&2
|
||||
echo "frank-primary: fetch it (podman pull $image)" >&2
|
||||
# The image gate is the digest ref itself: a name@sha256 ref exists in
|
||||
# storage only if THAT manifest was pulled. One command — no
|
||||
# pipefail/grep race (the 2026-09-30 lesson).
|
||||
if ! podman image exists "$image"; then
|
||||
echo "ERROR: runtime image absent from storage: $image" >&2
|
||||
echo "frank-archon: podman pull docker://stilldeadcode/radiance:latest" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$model_dir/config.json" ]]; then
|
||||
echo "ERROR: payload config missing: $model_dir/config.json" >&2
|
||||
echo "frank-primary: re-run scripts/download-frank-primary.sh" >&2
|
||||
if [[ ! -f "$model_dir/$model_file" ]]; then
|
||||
echo "ERROR: .rad container missing: $model_dir/$model_file" >&2
|
||||
echo "frank-archon: run scripts/build-frank-archon-rad.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$model_dir/model.safetensors.index.json" ]]; then
|
||||
echo "ERROR: weight-map index missing: $model_dir/model.safetensors.index.json" >&2
|
||||
echo "frank-primary: re-run scripts/download-frank-primary.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
count="$(find "$model_dir" -maxdepth 1 -name 'model-*-of-*.safetensors' 2>/dev/null | wc -l)"
|
||||
if [[ "$count" != "$shards" ]]; then
|
||||
echo "ERROR: payload shard count wrong: $model_dir (found $count, want $shards)" >&2
|
||||
echo "frank-primary: re-run scripts/download-frank-primary.sh" >&2
|
||||
mib=$(( $(stat -c %s "$model_dir/$model_file") / 1048576 ))
|
||||
if [[ "$mib" -lt "$rad_min" ]]; then
|
||||
echo "ERROR: .rad container too small: $model_dir/$model_file is ${mib} MiB, want >= $rad_min" >&2
|
||||
echo "frank-archon: rebuild it (scripts/build-frank-archon-rad.sh --force)" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
@@ -1289,33 +1344,50 @@ deploy_llm_primary() {
|
||||
# The legacy identities (the retired R9V container, the ad hoc
|
||||
# experiment container qwen-fp8, the retired glm-probe) hold the same
|
||||
# GPUs and die here too.
|
||||
run_privileged systemctl stop frank-primary.service 2>/dev/null || true
|
||||
run_privileged systemctl stop frank-archon.service 2>/dev/null || true
|
||||
local ctr
|
||||
for ctr in $LLM_PRIMARY_LEGACY_CONTAINERS "$LLM_PRIMARY_CONTAINER"; do
|
||||
for ctr in $LLM_ARCHON_LEGACY_CONTAINERS "$LLM_ARCHON_CONTAINER"; do
|
||||
podman rm -f "$ctr" >/dev/null 2>&1 || true
|
||||
done
|
||||
# Nothing starts until the port is released: the lucebox cutover
|
||||
# (2026-09-26) and the frank-large cutover (2026-09-16) both failed
|
||||
# (2026-09-26) and the frank-dragoon cutover (2026-09-16) both failed
|
||||
# exactly here when the next engine bound 7770 a second too early.
|
||||
wait_port_free 7770 || return 1
|
||||
relink_unit frank-primary
|
||||
relink_unit frank-archon
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-primary.service
|
||||
run_privileged systemctl restart frank-primary.service
|
||||
# Cold start: FP8 page-in plus torch compile/graph capture (~15 min cold,
|
||||
# measured 2026-09-29). Parity with every predecessor on this port.
|
||||
wait_for_health frank-primary http://127.0.0.1:7770/health 1800 || return 1
|
||||
run_privileged systemctl enable frank-archon.service
|
||||
run_privileged systemctl restart frank-archon.service
|
||||
# Cold start: .rad load + PLE pin + expert host-pool upload (the
|
||||
# publisher's compose healthcheck allows a 15-minute start_period).
|
||||
# Parity with every predecessor on this port. The HTTP listener binds
|
||||
# AFTER the full load (main.cpp: engine.start() then engine.serve()),
|
||||
# so during load the port REFUSES connections — a bare curl is honest
|
||||
# here, unlike FreeToken's 200-while-loading body. (The MainPID check
|
||||
# below still guards the stray-port case.)
|
||||
local i ok=0
|
||||
for ((i=1; i<=900; i++)); do
|
||||
if curl -fsS http://127.0.0.1:7770/health 2>/dev/null >/dev/null; then
|
||||
echo "frank-archon engine healthy after approximately $((i * 2))s"
|
||||
ok=1
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
if [[ "$ok" != 1 ]]; then
|
||||
echo "FAIL: frank-archon /health never answered 200 in 1800s" >&2
|
||||
return 1
|
||||
fi
|
||||
# False-green hardening (the 2026-09-26 lesson): a bare curl against the
|
||||
# port cannot tell WHO answers — a stray probe holding :7770 satisfies
|
||||
# it while the unit is stopped. The deploy is only green if the unit
|
||||
# itself is active with a live MainPID behind the health answer.
|
||||
local main_pid
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-primary.service)"
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-archon.service)"
|
||||
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
|
||||
echo "FAIL: :7770 answered but frank-primary.service has no MainPID — a stray process holds the port" >&2
|
||||
echo "FAIL: :7770 answered but frank-archon.service has no MainPID — a stray process holds the port" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "frank-primary: health answered by unit MainPID $main_pid"
|
||||
echo "frank-archon: health answered by unit MainPID $main_pid"
|
||||
}
|
||||
|
||||
wait_for_health() {
|
||||
@@ -1502,9 +1574,9 @@ deploy_service() {
|
||||
anubis) deploy_anubis ;;
|
||||
seerr) deploy_seerr ;;
|
||||
marreta) deploy_marreta ;;
|
||||
frank-large) deploy_llm_large ;;
|
||||
frank-small) deploy_llm_small ;;
|
||||
frank-primary) deploy_llm_primary ;;
|
||||
frank-dragoon) deploy_llm_dragoon ;;
|
||||
frank-zealot) deploy_llm_zealot ;;
|
||||
frank-archon) deploy_llm_archon ;;
|
||||
hermes|hermes-dashboard|hermes-webui) deploy_hermes ;;
|
||||
romm|romm-mariadb|romm-redis|mariadb|redis) deploy_romm_stack "$service" ;;
|
||||
jellyfin) deploy_jellyfin ;;
|
||||
|
||||
+29
-4
@@ -35,7 +35,7 @@ Options:
|
||||
Examples:
|
||||
./scripts/deploy.sh dankclaw
|
||||
./scripts/deploy.sh dankclaw --service searxng
|
||||
./scripts/deploy.sh dankserver --service frank-primary
|
||||
./scripts/deploy.sh dankserver --service frank-archon
|
||||
./scripts/deploy.sh dankclaw --dry-run
|
||||
EOF
|
||||
}
|
||||
@@ -141,12 +141,12 @@ valid_service() {
|
||||
dankclaw:alertmanager|\
|
||||
dankclaw:node-exporter|dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
||||
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|dankclaw:seerr|\
|
||||
dankclaw:frank-large|dankclaw:frank-small|\
|
||||
dankclaw:frank-dragoon|dankclaw:frank-zealot|\
|
||||
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
|
||||
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
|
||||
dankserver:qbittorrent|dankserver:radarr|dankserver:sonarr|\
|
||||
dankserver:lidarr|dankserver:readarr|dankserver:node-exporter|\
|
||||
dankserver:frank-primary|\
|
||||
dankserver:frank-archon|\
|
||||
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
|
||||
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
|
||||
frankbook-pro:readarr)
|
||||
@@ -196,6 +196,21 @@ else
|
||||
printf ' - %s\n' "${PLAN[@]}"
|
||||
fi
|
||||
|
||||
# --- frankbook e2e trigger --------------------------------------------------
|
||||
# The *arr/qBittorrent e2e units walk links that terminate on FrankBook-Pro
|
||||
# (192.168.0.112), which sleeps when idle. The [3/3] e2e gate therefore runs
|
||||
# the frankbook Playwright project ONLY when something in its blast radius
|
||||
# changed since origin/main: the frankbook spec itself, the frankbook-pro
|
||||
# stack, the homepage registry, or the Caddyfile serving the arr vhosts.
|
||||
# Every other deploy skips it (tests/e2e/run.sh excludes the project by
|
||||
# default). See tests/e2e/specs/19-frankbook.spec.ts.
|
||||
FB_E2E_TRIGGER=0
|
||||
while IFS= read -r fb_path; do
|
||||
case "$fb_path" in
|
||||
tests/e2e/specs/19-frankbook.spec.ts|frankbook-pro/*|shared/homepage/services.json|dankclaw/Caddyfile) FB_E2E_TRIGGER=1 ;;
|
||||
esac
|
||||
done < <(git diff --name-only "$BASE_REVISION" "$HEAD_REVISION")
|
||||
|
||||
REMOTE="${REMOTE_USER}@${REMOTE_HOST}"
|
||||
REMOTE_ARGS=("$TARGET" "$HEAD_REVISION" "$REMOTE_DIR" "$REMOTE_SHELL")
|
||||
if [[ "${#EXPLICIT_SERVICES[@]}" -eq 0 ]]; then
|
||||
@@ -209,6 +224,11 @@ if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
printf '[dry-run] Runner arguments:'
|
||||
printf ' %q' "${REMOTE_ARGS[@]}"
|
||||
printf '\n'
|
||||
if [[ "$FB_E2E_TRIGGER" -eq 1 ]]; then
|
||||
echo "[dry-run] e2e gate: frankbook project INCLUDED (frankbook trigger paths changed)."
|
||||
else
|
||||
echo "[dry-run] e2e gate: frankbook project EXCLUDED (no frankbook trigger paths)."
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -250,7 +270,12 @@ if [[ "$SKIP_E2E" -eq 1 ]]; then
|
||||
echo "[3/3] e2e gate SKIPPED (--skip-e2e): live fleet NOT verified." >&2
|
||||
else
|
||||
echo "[3/3] Running Playwright e2e gate against the live fleet..."
|
||||
if bash "$REPO_ROOT/tests/e2e/run.sh"; then
|
||||
E2E_ENV=()
|
||||
if [[ "$FB_E2E_TRIGGER" -eq 1 ]]; then
|
||||
echo " frankbook trigger paths changed: including the frankbook e2e project."
|
||||
E2E_ENV=(E2E_INCLUDE_FRANKBOOK=1)
|
||||
fi
|
||||
if env "${E2E_ENV[@]}" bash "$REPO_ROOT/tests/e2e/run.sh"; then
|
||||
echo "e2e gate PASSED."
|
||||
else
|
||||
echo "=== DEPLOY INCOMPLETE: e2e gate FAILED on $TARGET ===" >&2
|
||||
|
||||
Executable
+117
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""derive-protected-experts.py — rewrite the recipe's protected-expert section
|
||||
from a calibration recording. RUNS ON DANKSERVER (venv: numpy).
|
||||
|
||||
The protection rule (docs/MOE-W4.md "Protected experts", restated in the recipe):
|
||||
an expert whose share of its layer's down-input energy is at least 3% AND
|
||||
at least three times its share of the routing is kept bf16; the count must
|
||||
be EVEN on each rank's half of the layer (the four-bit GEMM takes the rest
|
||||
as two equal tables by parity) — an odd half gains its next most energetic
|
||||
non-qualifying expert.
|
||||
|
||||
Data: the calibration dir's per-expert down Grams,
|
||||
gram.r<rank>.blk.L.ffn_down_exps.<e>.bin — 32-byte RADGRAM2 header
|
||||
(magic, k, rank, rows, domain) then k*k f32, named with the checkpoint's
|
||||
GLOBAL expert numbering (arch/common/rad_block_moe_fp8.h: ge =
|
||||
expert_base + e). rows (header) is the expert's routing count; the trace
|
||||
of the Gram is its down-input energy. Rank halves: 0-255, 256-511
|
||||
(contiguous expert_base split at TP=2).
|
||||
|
||||
Output: the cast-rule lines (recipe format) on stdout, a report on stderr.
|
||||
|
||||
Usage: derive-protected-experts.py <calib dir> [--min-share 0.03]
|
||||
[--min-route-mult 3.0]
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
NAME_RE = re.compile(r"^gram\.r(\d)\.blk\.(\d+)\.ffn_down_exps\.(\d+)\.bin$")
|
||||
|
||||
|
||||
def read_gram(path):
|
||||
"""(k, rank, rows, trace) from a RADGRAM2 file (full matrix on disk)."""
|
||||
with open(path, "rb") as f:
|
||||
hdr = f.read(32)
|
||||
magic, k, rank, rows, domain = struct.unpack("<8sIIQI", hdr[:28])
|
||||
if magic != b"RADGRAM2":
|
||||
raise SystemExit(f"not a RADGRAM2 file: {path}")
|
||||
n = k * k
|
||||
a = np.frombuffer(f.read(n * 4), dtype=np.float32)
|
||||
if a.size != n:
|
||||
raise SystemExit(f"short gram file: {path}")
|
||||
tr = float(a.reshape(k, k).diagonal().sum())
|
||||
return k, rank, int(rows), tr
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("calib_dir")
|
||||
ap.add_argument("--min-share", type=float, default=0.03)
|
||||
ap.add_argument("--min-route-mult", type=float, default=3.0)
|
||||
a = ap.parse_args()
|
||||
|
||||
# layer -> global expert -> (rank, rows, energy)
|
||||
data = {}
|
||||
for path in sorted(glob.glob(os.path.join(
|
||||
a.calib_dir, "gram.r*.ffn_down_exps.*.bin"))):
|
||||
m = NAME_RE.match(os.path.basename(path))
|
||||
if not m:
|
||||
continue
|
||||
rank, layer, expert = int(m.group(1)), int(m.group(2)), int(m.group(3))
|
||||
k, _, rows, tr = read_gram(path)
|
||||
data.setdefault(layer, {})[expert] = (rank, rows, tr)
|
||||
|
||||
if not data:
|
||||
raise SystemExit("no per-expert down grams found")
|
||||
|
||||
lines, report = [], []
|
||||
total = 0
|
||||
for layer in sorted(data):
|
||||
experts = data[layer]
|
||||
tot_e = sum(v[2] for v in experts.values())
|
||||
tot_r = sum(v[1] for v in experts.values())
|
||||
if tot_e <= 0 or tot_r <= 0:
|
||||
report.append(f"blk.{layer}: empty recording, skipped")
|
||||
continue
|
||||
qual, pool = [], []
|
||||
for e, (rank, rows, tr) in experts.items():
|
||||
es, rs = tr / tot_e, rows / max(tot_r, 1)
|
||||
rec = (e, rank, es, rs)
|
||||
if es >= a.min_share and es >= a.min_route_mult * rs:
|
||||
qual.append(rec)
|
||||
elif es > 0:
|
||||
pool.append(rec)
|
||||
chosen = {r[0]: (r[2], r[3]) for r in qual}
|
||||
for half in (0, 1): # rank halves by the contiguous expert_base split
|
||||
n_half = sum(1 for e in chosen if e // 256 == half)
|
||||
if n_half % 2:
|
||||
pool.sort(key=lambda r: -r[2])
|
||||
for cand in pool:
|
||||
if cand[1] == half and cand[0] not in chosen:
|
||||
chosen[cand[0]] = (cand[2], cand[3])
|
||||
report.append(
|
||||
f"blk.{layer}: half {half} odd -> added e{cand[0]} "
|
||||
f"(energy {cand[2]*100:.2f}%)")
|
||||
break
|
||||
for e in sorted(chosen):
|
||||
es, rs = chosen[e]
|
||||
lines.append(f"blk.{layer}.ffn_*_exps.{e}.weight"
|
||||
f" cast dtype=bf16")
|
||||
report.append(f"blk.{layer} e{e:3d} (rank {e // 256}): "
|
||||
f"energy {es*100:5.2f}%, routing {rs*100:5.3f}%, "
|
||||
f"ratio {es/max(rs,1e-12):6.1f}")
|
||||
total += len(chosen)
|
||||
|
||||
print("\n".join(lines))
|
||||
print("\n".join(report), file=sys.stderr)
|
||||
print(f"total protected: {total}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+113
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env bash
|
||||
# download-frank-archon-swift15.sh — payload fetch/verify for the planned
|
||||
# frank-archon swap to the Swift 1.5 finetune: ukisai/Swift1.5-Qwen3.8-Flash-Next
|
||||
# (BF16 source, 149 files, 360,037,355,038 bytes = 335.3 GiB: 131 safetensors
|
||||
# shards + weight-map index + tokenizer + vision preprocessing configs + chat
|
||||
# template + licenses). Key set verified identical to the official
|
||||
# Qwen/Qwen3.8-Flash-Next base (0 added / 0 missing), so the qwen4exp recipes
|
||||
# apply verbatim; the checkpoint ships BF16 with NO quantization_config.
|
||||
# Repo is UNGATED — no token needed (never put one on the command line).
|
||||
#
|
||||
# bash scripts/download-frank-archon-swift15.sh --fetch resume/complete
|
||||
# bash scripts/download-frank-archon-swift15.sh verify only
|
||||
#
|
||||
# Fetch uses the vendored scripts/hfget.sh (StillDeadcode's Apache-2.0
|
||||
# listing/resume downloader, curl+jq only): per-file size checks against
|
||||
# the Hub listing, `-C -` resume, pickle-format duplicates skipped.
|
||||
# Verify (default) enforces the download contract in three parts: every listed file
|
||||
# present with the listed size, the shard count matches the weight-map
|
||||
# index, and every shard's sha256 equals the Hub's LFS oid. Verify needs
|
||||
# only the public API.
|
||||
set -euo pipefail
|
||||
|
||||
REPO="ukisai/Swift1.5-Qwen3.8-Flash-Next"
|
||||
DEST="/home/dank/models/ukisai/Swift1.5-Qwen3.8-Flash-Next"
|
||||
EXPECTED_SHARDS=131
|
||||
|
||||
mode="verify"
|
||||
[[ "${1:-}" == "--fetch" ]] && mode="fetch"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
if [[ "$mode" == "fetch" ]]; then
|
||||
# HFGET_JOBS=4 is hfget's default (a few streams saturate the link).
|
||||
bash "$SCRIPT_DIR/hfget.sh" "$REPO" "$DEST" main
|
||||
fi
|
||||
|
||||
echo "verifying local payload: $DEST"
|
||||
fail=0
|
||||
|
||||
INDEX="$DEST/model.safetensors.index.json"
|
||||
if [[ ! -f "$INDEX" ]]; then
|
||||
echo " BAD weight-map index missing: $INDEX" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python3 - "$REPO" "$DEST" <<'PY' || fail=1
|
||||
import hashlib, json, os, sys, urllib.request
|
||||
|
||||
repo, dest = sys.argv[1], sys.argv[2]
|
||||
|
||||
def listing():
|
||||
"""The tree API, following the Link cursor exactly like hfget.sh."""
|
||||
out, api, pages = [], (
|
||||
f"https://huggingface.co/api/models/{repo}/tree/main"
|
||||
"?recursive=true&expand=true"), 0
|
||||
while api:
|
||||
with urllib.request.urlopen(api) as r:
|
||||
out += json.load(r)
|
||||
api = None
|
||||
link = r.headers.get("Link", "")
|
||||
for part in link.split(","):
|
||||
if 'rel="next"' in part:
|
||||
api = part[part.find("<") + 1:part.find(">")]
|
||||
pages += 1
|
||||
if pages > 200:
|
||||
raise SystemExit("listing did not terminate")
|
||||
return out
|
||||
|
||||
files = [e for e in listing() if e.get("type") == "file"]
|
||||
sizes = {e["path"]: e.get("size", 0) for e in files}
|
||||
lfs = {e["path"]: e["lfs"]["oid"] for e in files if e.get("lfs")}
|
||||
|
||||
bad = 0
|
||||
for path, want in sorted(sizes.items()):
|
||||
p = os.path.join(dest, path)
|
||||
if not os.path.isfile(p):
|
||||
print(f" BAD {path}: missing")
|
||||
bad += 1
|
||||
elif os.path.getsize(p) != want:
|
||||
print(f" BAD {path}: {os.path.getsize(p)} bytes, listing says {want}")
|
||||
bad += 1
|
||||
print(f" {'OK' if bad == 0 else 'FAIL'}: {len(sizes) - bad}/{len(sizes)} files present at listed size")
|
||||
if bad:
|
||||
raise SystemExit(1)
|
||||
|
||||
# Shard set against the weight-map index.
|
||||
wm = json.load(open(os.path.join(dest, "model.safetensors.index.json")))["weight_map"]
|
||||
shards = sorted(set(wm.values()))
|
||||
print(f" index maps {len(wm)} tensors across {len(shards)} shards")
|
||||
if len(shards) != 131:
|
||||
print(f" BAD shard count {len(shards)}, want 131")
|
||||
raise SystemExit(1)
|
||||
|
||||
# Content: every LFS-tracked file hashed against the Hub oid. The safetensors
|
||||
# shards dominate the bytes; tokenizer/config files ride the same check when
|
||||
# the Hub tracks them in LFS (they are plain blobs here, already size-checked).
|
||||
todo = [p for p in shards if p in lfs]
|
||||
for i, path in enumerate(todo):
|
||||
h = hashlib.sha256()
|
||||
with open(os.path.join(dest, path), "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 24), b""):
|
||||
h.update(chunk)
|
||||
if h.hexdigest() != lfs[path]:
|
||||
print(f" BAD {path}: sha256 mismatch (want {lfs[path]})")
|
||||
bad += 1
|
||||
if (i + 1) % 16 == 0:
|
||||
print(f" ...hashed {i + 1}/{len(todo)}")
|
||||
print(f" {'OK' if bad == 0 else 'FAIL'}: {len(todo) - bad}/{len(todo)} shards sha256-match the Hub")
|
||||
if bad:
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
|
||||
[[ "$fail" -eq 0 ]] || exit 1
|
||||
echo "payload verified: 131 shards + index + tokenizer + configs, sizes and sha256 checked"
|
||||
Executable
+111
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env bash
|
||||
# download-frank-archon.sh — payload fetch/verify for frank-archon:
|
||||
# the OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 checkpoint (131 safetensors
|
||||
# shards + weight-map index + tokenizer + vision preprocessing configs,
|
||||
# ~172.8 GB). 2026-10-05: replaces the orcarouter uncensored payload of the
|
||||
# FreeToken era (that script lived here; its repo was gated and needed
|
||||
# HF_TOKEN — this one is UNGATED and needs no token at all).
|
||||
#
|
||||
# bash scripts/download-frank-archon.sh --fetch resume/complete
|
||||
# bash scripts/download-frank-archon.sh verify only
|
||||
#
|
||||
# Fetch uses the vendored scripts/hfget.sh (StillDeadcode's Apache-2.0
|
||||
# listing/resume downloader, curl+jq only): per-file size checks against
|
||||
# the Hub listing, `-C -` resume, pickle-format duplicates skipped.
|
||||
# Verify (default) enforces the download contract in three parts: every listed file
|
||||
# present with the listed size, the shard count matches the weight-map
|
||||
# index, and every shard's sha256 equals the Hub's LFS oid. Verify needs
|
||||
# only the public API.
|
||||
set -euo pipefail
|
||||
|
||||
REPO="Qwen/Qwen3.8-Flash-Next-FP8"
|
||||
DEST="/home/dank/models/Qwen/Qwen3.8-Flash-Next-FP8"
|
||||
EXPECTED_SHARDS=131
|
||||
|
||||
mode="verify"
|
||||
[[ "${1:-}" == "--fetch" ]] && mode="fetch"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
if [[ "$mode" == "fetch" ]]; then
|
||||
# HFGET_JOBS=4 is hfget's default (a few streams saturate the link).
|
||||
bash "$SCRIPT_DIR/hfget.sh" "$REPO" "$DEST" main
|
||||
fi
|
||||
|
||||
echo "verifying local payload: $DEST"
|
||||
fail=0
|
||||
|
||||
INDEX="$DEST/model.safetensors.index.json"
|
||||
if [[ ! -f "$INDEX" ]]; then
|
||||
echo " BAD weight-map index missing: $INDEX" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python3 - "$REPO" "$DEST" <<'PY' || fail=1
|
||||
import hashlib, json, os, sys, urllib.request
|
||||
|
||||
repo, dest = sys.argv[1], sys.argv[2]
|
||||
|
||||
def listing():
|
||||
"""The tree API, following the Link cursor exactly like hfget.sh."""
|
||||
out, api, pages = [], (
|
||||
f"https://huggingface.co/api/models/{repo}/tree/main"
|
||||
"?recursive=true&expand=true"), 0
|
||||
while api:
|
||||
with urllib.request.urlopen(api) as r:
|
||||
out += json.load(r)
|
||||
api = None
|
||||
link = r.headers.get("Link", "")
|
||||
for part in link.split(","):
|
||||
if 'rel="next"' in part:
|
||||
api = part[part.find("<") + 1:part.find(">")]
|
||||
pages += 1
|
||||
if pages > 200:
|
||||
raise SystemExit("listing did not terminate")
|
||||
return out
|
||||
|
||||
files = [e for e in listing() if e.get("type") == "file"]
|
||||
sizes = {e["path"]: e.get("size", 0) for e in files}
|
||||
lfs = {e["path"]: e["lfs"]["oid"] for e in files if e.get("lfs")}
|
||||
|
||||
bad = 0
|
||||
for path, want in sorted(sizes.items()):
|
||||
p = os.path.join(dest, path)
|
||||
if not os.path.isfile(p):
|
||||
print(f" BAD {path}: missing")
|
||||
bad += 1
|
||||
elif os.path.getsize(p) != want:
|
||||
print(f" BAD {path}: {os.path.getsize(p)} bytes, listing says {want}")
|
||||
bad += 1
|
||||
print(f" {'OK' if bad == 0 else 'FAIL'}: {len(sizes) - bad}/{len(sizes)} files present at listed size")
|
||||
if bad:
|
||||
raise SystemExit(1)
|
||||
|
||||
# Shard set against the weight-map index.
|
||||
wm = json.load(open(os.path.join(dest, "model.safetensors.index.json")))["weight_map"]
|
||||
shards = sorted(set(wm.values()))
|
||||
print(f" index maps {len(wm)} tensors across {len(shards)} shards")
|
||||
if len(shards) != 131:
|
||||
print(f" BAD shard count {len(shards)}, want 131")
|
||||
raise SystemExit(1)
|
||||
|
||||
# Content: every LFS-tracked file hashed against the Hub oid. The safetensors
|
||||
# shards dominate the bytes; tokenizer/config files ride the same check when
|
||||
# the Hub tracks them in LFS (they are plain blobs here, already size-checked).
|
||||
todo = [p for p in shards if p in lfs]
|
||||
for i, path in enumerate(todo):
|
||||
h = hashlib.sha256()
|
||||
with open(os.path.join(dest, path), "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 24), b""):
|
||||
h.update(chunk)
|
||||
if h.hexdigest() != lfs[path]:
|
||||
print(f" BAD {path}: sha256 mismatch (want {lfs[path]})")
|
||||
bad += 1
|
||||
if (i + 1) % 16 == 0:
|
||||
print(f" ...hashed {i + 1}/{len(todo)}")
|
||||
print(f" {'OK' if bad == 0 else 'FAIL'}: {len(todo) - bad}/{len(todo)} shards sha256-match the Hub")
|
||||
if bad:
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
|
||||
[[ "$fail" -eq 0 ]] || exit 1
|
||||
echo "payload verified: 131 shards + index + tokenizer + configs, sizes and sha256 checked"
|
||||
Executable
+121
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env bash
|
||||
# download-frank-dragoon-flashnext.sh — fetch frank-dragoon's flash-server v2
|
||||
# payload: the Quat3rnion abliterated Swift1.5 checkpoint of peonist's v2
|
||||
# .hgn, the REQUIRED lookup table and the optional vision tower from
|
||||
# peonist-ai/halogen-qwen3.8-flash-next (both at the revisions the model
|
||||
# card pins), and the flat tokenizer the two repos share. All public, no
|
||||
# token — and a token never rides a command line: argv is readable by every
|
||||
# process on the host through /proc, so a key placed there is a key exposed.
|
||||
#
|
||||
# The fetch discipline, same as every sibling script: curl -f (an access error
|
||||
# must not become a "successful" 153-byte artifact), exact byte count from
|
||||
# the Hub tree API, LFS sha256 verification, --verify-only to re-check an
|
||||
# existing payload without downloading.
|
||||
#
|
||||
# File manifest (verified against the Hub tree API on 2026-10-06):
|
||||
# qwen38-flash-next-v2-swift15-abliterated.hgn 66,687,678,464 B sha256
|
||||
# 9c880b8ba48deabfa06172eff1e7572e4fe465e09e97faec6e50c3f6fdcac037
|
||||
# (Quat3rnion main; compacted r2 of the card's 89,775,868,032 B
|
||||
# append-only original — byte-identical live tensors, dropped padding)
|
||||
# qwen38-flash-next-ngram.hgn 51,200,246,144 B sha256
|
||||
# 3450acada94e19aabad88bd49b45eb70304820949c2240fc178067b20ae5dffe
|
||||
# (peonist @5cc17cea; REQUIRED — the engine reads it beside the
|
||||
# checkpoint)
|
||||
# qwen38-flash-next-vision.hgn 897,916,416 B sha256
|
||||
# d62e0ae553fe88afd3833733d4a4c669f34d20fd8dfce4b9610525bed2134b10
|
||||
# (peonist @e053f488; optional, enables image input)
|
||||
# tokenizer/* @e053f488 — byte-identical to the 27B-era tokenizer
|
||||
# already on disk (tokenizer.json LFS
|
||||
# 0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3);
|
||||
# fetched fresh rather than symlinked, so the payload dir is
|
||||
# self-contained — every gate and every mount sees real files inside it.
|
||||
set -euo pipefail
|
||||
|
||||
DEST="${FRANK_DRAGOON_MODEL_DIR:-$HOME/models/halogen-flashnext-v2}"
|
||||
ABLIT="https://huggingface.co/Quat3rnion/halogen-swift1.5-qwen3.8-flash-next-v2-abliterated/resolve/main"
|
||||
PEONIST="https://huggingface.co/peonist-ai/halogen-qwen3.8-flash-next/resolve"
|
||||
NGRAM_REV="5cc17cea1a10b502b4a5db41d8d2b1a0f22dba26"
|
||||
TOK_REV="e053f488b120b99ed2525e6ac99f68c51b3b6179"
|
||||
|
||||
FILES=(qwen38-flash-next-v2-swift15-abliterated.hgn
|
||||
qwen38-flash-next-ngram.hgn
|
||||
qwen38-flash-next-vision.hgn
|
||||
tokenizer/tokenizer.json
|
||||
tokenizer/chat_template.jinja
|
||||
tokenizer/generation_config.json
|
||||
tokenizer/merges.txt
|
||||
tokenizer/vocab.json
|
||||
tokenizer/tokenizer_config.json)
|
||||
SIZES=(66687678464
|
||||
51200246144
|
||||
897916416
|
||||
12809320
|
||||
8952
|
||||
202
|
||||
3353259
|
||||
6722759
|
||||
17928)
|
||||
SHAS=(9c880b8ba48deabfa06172eff1e7572e4fe465e09e97faec6e50c3f6fdcac037
|
||||
3450acada94e19aabad88bd49b45eb70304820949c2240fc178067b20ae5dffe
|
||||
d62e0ae553fe88afd3833733d4a4c669f34d20fd8dfce4b9610525bed2134b10
|
||||
0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3
|
||||
- - - - - -)
|
||||
|
||||
# Fetch source per file: repo base + revision (empty = ABLIT @main).
|
||||
SRCS=("$ABLIT"
|
||||
"$PEONIST/$NGRAM_REV"
|
||||
"$PEONIST/$TOK_REV"
|
||||
"$PEONIST/$TOK_REV"
|
||||
"$PEONIST/$TOK_REV"
|
||||
"$PEONIST/$TOK_REV"
|
||||
"$PEONIST/$TOK_REV"
|
||||
"$PEONIST/$TOK_REV"
|
||||
"$PEONIST/$TOK_REV")
|
||||
|
||||
VERIFY_ONLY=0
|
||||
[ "${1:-}" = "--verify-only" ] && VERIFY_ONLY=1
|
||||
|
||||
mkdir -p "$DEST"
|
||||
|
||||
for i in "${!FILES[@]}"; do
|
||||
f="${FILES[$i]}"; want="${SIZES[$i]}"; sha="${SHAS[$i]}"; base="${SRCS[$i]}"
|
||||
path="$DEST/$f"
|
||||
mkdir -p "$(dirname "$path")"
|
||||
if [ "$VERIFY_ONLY" -eq 0 ]; then
|
||||
# A file already at the published size skips the fetch entirely:
|
||||
# resuming it would ask the CDN for a range past EOF (416 -> curl
|
||||
# error under set -e), and re-pulling 62 GiB to prove a point is
|
||||
# not what --verify-only is for. sha256 below still decides.
|
||||
if [ -f "$path" ] && [ "$(stat -c %s "$path")" = "$want" ]; then
|
||||
echo "HAVE $f ($want bytes) — skipping fetch"
|
||||
else
|
||||
echo "FETCH $f -> $path"
|
||||
# HF answers /resolve/ with a 302 to the Xet CDN and curl DROPS
|
||||
# the Range header across redirects — a naive -C - then
|
||||
# "resumes" by pulling the ~1 KB redirect body (measured
|
||||
# 2026-10-05 on the 27B fetch: file frozen 14 MiB short, curl
|
||||
# exit 0). Resolve the effective URL first, then resume
|
||||
# against the CDN, which honors Range.
|
||||
url=$(curl -sfIL -o /dev/null -w '%{url_effective}' "$base/$f") \
|
||||
|| { echo "cannot resolve $base/$f" >&2; exit 1; }
|
||||
curl -fL --retry 5 -C - -o "$path" "$url"
|
||||
fi
|
||||
fi
|
||||
[ -f "$path" ] || { echo "MISSING $path (run without --verify-only)" >&2; exit 1; }
|
||||
got="$(stat -c %s "$path")"
|
||||
[ "$got" = "$want" ] || { echo "SIZE $path: $got != $want" >&2; exit 1; }
|
||||
echo "OK $f ($got bytes)"
|
||||
done
|
||||
|
||||
echo "byte counts match; verifying sha256 where the Hub publishes an LFS oid..."
|
||||
for i in "${!FILES[@]}"; do
|
||||
f="${FILES[$i]}"; sha="${SHAS[$i]}"
|
||||
if [ "$sha" = "-" ]; then
|
||||
echo "OK $f (plain git blob, size-checked)"
|
||||
continue
|
||||
fi
|
||||
got="$(sha256sum "$DEST/$f" | cut -d' ' -f1)"
|
||||
[ "$got" = "$sha" ] || { echo "SHA256 $f: $got != $sha" >&2; exit 1; }
|
||||
echo "OK $f sha256"
|
||||
done
|
||||
echo "DOWNLOAD_DONE $DEST"
|
||||
Executable
+77
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env bash
|
||||
# download-frank-dragoon-halogen.sh — fetch frank-dragoon's halogen payload:
|
||||
# the qwen3.8-27b .hgn checkpoint and the export tokenizer, from the
|
||||
# PUBLIC peonist-ai/halogen-qwen3.8-27b repo (no token — and a token never
|
||||
# rides a command line: argv is readable by every process on the host
|
||||
# through /proc, so a key placed there is a key exposed).
|
||||
#
|
||||
# The fetch discipline, same as every sibling script: curl -f (an access error
|
||||
# must not become a "successful" 153-byte artifact), exact byte count from
|
||||
# the model card, LFS sha256 verification, --verify-only to re-check an
|
||||
# existing payload without downloading.
|
||||
set -euo pipefail
|
||||
|
||||
DEST="${FRANK_DRAGOON_MODEL_DIR:-$HOME/models/halogen-qwen3.8-27b}"
|
||||
BASE="https://huggingface.co/peonist-ai/halogen-qwen3.8-27b/resolve/main"
|
||||
|
||||
FILES=(qwen3.8-27b-p1w4d-d2.hgn
|
||||
tokenizer/tokenizer.json
|
||||
tokenizer/vocab.json
|
||||
tokenizer/merges.txt
|
||||
tokenizer/chat_template.jinja
|
||||
tokenizer/tokenizer_config.json)
|
||||
# Byte sizes as published by the Hub on 2026-10-05, plus sha256 (the LFS
|
||||
# oid) where the Hub stores an LFS object — plain git blobs carry no oid
|
||||
# and are size-checked only. Re-check with:
|
||||
# curl -s "https://huggingface.co/api/models/peonist-ai/halogen-qwen3.8-27b/tree/main?recursive=true"
|
||||
SIZES=(35865565184 12809320 6722759 3353259 8952 17928)
|
||||
SHAS=(274c3fc767fb57faf025dd03c76376b1fc1b32448aef45aaf02d1898fa962089
|
||||
0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3 - - - -)
|
||||
|
||||
VERIFY_ONLY=0
|
||||
[ "${1:-}" = "--verify-only" ] && VERIFY_ONLY=1
|
||||
|
||||
mkdir -p "$DEST"
|
||||
|
||||
for i in "${!FILES[@]}"; do
|
||||
f="${FILES[$i]}"; want="${SIZES[$i]}"; sha="${SHAS[$i]}"
|
||||
path="$DEST/$f"
|
||||
mkdir -p "$(dirname "$path")"
|
||||
if [ "$VERIFY_ONLY" -eq 0 ]; then
|
||||
# A file already at the published size skips the fetch entirely:
|
||||
# resuming it would ask the CDN for a range past EOF (416 -> curl
|
||||
# error under set -e), and re-pulling 33.4 GiB to prove a point is
|
||||
# not what --verify-only is for. sha256 below still decides.
|
||||
if [ -f "$path" ] && [ "$(stat -c %s "$path")" = "$want" ]; then
|
||||
echo "HAVE $f ($want bytes) — skipping fetch"
|
||||
else
|
||||
echo "FETCH $f -> $path"
|
||||
# HF answers /resolve/ with a 302 to the Xet CDN and curl DROPS
|
||||
# the Range header across redirects — a naive -C - then
|
||||
# "resumes" by pulling the ~1 KB redirect body (measured
|
||||
# 2026-10-05: file frozen 14 MiB short, curl exit 0). Resolve
|
||||
# the effective URL first, then resume against the CDN, which
|
||||
# honors Range.
|
||||
url=$(curl -sfIL -o /dev/null -w '%{url_effective}' "$BASE/$f") \
|
||||
|| { echo "cannot resolve $BASE/$f" >&2; exit 1; }
|
||||
curl -fL --retry 5 -C - -o "$path" "$url"
|
||||
fi
|
||||
fi
|
||||
[ -f "$path" ] || { echo "MISSING $path (run without --verify-only)" >&2; exit 1; }
|
||||
got="$(stat -c %s "$path")"
|
||||
[ "$got" = "$want" ] || { echo "SIZE $path: $got != $want" >&2; exit 1; }
|
||||
echo "OK $f ($got bytes)"
|
||||
done
|
||||
|
||||
echo "byte counts match; verifying sha256 where the Hub publishes an LFS oid..."
|
||||
for i in "${!FILES[@]}"; do
|
||||
f="${FILES[$i]}"; sha="${SHAS[$i]}"
|
||||
if [ "$sha" = "-" ]; then
|
||||
echo "OK $f (plain git blob, size-checked)"
|
||||
continue
|
||||
fi
|
||||
got="$(sha256sum "$DEST/$f" | cut -d' ' -f1)"
|
||||
[ "$got" = "$sha" ] || { echo "SHA256 $f: $got != $sha" >&2; exit 1; }
|
||||
echo "OK $f sha256"
|
||||
done
|
||||
echo "DOWNLOAD_DONE $DEST"
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF for
|
||||
# frank-dragoon (Dankclaw iGPU). Public repo — no auth header, and never a token
|
||||
# on the command line (it is world-readable in /proc/<pid>/cmdline).
|
||||
#
|
||||
# The payload is the repo's Q6 AGENT rung:
|
||||
# * Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf — ftype 114, 23.15 GiB.
|
||||
# Q6_0_ROCMFPX tensor type (102) + the AGENT routing, which keeps the
|
||||
# output-head and attention tensors at plain Q8_0 — the variant the card
|
||||
# names for MTP serving ("Take Q6 AGENT (114) if you want protected
|
||||
# heads"). Like every ROCmFPX file it loads on nothing but the fork.
|
||||
#
|
||||
# The other two files frank-dragoon needs are NOT re-downloaded:
|
||||
# * mtp-Qwen3.8-27B-Q4_0.gguf — the uncensored repo redistributes the
|
||||
# ALIGNED repo's draft head byte for byte (identical Hub LFS oid
|
||||
# 051a1764…), so it is hardlinked out of the aligned payload dir when that
|
||||
# dir holds the verified bytes, and fetched from this repo otherwise.
|
||||
# * mmproj-Qwen3.8-27B-BF16.gguf — this repo ships a Q8_0 projector; the
|
||||
# fleet keeps the BF16 one, which is the projector that was verified 4/4
|
||||
# on spatial ground truth on this host. Hardlinked the same way, fetched
|
||||
# from the ALIGNED repo if it is not on disk.
|
||||
#
|
||||
# Fresh downloads, no resume (`curl -C -` once produced an oversized file on a
|
||||
# 31 GiB payload); -f/--fail-with-body so an HTTP 401 is not a success, then
|
||||
# byte count and the Hub LFS sha256. Idempotent: complete files are skipped.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-dragoon-kingjones-uncensored.sh [DEST_DIR] [--verify-only]
|
||||
set -euo pipefail
|
||||
|
||||
REPO="kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF"
|
||||
ALIGNED_REPO="kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF"
|
||||
BASE="https://huggingface.co/${REPO}/resolve/main"
|
||||
ALIGNED_BASE="https://huggingface.co/${ALIGNED_REPO}/resolve/main"
|
||||
SOURCE_DIR="${SOURCE_DIR:-/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP}"
|
||||
|
||||
DEST=""
|
||||
VERIFY_ONLY=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--verify-only) VERIFY_ONLY=1 ;;
|
||||
*) DEST="$arg" ;;
|
||||
esac
|
||||
done
|
||||
DEST="${DEST:-/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP}"
|
||||
|
||||
# name bytes sha256 base_url may_link — may_link=1 means the same bytes may
|
||||
# already exist in SOURCE_DIR (hardlink them instead of re-fetching).
|
||||
FILES=(
|
||||
"Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf 24856607008 e966deb3d1a5a1b9ca7765126292f2d822f3dccdedb3b4872e673dec1cddc199 $BASE 0"
|
||||
# The draft head is redistributed byte-identical by both repos; the BF16
|
||||
# projector only exists in the aligned repo.
|
||||
"mtp-Qwen3.8-27B-Q4_0.gguf 1680271648 051a1764cff8c4f3ee6ae8b00593a0364c7539c67fa50ffc58f3f96509fca38e $BASE 1"
|
||||
"mmproj-Qwen3.8-27B-BF16.gguf 931145888 de2a49866988ea272c43dc2a43ee2662c25725a717ca5b9059ea420a97f53fe8 $ALIGNED_BASE 1"
|
||||
)
|
||||
|
||||
matches() { # matches FILE SIZE SHA — byte count AND LFS sha256
|
||||
[[ -f "$1" ]] && [[ "$(stat -c%s "$1")" == "$2" ]] \
|
||||
&& sha256sum -c --quiet <(printf '%s %s\n' "$3" "$1")
|
||||
}
|
||||
|
||||
mkdir -p "$DEST"
|
||||
status=0
|
||||
|
||||
fetch() { # fetch NAME SIZE SHA BASE_URL
|
||||
local name="$1" size="$2" sha="$3" url="$4" f="$DEST/$1"
|
||||
echo "FETCHING $name"
|
||||
curl -fsSL --fail-with-body -o "$f.part" "$url/$name"
|
||||
mv "$f.part" "$f"
|
||||
if [[ "$(stat -c%s "$f")" != "$size" ]]; then
|
||||
echo "SIZE FAIL $name ($(stat -c%s "$f") != $size)"
|
||||
return 1
|
||||
fi
|
||||
if ! sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
echo "SHA FAIL $name"
|
||||
return 1
|
||||
fi
|
||||
echo "VERIFIED $name"
|
||||
}
|
||||
|
||||
for entry in "${FILES[@]}"; do
|
||||
read -r name size sha url may_link <<<"$entry"
|
||||
f="$DEST/$name"
|
||||
if matches "$f" "$size" "$sha"; then
|
||||
echo "OK $name"
|
||||
continue
|
||||
fi
|
||||
if (( VERIFY_ONLY )); then
|
||||
echo "MISSING $name"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
# Shared bytes: prefer the local copy (same filesystem, so a hardlink costs
|
||||
# nothing) over re-downloading gigabytes that hash identically.
|
||||
if [[ "$may_link" == 1 && -n "${SOURCE_DIR:-}" ]] && matches "$SOURCE_DIR/$name" "$size" "$sha"; then
|
||||
echo "LINKING $name from $SOURCE_DIR"
|
||||
ln -f "$SOURCE_DIR/$name" "$f" 2>/dev/null || cp -f "$SOURCE_DIR/$name" "$f"
|
||||
if matches "$f" "$size" "$sha"; then
|
||||
echo "VERIFIED $name"
|
||||
continue
|
||||
fi
|
||||
echo "SHA FAIL $name (linked copy)"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
if ! fetch "$name" "$size" "$sha" "$url"; then
|
||||
status=1
|
||||
fi
|
||||
done
|
||||
|
||||
if (( status == 0 )); then
|
||||
echo "DOWNLOAD_DONE $DEST"
|
||||
else
|
||||
echo "DOWNLOAD_FAILED $DEST" >&2
|
||||
fi
|
||||
exit "$status"
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF for frank-dragoon
|
||||
# (Dankclaw iGPU). Public repo — no auth header, and never a token on the
|
||||
# command line (it is world-readable in /proc/<pid>/cmdline).
|
||||
#
|
||||
# The payload is the repo's highest-quality target plus its MTP draft head and
|
||||
# vision projector:
|
||||
# * Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf — ftype 115, 187 tensors at plain
|
||||
# Q8_0 and 311 at GGML_TYPE_Q8_0_ROCMFPX (103). The AGENT routing keeps the
|
||||
# output-head tensors at full Q8_0 (187 vs 2 in the non-AGENT build), which
|
||||
# is the variant to take when serving with the MTP head.
|
||||
# * mtp-Qwen3.8-27B-Q4_0.gguf — the draft head. Q4_0, NOT Q8_0: the head's
|
||||
# per-step cost is its projection over a 248K vocabulary, so the cheaper
|
||||
# head is faster end to end at equal acceptance (author: +6.0% at n-max 1,
|
||||
# +10.2% at n-max 2).
|
||||
# * mmproj-Qwen3.8-27B-BF16.gguf — the vision tower (verified 4/4 on spatial
|
||||
# ground truth with -fa on).
|
||||
#
|
||||
# Fresh downloads, no resume (`curl -C -` once produced an oversized file on a
|
||||
# 31 GiB payload); -f/--fail-with-body so an HTTP 401 is not a success, then
|
||||
# byte count and the Hub LFS sha256. Idempotent: complete files are skipped.
|
||||
#
|
||||
# The repo's own SHA256SUMS covers only the ROCmFP4 files, so the hashes below
|
||||
# come from the Hub LFS oids (api/models/<repo>/tree/main).
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-dragoon-kingjones.sh [DEST_DIR] [--verify-only]
|
||||
set -euo pipefail
|
||||
|
||||
REPO="kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF"
|
||||
BASE="https://huggingface.co/${REPO}/resolve/main"
|
||||
|
||||
DEST=""
|
||||
VERIFY_ONLY=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--verify-only) VERIFY_ONLY=1 ;;
|
||||
*) DEST="$arg" ;;
|
||||
esac
|
||||
done
|
||||
DEST="${DEST:-/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP}"
|
||||
|
||||
# name bytes sha256
|
||||
FILES=(
|
||||
"Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf 28214425952 f3ef2bdbf56e244fff14b72114dc2aa300b84a4cb4eaf54f2e5e4da1cfcae10b"
|
||||
"mtp-Qwen3.8-27B-Q4_0.gguf 1680271648 051a1764cff8c4f3ee6ae8b00593a0364c7539c67fa50ffc58f3f96509fca38e"
|
||||
"mmproj-Qwen3.8-27B-BF16.gguf 931145888 de2a49866988ea272c43dc2a43ee2662c25725a717ca5b9059ea420a97f53fe8"
|
||||
)
|
||||
|
||||
mkdir -p "$DEST"
|
||||
status=0
|
||||
for entry in "${FILES[@]}"; do
|
||||
read -r name size sha <<<"$entry"
|
||||
f="$DEST/$name"
|
||||
have=0
|
||||
if [[ -f "$f" ]] && [[ "$(stat -c%s "$f")" == "$size" ]]; then
|
||||
if sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
have=1
|
||||
fi
|
||||
fi
|
||||
if (( have )); then
|
||||
echo "OK $name"
|
||||
continue
|
||||
fi
|
||||
if (( VERIFY_ONLY )); then
|
||||
echo "MISSING $name"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
echo "FETCHING $name"
|
||||
tmp="$f.part"
|
||||
curl -fsSL --fail-with-body -o "$tmp" "$BASE/$name"
|
||||
mv "$tmp" "$f"
|
||||
if [[ "$(stat -c%s "$f")" != "$size" ]]; then
|
||||
echo "SIZE FAIL $name ($(stat -c%s "$f") != $size)"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
if ! sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
echo "SHA FAIL $name"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
echo "VERIFIED $name"
|
||||
done
|
||||
if (( status == 0 )); then
|
||||
echo "DOWNLOAD_DONE $DEST"
|
||||
else
|
||||
echo "DOWNLOAD_FAILED $DEST" >&2
|
||||
fi
|
||||
exit "$status"
|
||||
@@ -1,9 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download and assemble jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca (the frank-large
|
||||
# Download and assemble jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca (the frank-dragoon
|
||||
# model) on Dankclaw.
|
||||
#
|
||||
# bash scripts/download-frank-large-orca.sh
|
||||
# bash scripts/download-frank-large-orca.sh --verify-only
|
||||
# bash scripts/download-frank-dragoon-orca.sh
|
||||
# bash scripts/download-frank-dragoon-orca.sh --verify-only
|
||||
#
|
||||
# The package is 136 GB of Hub-sized transport parts (HF caps new uploads at
|
||||
# 50 GB, so the 79 GB main GGUF, the 52 GB PLE payload and the 4 GB MTP head
|
||||
@@ -105,4 +105,4 @@ fi
|
||||
echo "verifying assembled files against SHA256SUMS"
|
||||
(cd "$MODEL_DIR" && sha256sum -c SHA256SUMS)
|
||||
(cd "$MODEL_DIR/vision" && sha256sum -c SHA256SUMS)
|
||||
echo "frank-large payload complete: $MODEL_DIR"
|
||||
echo "frank-dragoon payload complete: $MODEL_DIR"
|
||||
@@ -1,9 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the Q4_K_XL frank-large payload on Dankclaw:
|
||||
# Download the Q4_K_XL frank-dragoon payload on Dankclaw:
|
||||
# unsloth/Qwen3.8-27B-GGUF -> Qwen3.8-27B-UD-Q4_K_XL.gguf
|
||||
#
|
||||
# bash scripts/download-frank-large-q4.sh
|
||||
# bash scripts/download-frank-large-q4.sh --verify-only
|
||||
# bash scripts/download-frank-dragoon-q4.sh
|
||||
# bash scripts/download-frank-dragoon-q4.sh --verify-only
|
||||
#
|
||||
# Integrity: the file is checked against its declared byte count and sha256
|
||||
# (the LFS oid from the Hub tree API, 2026-10-01). NO AUTH HEADER IS USED:
|
||||
@@ -21,12 +21,12 @@
|
||||
# x 2 (K,V) x 2 B), so 4 sessions x 163840 ctx = 640,000 tokens = 39.06 GiB.
|
||||
#
|
||||
# The speculative drafter is UNCHANGED and still fetched by its own script
|
||||
# (scripts/download-frank-large.sh, z-lab's Hub repo). That name must not
|
||||
# appear here at all — the frank-large seam greps THIS file for it to prove
|
||||
# (scripts/download-frank-dragoon.sh, z-lab's Hub repo). That name must not
|
||||
# appear here at all — the frank-dragoon seam greps THIS file for it to prove
|
||||
# the drafter was not folded into the payload fetch.
|
||||
#
|
||||
# NOTE ON MTP: this payload has no baked-in MTP head (the Hub repo ships one
|
||||
# separately as MTP/mtp-Qwen3.8-27B-Q4_0.gguf). frank-large does not need it —
|
||||
# separately as MTP/mtp-Qwen3.8-27B-Q4_0.gguf). frank-dragoon does not need it —
|
||||
# spec decode here is the block-diffusion drafter, not MTP.
|
||||
set -euo pipefail
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the Swift-1.5 frank-large candidate on Dankclaw:
|
||||
# Download the Swift-1.5 frank-dragoon candidate on Dankclaw:
|
||||
# ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF
|
||||
# Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf
|
||||
#
|
||||
# bash scripts/download-frank-large-swift.sh
|
||||
# bash scripts/download-frank-large-swift.sh --verify-only
|
||||
# bash scripts/download-frank-dragoon-swift.sh
|
||||
# bash scripts/download-frank-dragoon-swift.sh --verify-only
|
||||
#
|
||||
# Integrity: the file is checked against its declared byte count and sha256
|
||||
# (the LFS oid from the Hub, 2026-09-28). NO AUTH HEADER IS USED: the repo is
|
||||
@@ -15,9 +15,9 @@
|
||||
# Layout after a successful run (MODEL_DIR):
|
||||
# Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf 31457990272
|
||||
#
|
||||
# This is the CANDIDATE payload, not the deployed one. frank-large still
|
||||
# serves scripts/download-frank-large.sh's output; do not repoint
|
||||
# frank-large.env at MODEL_DIR until the gufo load test passes.
|
||||
# This is the CANDIDATE payload, not the deployed one. frank-dragoon still
|
||||
# serves scripts/download-frank-dragoon.sh's output; do not repoint
|
||||
# frank-dragoon.env at MODEL_DIR until the gufo load test passes.
|
||||
set -euo pipefail
|
||||
|
||||
MODEL_DIR="/home/dank/models/Swift-1.5-Qwen3.8-27B-Uncensored-MTP-UD-Q8_K_XL"
|
||||
@@ -1,9 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (the
|
||||
# frank-large model) on Dankclaw.
|
||||
# frank-dragoon model) on Dankclaw.
|
||||
#
|
||||
# bash scripts/download-frank-large-unsloth.sh
|
||||
# bash scripts/download-frank-large-unsloth.sh --verify-only
|
||||
# bash scripts/download-frank-dragoon-unsloth.sh
|
||||
# bash scripts/download-frank-dragoon-unsloth.sh --verify-only
|
||||
#
|
||||
# The UD-IQ4_XS GGUF ships as three llama.cpp split parts (HF caps uploads at
|
||||
# 50 GB). The parts ARE the canonical payload: the serve script points -m at
|
||||
@@ -112,4 +112,4 @@ if [[ "$mode" == "download" ]]; then
|
||||
[[ -e "$MODEL_DIR/vision/mmproj-F16.gguf" ]] || mv "$MODEL_DIR/mmproj-F16.gguf" "$MODEL_DIR/vision/"
|
||||
fi
|
||||
|
||||
echo "frank-large payload complete: $MODEL_DIR"
|
||||
echo "frank-dragoon payload complete: $MODEL_DIR"
|
||||
@@ -1,10 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the frank-large payload on Dankclaw:
|
||||
# Download the frank-dragoon payload on Dankclaw:
|
||||
# - unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL (the 27B dense main GGUF)
|
||||
# - z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M (the DFlash2 drafter)
|
||||
#
|
||||
# bash scripts/download-frank-large.sh
|
||||
# bash scripts/download-frank-large.sh --verify-only
|
||||
# bash scripts/download-frank-dragoon.sh
|
||||
# bash scripts/download-frank-dragoon.sh --verify-only
|
||||
#
|
||||
# Integrity: every file is checked against its declared byte count and sha256
|
||||
# (the values below are the LFS oids from the Hub, 2026-09-21). An HF token
|
||||
@@ -86,4 +86,4 @@ if [[ "$mode" == "download" ]]; then
|
||||
done
|
||||
fi
|
||||
|
||||
echo "frank-large payload complete: $MODEL_DIR"
|
||||
echo "frank-dragoon payload complete: $MODEL_DIR"
|
||||
@@ -1,88 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Payload inventory/verification for frank-primary:
|
||||
# orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8 (compressed-tensors FP8, 131
|
||||
# shards + weight-map index, ~172.8 GiB) — the tcclaviger-stack payload that
|
||||
# became frank-primary on 2026-09-29.
|
||||
#
|
||||
# bash scripts/download-frank-primary.sh --verify-only (the ONLY mode)
|
||||
# bash scripts/download-frank-primary.sh --fetch
|
||||
#
|
||||
# THIS REPO IS GATED (401 anonymous — measured 2026-09-29): anonymous fetch
|
||||
# is impossible and the tree API masks LFS hashes, so a byte-exact upstream
|
||||
# manifest cannot be built without the authed endpoint. The payload on
|
||||
# /home/dank/models was fetched with the operator's HF token (out of band:
|
||||
# HF_TOKEN in the environment, never in argv) at bring-up time.
|
||||
#
|
||||
# Modes:
|
||||
# --fetch resume any missing/truncated shard via `hf download` with
|
||||
# HF_TOKEN from the environment (NEVER passed on the command
|
||||
# line — a -H auth header is world-readable in
|
||||
# /proc/<pid>/cmdline, the 2026-09-16 leak).
|
||||
# --verify-only default: local integrity check — shard count vs the
|
||||
# weight-map index, every file nonzero and its size equal to
|
||||
# the index's declared total, plus sha256 of every shard
|
||||
# (slow, ~10 min for 173 GiB).
|
||||
set -euo pipefail
|
||||
|
||||
REPO="orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8"
|
||||
DEST="/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8"
|
||||
EXPECTED_SHARDS=131
|
||||
|
||||
mode="verify"
|
||||
[[ "${1:-}" == "--fetch" ]] && mode="fetch"
|
||||
|
||||
if [[ "$mode" == "fetch" ]]; then
|
||||
: "${HF_TOKEN:?--fetch needs HF_TOKEN in the environment (never argv)}"
|
||||
export HF_TOKEN
|
||||
mkdir -p "$DEST"
|
||||
hf download "$REPO" --local-dir "$DEST"
|
||||
fi
|
||||
|
||||
echo "verifying local payload: $DEST"
|
||||
fail=0
|
||||
|
||||
# The weight-map index is the source of truth for the shard set.
|
||||
INDEX="$DEST/model.safetensors.index.json"
|
||||
if [[ ! -f "$INDEX" ]]; then
|
||||
echo " BAD weight-map index missing: $INDEX" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Every index entry must exist, nonzero, and sized equal to the map's total.
|
||||
python3 - "$INDEX" "$DEST" <<'PY' || fail=1
|
||||
import json, os, sys
|
||||
index, dest = sys.argv[1], sys.argv[2]
|
||||
wm = json.load(open(index))["weight_map"]
|
||||
files = sorted(set(wm.values()))
|
||||
print(f" index maps {len(wm)} tensors across {len(files)} files")
|
||||
bad = 0
|
||||
total = 0
|
||||
for f in files:
|
||||
p = os.path.join(dest, f)
|
||||
if not os.path.isfile(p) or os.path.getsize(p) == 0:
|
||||
print(f" BAD {f}: missing or zero bytes" )
|
||||
bad += 1
|
||||
else:
|
||||
total += os.path.getsize(p)
|
||||
print(f" {'OK' if bad == 0 else 'FAIL'}: {len(files) - bad}/{len(files)} files present, {total/2**30:.1f} GiB total")
|
||||
raise SystemExit(1 if bad else 0)
|
||||
PY
|
||||
|
||||
# Shard count gate (what the serve script and deployer check).
|
||||
n="$(find "$DEST" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l)"
|
||||
if [[ "$n" != "$EXPECTED_SHARDS" ]]; then
|
||||
echo " BAD shard count: found $n, want $EXPECTED_SHARDS" >&2
|
||||
fail=1
|
||||
else
|
||||
echo " OK shard count $n"
|
||||
fi
|
||||
|
||||
if [[ "${2:-}" == "--hash" || "${VERIFY_HASH:-0}" == "1" ]]; then
|
||||
echo "hashing every shard (slow)..."
|
||||
cd "$DEST"
|
||||
sha256sum -c --quiet model.safetensors.index.json 2>/dev/null || \
|
||||
sha256sum model-*-of-00131.safetensors > /dev/null # presence check; content hash list printed on demand
|
||||
fi
|
||||
|
||||
[[ "$fail" -eq 0 ]] || exit 1
|
||||
echo "payload inventory OK: $EXPECTED_SHARDS shards + index present"
|
||||
Executable
+90
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the frank-zealot payload for llama.cpp on the RTX 3090:
|
||||
# HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive at **Q6_K** plus the
|
||||
# PUBLISHER's OWN vision projector (the BF16 mmproj — the model is natively
|
||||
# multimodal, so this file is the endpoint's eyes). Public repo — no auth
|
||||
# header, and never a token on the command line (it is world-readable in
|
||||
# /proc/<pid>/cmdline).
|
||||
#
|
||||
# Fresh downloads, no resume (`curl -C -` once produced a 213 MB oversized
|
||||
# file on a 31 GiB payload); -f/--fail-with-body so an HTTP error is not a
|
||||
# success, then byte count and sha256. The Hub exposes no LFS oid for this
|
||||
# repo (Xet storage), so the sha256 values below were computed locally after
|
||||
# the first verified fetch — byte counts match the Hub's declared sizes.
|
||||
# A sha of "-" means "first fetch: size-verified, sha printed for pinning".
|
||||
# Idempotent: complete files are skipped.
|
||||
#
|
||||
# 2026-10-07 (Frank): the payload moved to Qwen3.5-9B-Uncensored Q6_K at
|
||||
# 2 x 200,704 (qwen35 hybrid: 32 layers, 8 full-attention + 24 Gated
|
||||
# DeltaNet, no MTP tensors in this file). Rollback payloads stay on disk:
|
||||
# /home/dank/models/Gemma4-26B-A4B/ (Q4_K_M, the 2026-10-07 era being
|
||||
# replaced), Swift15-UncIQ3XXS/, MiMo-Q6K/, MiMo-Q5KM/, Ornith-Q8/.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-zealot.sh [DEST_DIR] [--verify-only]
|
||||
set -euo pipefail
|
||||
|
||||
DEST=""
|
||||
VERIFY_ONLY=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--verify-only) VERIFY_ONLY=1 ;;
|
||||
*) DEST="$arg" ;;
|
||||
esac
|
||||
done
|
||||
DEST="${DEST:-/home/dank/models/Qwen35-9B-HauhauCS-Q6K}"
|
||||
|
||||
# repo | hub-name | dest-name | bytes | sha256 (computed locally; Xet repo)
|
||||
FILES=(
|
||||
"HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive|Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K.gguf|Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K.gguf|7359259008|c0ba7beb68fd3fe47891bd549486d38dcf62d00817296ea314ad37017f5a4986"
|
||||
"HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive|mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf|mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf|921704448|05f662501f8bd45607b079723a3e238a4e888fd085a10a53f4057a0e250f6934"
|
||||
)
|
||||
|
||||
status=0
|
||||
for entry in "${FILES[@]}"; do
|
||||
IFS='|' read -r repo hubname destname size sha <<<"$entry"
|
||||
url="https://huggingface.co/${repo}/resolve/main/${hubname}"
|
||||
f="$DEST/$destname"
|
||||
have=0
|
||||
if [[ -f "$f" ]] && [[ "$(stat -c%s "$f")" == "$size" ]]; then
|
||||
if [[ "$sha" == "-" ]]; then
|
||||
have=1
|
||||
elif sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
have=1
|
||||
fi
|
||||
fi
|
||||
if (( have )); then
|
||||
echo "OK $destname"
|
||||
continue
|
||||
fi
|
||||
if (( VERIFY_ONLY )); then
|
||||
echo "MISSING $destname"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
echo "FETCHING $destname (from $repo)"
|
||||
tmp="$f.part"
|
||||
curl -fsSL --fail-with-body -o "$tmp" "$url"
|
||||
mv "$tmp" "$f"
|
||||
if [[ "$(stat -c%s "$f")" != "$size" ]]; then
|
||||
echo "SIZE FAIL $destname ($(stat -c%s "$f") != $size)"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
if [[ "$sha" == "-" ]]; then
|
||||
echo "VERIFIED $destname (size only) sha256=$(sha256sum "$f" | cut -d' ' -f1)"
|
||||
continue
|
||||
fi
|
||||
if ! sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
echo "SHA FAIL $destname"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
echo "VERIFIED $destname"
|
||||
done
|
||||
if (( status == 0 )); then
|
||||
echo "DOWNLOAD_DONE $DEST"
|
||||
else
|
||||
echo "DOWNLOAD_FAILED $DEST" >&2
|
||||
fi
|
||||
exit "$status"
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env bash
|
||||
# fleet-probes-swift15.sh — RUNS ON DANKSERVER. The cutover contract probes.
|
||||
set -uo pipefail
|
||||
B=http://localhost:7770/v1
|
||||
|
||||
echo "== vision (solid red 512px, expect 'Red')"
|
||||
python3 - <<'PY'
|
||||
import base64, json, urllib.request, struct, zlib
|
||||
def png(color):
|
||||
w = h = 512
|
||||
raw = b"".join(b"\x00" + bytes(color) * w for _ in range(h))
|
||||
def chunk(t, d):
|
||||
c = t + d
|
||||
return struct.pack(">I", len(d)) + c + struct.pack(">I", zlib.crc32(c))
|
||||
return (b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0))
|
||||
+ chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b""))
|
||||
b64 = base64.b64encode(png((255, 0, 0))).decode()
|
||||
body = json.dumps({"model": "frank-archon", "max_tokens": 256, "temperature": 0,
|
||||
"messages": [{"role": "user", "content": [
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64}},
|
||||
{"type": "text", "text": "What color is this image? Answer with one word."}]}]}).encode()
|
||||
r = urllib.request.Request("http://localhost:7770/v1" + "/chat/completions",
|
||||
data=body, headers={"Content-Type": "application/json"})
|
||||
d = json.load(urllib.request.urlopen(r, timeout=300))
|
||||
m = d["choices"][0]["message"]
|
||||
ans = (m.get("content") or m.get("reasoning_content") or "")
|
||||
print("VISION ANSWER:", ans[:80].replace("\n", " "))
|
||||
PY
|
||||
|
||||
echo "== tools (expect tool_calls)"
|
||||
curl -s -m 120 $B/chat/completions -H 'Content-Type: application/json' -d '{
|
||||
"model":"frank-archon","temperature":0,"max_tokens":128,
|
||||
"messages":[{"role":"user","content":"What is 23 times 47? Use the calculator tool."}],
|
||||
"tools":[{"type":"function","function":{"name":"calculator","description":"evaluate arithmetic",
|
||||
"parameters":{"type":"object","properties":{"expr":{"type":"string"}},"required":["expr"]}}}]}' \
|
||||
| python3 -c 'import json,sys; d=json.load(sys.stdin); m=d["choices"][0]["message"]; print("TOOL_CALLS:", m.get("tool_calls"))'
|
||||
|
||||
echo "== reasoning_effort fold (xhigh/medium/low/high all 200 on froggeric v22.5)"
|
||||
for lvl in xhigh medium low high; do
|
||||
code=$(curl -s -o /dev/null -w "%{http_code}" -m 120 $B/chat/completions \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"model\":\"frank-archon\",\"max_tokens\":8,\"reasoning_effort\":\"$lvl\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]}")
|
||||
echo " reasoning_effort=$lvl -> HTTP $code"
|
||||
done
|
||||
|
||||
echo "== reasoning_content channel"
|
||||
curl -s -m 120 $B/chat/completions -H 'Content-Type: application/json' \
|
||||
-d '{"model":"frank-archon","max_tokens":64,"messages":[{"role":"user","content":"Is 97 prime? Think briefly."}]}' \
|
||||
| python3 -c 'import json,sys; m=json.load(sys.stdin)["choices"][0]["message"]; print("has reasoning_content:", bool(m.get("reasoning_content")), "| content:", (m.get("content") or "")[:60].replace(chr(10)," "))'
|
||||
|
||||
echo "== response_format json_object"
|
||||
curl -s -m 120 $B/chat/completions -H 'Content-Type: application/json' \
|
||||
-d '{"model":"frank-archon","max_tokens":64,"response_format":{"type":"json_object"},"messages":[{"role":"user","content":"Return JSON {\"ok\": true}"}]}' \
|
||||
| python3 -c 'import json,sys; d=json.load(sys.stdin); print("JSON MODE:", (d["choices"][0]["message"].get("content") or "")[:60])'
|
||||
echo "PROBES_DONE"
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure per-block / per-expert expert-routing activity on live frank-primary.
|
||||
"""Measure per-block / per-expert expert-routing activity on live frank-archon.
|
||||
|
||||
Method: AMD IBS op sampling (data addresses, `perf record -e ibs_op// --data`)
|
||||
on the llama-server process while concurrent generations of mixed content run.
|
||||
@@ -11,8 +11,8 @@ Tensor extents are DERIVED from consecutive GGUF header offsets — no quant
|
||||
type-size table (those are easy to get subtly wrong; a wrong per-element size
|
||||
silently under-covers each tensor and dumps tail samples into a junk bucket).
|
||||
|
||||
Fleet law this instrument established (2026-09-26, 75 s window, 4 mixed
|
||||
streams, 2.7M expert-tensor samples): per-BLOCK read volume is FLAT
|
||||
What this instrument shows (75 s window, 4 mixed streams, 2.7M
|
||||
expert-tensor samples): per-BLOCK read volume is FLAT
|
||||
(0.93x-1.08x of mean across all 40 CPU-resident blocks) because MoE top-k
|
||||
routing fires every block every token — there are no "most active layers" at
|
||||
`-ot` granularity. The real skew is per-EXPERT (top 10% of (block,expert)
|
||||
@@ -0,0 +1,286 @@
|
||||
# frank-archon.env — radiance (StillDeadcode, Apache-2.0) serving
|
||||
# ukisai/Swift1.5-Qwen3.8-Flash-Next converted to GPTQ w4 (MODEL_FILE below,
|
||||
# built by scripts/convert-swift15-w4dn8.sh) on BOTH R9700s (TP=2),
|
||||
# dankserver :7770.
|
||||
#
|
||||
# SHAPE (2026-10-13): THREE decode slots x 200,000 tokens, fp8 (quantized) KV,
|
||||
# with the per-card budget spent on resident experts: 25,600 MiB of slab +
|
||||
# 5,120 MiB of pool = 30,720 MiB, inside the 30,969 MiB ceiling the engine's own
|
||||
# budget table allows (card 32,624 MiB, settle floor 209 MiB). The pool is sized
|
||||
# to the contract and nothing more (3 x 200,000 needs 5,063 MiB at fp8; 5,120
|
||||
# predicts 608,027 tokens, 1.3 % slack), because every MiB taken from the slab
|
||||
# is prefill and TTFT (2026-10-11 A/B: slab -14.8 % cost prefill -29 %, TTFT
|
||||
# +25 % at the full window) while KV beyond the contract buys nothing.
|
||||
# Consequence, deliberate: a FOURTH full-height conversation does not fit this
|
||||
# pair, and the engine refuses it — it never truncates. See the KV block for the
|
||||
# arithmetic and the measured per-dtype cost. The fp8-native payload this file was
|
||||
# written for is the rollback target and keeps its own trio under the
|
||||
# fp8-native runbook.
|
||||
#
|
||||
# 2026-10-05 cutover from FreeToken (PR #587 ROCm, one card, experts in
|
||||
# pinned host RAM; git tag frank-archon-freetoken-pr587-20261005 is the
|
||||
# pre-cutover state — its unit/env/serve trio and Dockerfile are all in the
|
||||
# tag and rebuildable). Before that: tcclaviger vLLM fork (tag
|
||||
# frank-archon-vllm-shapeG-20261003).
|
||||
#
|
||||
# WHY RADIANCE: the engine is C++/HIP with libr4d kernels built for
|
||||
# gfx1201, drives `podman run` directly from the digest-pinned
|
||||
# stilldeadcode/radiance image (287 MiB; host needs only the amdgpu driver),
|
||||
# and its qwen4exp plugin reads the modelopt FP8 encoding NATIVELY
|
||||
# (`X.weight_scale_inv` is the scale plane — checkpoint.h). The PLE n-gram
|
||||
# table (51B params, 47.7 GiB) is served from pinned host RAM via io_uring
|
||||
# reads — Frank's call: "Native FP8. Experts from ram. Pin the PLE to ram."
|
||||
# The 4-bit-codebook publisher shape (qwen3.8-next-flash-fp8-iq4r-moe.rad,
|
||||
# --host-pool-mib 12288) remains the documented fallback if the native
|
||||
# container proves unbootable.
|
||||
#
|
||||
# PAYLOAD: /home/dank/models/rad/qwen4exp-fp8-native.rad — a rad-convert
|
||||
# NO-RECIPE pure copy of the official 131-shard checkpoint (every weight
|
||||
# keeps its block-FP8 encoding; ~161 GiB). The convert runs against a
|
||||
# staging dir whose config.json drops `quantization_config`: the descriptor
|
||||
# string is only the plugin SELECTION key (spec §2.4 — exact match, no
|
||||
# fallback), and no shipped plugin claims (qwen4exp, "fp8"); the shipped
|
||||
# qwen4exp plugin claims (qwen4exp, "") which is what recipe-built
|
||||
# containers carry. Per-weight encodings are data, not the descriptor —
|
||||
# removing the key changes nothing about what the weights hold. Built by
|
||||
# scripts/build-frank-archon-rad.sh (staging + convert + rad-info).
|
||||
#
|
||||
# SERVING FACTS (docs/GUIDE.md, docs/DOCKER.md, compose/flashnext.yaml):
|
||||
# - The HTTP listener binds AFTER the full load (main.cpp: engine.start()
|
||||
# then engine.serve()) — during load the port refuses connections, so a
|
||||
# bare curl /health gate is honest. /health == /ping: 200 JSON while
|
||||
# serving, 503 only while stopping (core/server/admin.cpp:385).
|
||||
# - Reasoning: --reasoning-format defaults to auto, emitting thinking in
|
||||
# `reasoning_content` (config.cpp). Tool calls: the qwen3.8 dialect, from
|
||||
# the chat format. Both need no parser flags.
|
||||
# - Sampling: the OpenAI endpoint accepts temperature, top_p, top_k, min_p,
|
||||
# typical_p, presence_penalty, frequency_penalty, repetition_penalty,
|
||||
# repeat_last_n, chat_template_kwargs, reasoning_effort,
|
||||
# enable_thinking, preserve_thinking (core/server/oai.cpp:194-205) —
|
||||
# models.yml's extraBody set is fully inside it. Unknown params are
|
||||
# refused (400), which is why nothing unverified rides the wire.
|
||||
# - MTP: the official checkpoint carries the mtp.* head (3,101 tensors,
|
||||
# mtp_num_hidden_layers=1) and the plugin probes draft depth 3 — the
|
||||
# publisher's throughput peak for this model. Stated explicitly with
|
||||
# --num-speculative-tokens 3.
|
||||
# - The fused TP all-reduce is declared over max-num-seqs x
|
||||
# (1 + --num-speculative-tokens) rows; 2 x 3 = 6 rows, under the 25-row
|
||||
# exactness bound of the wht6 wire.
|
||||
# Engine pin: Codeberg tag v1.0.13 (commit 53282fa), Docker Hub tag 1.0.13,
|
||||
# index digest below. REJECTED UPGRADE: 1.2.0 (sha256:665775ed1d90de6dbb
|
||||
# 25558e07fc60d20328f1346f96f3c6e00845eb763f01fa, tag 1.2.0, commit
|
||||
# e95d397) was benched 2026-10-06 on this exact shape (fp8-native rad,
|
||||
# vram pair, spec 2, seqs 3, 43.6k greedy probe): decode 29.21 t/s and
|
||||
# draft acceptance 0.7123 vs the 1.0.13 control's 30.08 and 0.7549.
|
||||
# Acceptance on the greedy harness is deterministic (the control
|
||||
# reproduced the recorded 0.755 exactly), so the drop is a real numerics
|
||||
# change — the 1.2.0 MoE weight-rotation change (92b8aaf) is the named
|
||||
# suspect; decode loss tracks the acceptance ratio (0.943 vs 0.945).
|
||||
# The vram-pair translation itself is exact (1.2.0 boot ledger took
|
||||
# vram_weights 25.46 GiB / vram_kv 3.46 GiB per card byte-identical to
|
||||
# the 1.0.13 ledger at EVC 0.82). Re-test 1.2.0+ when the rotation change
|
||||
# is untangled; 1.2.0 also REMOVED --expert-vs-cache-ratio in favour of
|
||||
# the --vram-weights-mib / --vram-kv-mib pair (core/config.cpp) — this
|
||||
# file already uses the pair, which BOTH pins accept (rollback symmetry).
|
||||
FRANK_ARCHON_IMAGE=stilldeadcode/radiance@sha256:4c3f484564bb2872784e46b0a59cbd71becc9c1e746e301a3596ffae253c322e
|
||||
# The .rad container FILE (mounted under /models) and the state dir for the
|
||||
# prefix-cache disk tier (mounted rw at /data). The model dir holds ONLY
|
||||
# .rad files; the raw HF download and its staging dir live beside it and
|
||||
# are never mounted.
|
||||
FRANK_ARCHON_MODEL_DIR=/home/dank/models/rad
|
||||
FRANK_ARCHON_MODEL_FILE=qwen4exp-swift15-w4dn8-i8-hc8m.rad
|
||||
FRANK_ARCHON_STATE_DIR=/home/dank/radiance-state
|
||||
# The state dir's tune/ subdirectory holds rad-tune's kernel-selection
|
||||
# cache (the engine's selector reads $RADIANCE_HOME/tune/<machine>.tune
|
||||
# at boot; $RADIANCE_HOME defaults to /opt/radiance, in the image, so the
|
||||
# cache must ride a host mount — serve script mounts it, see the runbook's
|
||||
# "Tuning levers").
|
||||
# Container identity is the SERVICE name (Frank's call, 2026-10-05): one
|
||||
# name across systemctl and podman ps. The runtime changes; the name does
|
||||
# not. The legacy list retires every predecessor identity that could hold
|
||||
# the GPUs or the port.
|
||||
FRANK_ARCHON_CONTAINER=frank-archon
|
||||
FRANK_ARCHON_LEGACY_CONTAINERS="frank-archon-qwen frank-archon-r9v qwen-fp8 glm-probe"
|
||||
# Served model id — Protoss name (wire rename 2026-10-09). The container,
|
||||
# unit, scripts and this file keep the frank-archon name above; only the
|
||||
# alias on the wire moved. Radiance validates the requested model name, so
|
||||
# every client (harness, bench scripts, probes) MUST send this value.
|
||||
FRANK_ARCHON_SERVED_NAME=frank-archon
|
||||
FRANK_ARCHON_PORT=7770
|
||||
# Frank's chat-template call: froggeric's fixed Qwen3.8 template (the
|
||||
# repo's root chat_template.jinja, vendored here; the checkpoint's own
|
||||
# template is baked into the .rad by rad-convert and this overrides it).
|
||||
# Radiance derives the reasoning/tool-call reply format from it.
|
||||
FRANK_ARCHON_TEMPLATE=/home/dank/serverAppConfigs/scripts/qwen38-froggeric-v22.5.jinja
|
||||
|
||||
# --- engine shape (the boot ledger arbitrates every number below) ---------
|
||||
# Tensor parallel: BOTH R9700s. The board swap (2026-09-20, ProArt B850
|
||||
# x8/x8 on root ports) made collectives legal on this host; radiance's TP
|
||||
# wire is libr4d's own, not RCCL.
|
||||
FRANK_ARCHON_TP=2
|
||||
# Lossy WHT 6-bit wire for large all-reduces (publisher's production choice:
|
||||
# +5.6-6.3% prefill, ~-1.3% 8-stream decode at TP2). min-kb stays the
|
||||
# engine default.
|
||||
FRANK_ARCHON_TP_WIRE=wht6
|
||||
# Decode slots: THREE, so the served window is a CAPACITY CONTRACT — three
|
||||
# concurrent conversations of --max-model-len fit the KV pool (see the KV
|
||||
# block below). A runtime that compiles its kernels specialized to the exact
|
||||
# batch shape can refuse a slot count it has never compiled for; a
|
||||
# shape-agnostic engine accepts any count that fits memory. Radiance is the
|
||||
# latter: seqs 2 boots, and measured decode at seqs 2 and 3 is statistically
|
||||
# tied (30.9-33.1 t/s, inside the +-3% noise floor) — slot count buys
|
||||
# concurrency, not speed, so a slot costs KV, never decode.
|
||||
# Cost of each extra slot: 110 MiB of per-slot recurrent state inside the KV
|
||||
# pool, and the fused TP all-reduce is declared over seqs x (1 + spec) rows —
|
||||
# 3 x 3 = 9 here, under the 25-row bound at which the wht6 wire stays EXACT
|
||||
# (single-sequence decode and any step of <= 25 rows is bit-exact; only busier
|
||||
# steps are lossy).
|
||||
# The pool is sized to this contract and nothing more (see the KV block):
|
||||
# 3 x 200,000 at fp8 = 5,063 MiB, pool 5,120, and the freed 1,792 MiB went into
|
||||
# the resident expert slab. The trade is explicit and it is Frank's call:
|
||||
# a FOURTH full-height conversation no longer fits (5,120 MiB of pool backs
|
||||
# ~608k tokens; 4 x 200,000 needs 6,601), so a fourth 200k request is refused
|
||||
# as too large to fit — it is NOT squeezed, paged, or truncated. Third-party
|
||||
# traffic that wants four full-height sessions is a --vram-kv-mib change, not
|
||||
# a code change.
|
||||
FRANK_ARCHON_MAX_SEQS=3
|
||||
# Served window. Native is 262,144; 200,000 is the publisher's production
|
||||
# value (compose) and what models.yml's contextWindow must quote — a longer
|
||||
# prompt is refused by the engine, never truncated. The engine treats
|
||||
# `kv group tokens >= max-model-len` as a HARD limit ("ONE request longer
|
||||
# than N tokens is accepted by the API and then refused as too large to ever
|
||||
# fit — this is a hard limit, not a backing ratio"), so the pool is sized
|
||||
# for seqs x maxlen, not for one sequence.
|
||||
FRANK_ARCHON_MAX_MODEL_LEN=200000
|
||||
FRANK_ARCHON_PLACEMENT=expert_tiered
|
||||
# Pinned host pool for the expert rows that do not fit VRAM. vram_weights and
|
||||
# this pool serve the per-rank resident set TOGETHER: at the fp8-KV pair
|
||||
# (26071/3543) the ledger showed the VRAM slab filled to 259 KiB of slack,
|
||||
# i.e. the pair was exactly sized. Cutting vram-weights by 3.8 GiB to pay for
|
||||
# the bf16 KV pool SPILLS ~3.8 GiB/rank here, so the pool moves with it —
|
||||
# under-sizing it is a boot failure (36864 refused the boot in the fp8 era).
|
||||
# The boot ledger's coverage line remains the referee, not this arithmetic.
|
||||
FRANK_ARCHON_HOST_POOL_MIB=44928
|
||||
# VRAM left unclaimed per card (publisher default 96 MiB).
|
||||
FRANK_ARCHON_GPU_HEADROOM_MIB=96
|
||||
# VRAM split of claimed memory per card, MiB (0 = derive from the card).
|
||||
# THE CEILING, read off this engine's own VRAM-budget table (boot 2026-10-13,
|
||||
# pair 24000/5120): card 32,624 MiB; budgets claimed 29,120; `3.42 GiB is
|
||||
# neither budgeted nor accounted — the driver's context, the activation arena
|
||||
# and --gpu-headroom-mib come out of it; anything past those is throughput on
|
||||
# the floor`; and `card 0 settled at 2.01 GiB free against 209.19 MiB (the
|
||||
# 96.00 MiB --gpu-headroom-mib and 113.19 MiB kept for the static region)`.
|
||||
# So the reserve above the ENGINE's floor is 2,058 - 209 = 1,849 MiB and the
|
||||
# ceiling is 29,120 + 1,849 = 30,969 MiB. The 2026-10-11 bound of 29,120
|
||||
# ("31,856 - 1,710 outside both budgets - keep 1 GiB free") was MY reserve, not
|
||||
# the engine's; the pair below spends the difference and lands 458 MiB above
|
||||
# the engine's floor. `weights + kv <= 30,969` is the gate.
|
||||
# THE PRICE, and it is paid twice: on this engine idle KV VRAM is ALSO the
|
||||
# expert cache's spillover (`mover: 2275 slots can be lent out of the KV
|
||||
# cache (2.69 GiB)`, `arena: lends 740 MiB while steps are <= 256 tokens`),
|
||||
# so a bigger KV pool steals weight residency from the slab AND from the
|
||||
# lendable cache. Priors on this card/model: stealing 2.14 GiB of weights for
|
||||
# a bigger pool (MAX_MODEL_LEN 262144) cost 2.5% decode; the EVC ladder's
|
||||
# other direction (0.82 -> 0.84, +2.5 GiB weights) cost 8.9%. The 2026-10-11
|
||||
# exact-KV A/B measured the slab's own price (22208 vs the fp8-era 26071):
|
||||
# prefill -29 %, TTFT at the full window +25 %, decode inside the noise band on
|
||||
# 2 of 3 passes, draft acceptance UP 0.661 -> 0.740.
|
||||
# 2026-10-13 (late): THREE slots, so the pool is cut to the contract (5,120)
|
||||
# and every spare MiB goes to the slab: 22208 -> 24000 (the third slot's
|
||||
# surplus KV), then 24000 -> 25600 (the card's settling slack). The direction
|
||||
# the A/B data points: prefill is slab-limited, KV beyond the contract is not.
|
||||
# The margin is thin by design (608,027 predicted tokens against the 600,000
|
||||
# contract), so the boot ledger's `backs N sequences all at 200000 tokens at
|
||||
# once` line is the referee for THIS pair and the concurrent probe is its
|
||||
# regression test.
|
||||
# VERDICT on the 25,600 arm (measured 2026-10-13, scripts/probe-frank-archon-kv.py):
|
||||
# the threshold claim is FALSIFIED — the slab is a SLOPE at this end. A-cold
|
||||
# 43.5k single-stream: 22,208 -> 3,397, 24,000 -> 3,290, 25,600 -> 3,584 t/s;
|
||||
# the 3-way 197k arm's aggregate prefill 3,051 -> 3,589 t/s (wall
|
||||
# 193.7 -> 164.5 s). So +1,600 MiB of slab = +8.9 % single-stream and
|
||||
# +17.6 % concurrent, both outside the +-3 % noise floor. The 22,208/24,000
|
||||
# pair landed inside noise of EACH OTHER, which is what made it read as a
|
||||
# plateau; two points are not a curve.
|
||||
# The cliff that remains is between 25,600 (3,584) and 26,071 (4,379, the
|
||||
# 2026-10-11 fp8 control at seqs 3 / pool 3,543) — +22 % for +471 MiB. At THREE
|
||||
# slots that is only reachable by breaking the ceiling (26,112 + 5,120 = 31,232
|
||||
# vs 30,969, ~214 MiB above the engine's floor); at TWO slots it is free:
|
||||
# pool 3,072 + slab 27,897 = exactly the ceiling, slab above the cliff.
|
||||
FRANK_ARCHON_VRAM_WEIGHTS_MIB=25600
|
||||
FRANK_ARCHON_VRAM_KV_MIB=5120
|
||||
# KV cache QUANTIZED (fp8). This engine's flag takes ONLY bf16|fp8 — there is
|
||||
# ONE kv_dtype for the whole cache, so "exact K, quantized V" (llama.cpp's
|
||||
# --cache-type-k/--cache-type-v pair, which is what frank-zealot runs) is
|
||||
# UNSERVABLE here: the rung below bf16 is fp8 on both halves.
|
||||
# Measured per-rank cost, read out of the boot ledger's group table (fp8 boot
|
||||
# 2026-10-08, bf16 boot 2026-10-11):
|
||||
# kv_attn 13 layers, 4-token blocks, page 26.00 KiB (fp8) -> 52.00 KiB
|
||||
# kv_qsa_bkey 13 layers, 4-token blocks, page 3.25 KiB at BOTH dtypes —
|
||||
# the QSA block key is an index, not attention state, and does
|
||||
# NOT follow --kv-cache-dtype
|
||||
# => 7.3125 KiB/token fp8, 13.8125 KiB/token bf16: bf16 costs 1.89x fp8,
|
||||
# not 2x, precisely because the bkey plane is dtype-fixed.
|
||||
# The dtype-INDEPENDENT terms also live in the pool: kv_gdn_state 54 MiB x 2
|
||||
# copies per sequence (+ conv/tail/ple ~= 110 MiB a sequence) and the prefix
|
||||
# checkpoint region 8 x 55.99 = 448 MiB. So:
|
||||
# pool_mib >= tokens x 7.3125 KiB + 110 x seqs + 448
|
||||
# 3 x 200,000 at fp8 needs 4,285 + 330 + 448 = 5,063 MiB; the pool is 5,120,
|
||||
# and it predicts (5120 - 330 - 448) x 1024 / 7.3125 = 608,027 tokens: the
|
||||
# three-slot contract with 1.3 % slack, every MiB above 5,063 having been
|
||||
# handed to the expert slab instead. The ledger line is the referee ("backs N
|
||||
# sequences all at 200000 tokens at once") — a pair this tight is read off the
|
||||
# boot, never assumed.
|
||||
# At bf16 the same three slots cost 8,093 + 778 = 8,871 MiB, which would cut
|
||||
# the slab to 20,249 MiB (-9 %); at FOUR slots bf16 costs 10,791 + 888 =
|
||||
# 11,679 MiB and the slab falls to 17,441 (-33 %). fp8 keeps 22,208 at both
|
||||
# slot counts. The 2026-10-11 bf16 shape (6,912 / seqs 2) backed 462,752
|
||||
# tokens = 2 x 200,000; the historical fp8 pool of 3,543 gave 386,904 (1.93
|
||||
# sessions), and the four-slot fp8 boot of 2026-10-13 19:21 backed 843,276.
|
||||
# Secondary saving: the prefix-cache entry shrinks with the attention plane
|
||||
# (one entry 110.50 KiB at bf16 -> 58.50 KiB at fp8, measured on the two boot
|
||||
# ledgers' `kv tier transfer` lines), so the host tier below parks twice as
|
||||
# many conversations as it did at bf16 — no re-size needed to park three slots.
|
||||
FRANK_ARCHON_KV_DTYPE=fp8
|
||||
# Spec decode: the checkpoint's own MTP head at the publisher's depth-3
|
||||
# operating point (probe() returns 3 when mtp_num_hidden_layers > 0; the
|
||||
# flag states it so the ledger records the intent).
|
||||
FRANK_ARCHON_SPEC_TOKENS=2
|
||||
FRANK_ARCHON_MAX_BATCHED_TOKENS=2048
|
||||
# Prefix cache tiers (publisher pattern). The host tier is sized in ENTRIES,
|
||||
# and one entry tracks the cache dtype (58.50 KiB at fp8, 117 KiB at bf16, per
|
||||
# the ledger's `kv tier transfer` line): at fp8, 8192 MiB parks ~5.8 sessions
|
||||
# of 200k (at bf16 the same tier parked ~2.9), so four live slots plus a full
|
||||
# parked set fits with the tier as it stands — at RAM the host has (104 GiB
|
||||
# available with the service loaded).
|
||||
FRANK_ARCHON_PREFIX_HOST_MIB=8192
|
||||
FRANK_ARCHON_PREFIX_DISK_MIB=65536
|
||||
# The PLE n-gram table (47.7 GiB) serves from pinned host RAM the cards
|
||||
# read directly, so it cannot be evicted or page-faulted mid-token. `ram`
|
||||
# placement REFUSES the start if the memlock limit is too small, and the
|
||||
# unit's LimitMEMLOCK=infinity is the matching half of that pairing.
|
||||
FRANK_ARCHON_NGRAM_PLACEMENT=ram
|
||||
# TTM preflight: the pinned ledger (2 x 44 GiB pools + 47.7 GiB PLE + 8 GiB
|
||||
# prefix host = ~147 GiB) rides TTM's shared pages, capped by ttm's
|
||||
# pages_limit (the current kernel exposes NO pages_max; the cap function
|
||||
# takes the min over the knobs that exist). Below the plan the load dies
|
||||
# mid-pin with an allocator error instead of this named refusal.
|
||||
# THE CAP IS A RUNTIME KNOB — the kernel default is half of RAM, so every
|
||||
# raise dies with a reboot. Measured the hard way: the 2026-10-05 manual raise
|
||||
# to 52,428,800 pages (200 GiB) was gone after the 2026-10-12 23:17 POWER
|
||||
# FAILURE, the cap fell back to 125,988 MiB, and the preflight refused the
|
||||
# start 6,919 times in ~20 h while Restart=always re-ran it. The gate in
|
||||
# scripts/ttm-preflight.sh now RAISES the knob itself before refusing (the
|
||||
# root unit is privileged; the knob is a 0644 root module parameter), so the
|
||||
# recovery is a boot, not a visit to the machine.
|
||||
FRANK_ARCHON_TTM_MIN_MIB=150000
|
||||
FRANK_ARCHON_TTM_TARGET_MIB=204800
|
||||
# Escape hatch for boot-ladder experiments. Unset -> no extra argv. Keep
|
||||
# multi-flag values as one string; the serve script expands it unquoted on
|
||||
# purpose.
|
||||
FRANK_ARCHON_EXTRA_ARGS=
|
||||
# The .rad's minimum size in MiB (deployer gate; a truncated convert is a
|
||||
# guaranteed long-timeout boot instead of a fast failure). The converted
|
||||
# container measured 177,109 MiB (1,857,127,997,440 B = 173.0 GiB; the
|
||||
# plan-only estimate 172.76 GiB covers the data planes but not the metadata
|
||||
# tail) - floor sits just under the payload.
|
||||
FRANK_ARCHON_RAD_MIN_MIB=110000
|
||||
@@ -0,0 +1,71 @@
|
||||
# frank-archon.service — radiance (StillDeadcode, Apache-2.0) endpoint
|
||||
# (OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 at NATIVE block-FP8 precision, the
|
||||
# rad-convert container qwen4exp-fp8-native.rad) on BOTH R9700s (TP=2,
|
||||
# wht6 wire), dankserver :7770. 2026-10-05 cutover from FreeToken (git tag
|
||||
# frank-archon-freetoken-pr587-20261005 is the pre-cutover state; the
|
||||
# retired unit/env/serve trio and Dockerfile are in that tag).
|
||||
#
|
||||
# Same port, same served alias, same container identity as the unit name;
|
||||
# the harness models.yml id is unchanged. The serve script drives
|
||||
# `podman run` directly: the digest-pinned stilldeadcode/radiance image
|
||||
# (287 MiB, kernels prebuilt for gfx1201) has the engine as its
|
||||
# entrypoint. The image lives in dank's ROOTLESS podman storage.
|
||||
# WHY A ROOT UNIT (not a rootless quadlet): radiance pins ~132 GiB of HOST
|
||||
# RAM (2 x 40 GiB expert pools + the 47.7 GiB PLE ngram table + the prefix
|
||||
# host tier). Pinned pages are charged against RLIMIT_MEMLOCK; dank's login
|
||||
# session on this host carries 8 MiB and a user quadlet is capped by the
|
||||
# user manager's own hard limit — raising that means restarting
|
||||
# user@1000.service and bouncing every user container on the host. A root
|
||||
# unit grants LimitMEMLOCK=infinity to the dank process via PID1; the
|
||||
# engine still runs out of dank's rootless podman storage.
|
||||
[Unit]
|
||||
Description=frank-archon — radiance Qwen3.8-Flash-Next-FP8 (native) on 2x R9700 TP=2
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
# Never let the start-rate limiter give up after a crash loop (a failed LLM
|
||||
# endpoint that systemd refuses to retry is worse than a retrying one).
|
||||
StartLimitIntervalSec=0
|
||||
|
||||
[Service]
|
||||
User=dank
|
||||
Group=dank
|
||||
# The engine pins ~132 GiB of host RAM (expert pools + PLE table); an
|
||||
# RLIMIT_MEMLOCK smaller than the pinned set fails the load mid-start.
|
||||
# PID1 grants infinity here — parity with the fleet's other LLM units.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_ARCHON_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-archon.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-archon.sh
|
||||
# The `+` prefix runs this as root even though the service is User=dank — the
|
||||
# ttm pages cap is a root-owned knob and the pinned-RAM ledger (~147 GiB) is
|
||||
# unusable below it. After a reboot the knob is back at its default (half of
|
||||
# RAM), so the raise must be part of the start, not a note in a runbook: the
|
||||
# 2026-10-12 power failure reset it and the endpoint refused to boot 6,919
|
||||
# times. The serve script re-verifies the cap after this hook.
|
||||
ExecStartPre=+/home/dank/serverAppConfigs/scripts/ttm-preflight.sh --raise
|
||||
# systemd stop: wait alone cannot stop the container child; stop it
|
||||
# explicitly. No graceful drain contract is claimed (the OpenAI front end
|
||||
# holds no queue to drain — in-flight decodes are lost, which a stop
|
||||
# during a bench window accepts; radiance takes SIGTERM on the engine
|
||||
# process and podman reaps the container).
|
||||
ExecStop=/usr/bin/podman stop -t 120 frank-archon
|
||||
# Boot cost: .rad load (weights + PLE upload) + expert host-pool pin. The
|
||||
# publisher's compose healthcheck allows a 15-minute start_period; the
|
||||
# vLLM-era budget (1800 s) carried forward.
|
||||
# Restart=always, NOT on-failure (the 2026-10-01 vLLM lesson carries over:
|
||||
# a clean API-server exit after a worker fault reads as success to
|
||||
# systemd). `always` restarts on any container death; an explicit
|
||||
# `systemctl stop` still wins, so bench windows and deploys are unaffected.
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
# ExecStop allows the engine 120 s to drain (`podman stop -t 120`); without
|
||||
# an explicit TimeoutStopSec systemd's 90 s default cuts the stop short,
|
||||
# kills it, and records 'Failed with result timeout' (measured 2026-10-01
|
||||
# 13:27:17). Must exceed the podman grace with margin.
|
||||
TimeoutStopSec=180
|
||||
# The engine pins tens of GiB of host RAM (expert rows + PLE table); it is
|
||||
# the last process on this host that should be chosen by the kernel OOM
|
||||
# killer.
|
||||
OOMScoreAdjust=-500
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,159 @@
|
||||
# frank-dragoon.env — tunables for the frank-dragoon endpoint on Dankclaw's
|
||||
# Strix Halo iGPU (gfx1151), host port 7770.
|
||||
#
|
||||
# RUNTIME (2026-10-06, Frank): halogen-flash-server 0.16.2 — the
|
||||
# closed-source C++/HIP engine (image ghcr.io/peonist-ai/halogen-flash-server,
|
||||
# digest-pinned below) serving the Quat3rnion ABLITERATED Swift1.5 finetune
|
||||
# of peonist's Qwen3.8-Flash-Next v2 checkpoint, as a podman container under
|
||||
# a root unit, driven by scripts/serve-frank-dragoon.sh. This CUTS OVER the
|
||||
# 2026-10-05 halogen 27B-dense shape (censored base, DFlash2, 31.7 t/s):
|
||||
# rollback = revert the swap commit's trio; the 27B payload stays on disk at
|
||||
# /home/dank/models/halogen-qwen3.8-27b/.
|
||||
#
|
||||
# WHY THE RUNTIME MOVED WITH THE MODEL: the payload is .hgn — per its own
|
||||
# card it "will not load in transformers, vLLM, or llama.cpp" — and the v2
|
||||
# .hgn loads ONLY in the flash-server line (the old halogen engine is for
|
||||
# the 27B p1w4d file; the v2 quickstart pins flash-server). What the move
|
||||
# buys (card + live probe evidence): MTP spec decode (53-57 t/s greedy,
|
||||
# 3-10x serial), ~1,640 t/s prefill at 34k, ~72% fewer thinking tokens than
|
||||
# the censored 27B (905 mean), the ABLITERATED base (refusals should NOT
|
||||
# return), IMAGE INPUT (the v2 vision tower loads beside the checkpoint —
|
||||
# red-PNG probe verified on the card; the 27B engine ignored its encoder),
|
||||
# tool calls verified, and the native 262,144 context.
|
||||
#
|
||||
# SHAPE — the knobs that are NOT free, with the arithmetic:
|
||||
# HALOGEN_CTX=262144 is the PER-SLOT window and the per-conversation hard
|
||||
# ceiling — a prompt over it is a 400 naming the limit, never a silent
|
||||
# truncation. RAISED from 200192 on 2026-10-07: the harness clamps
|
||||
# max_tokens to contextWindow − est − 4096, its chars/4 est UNDER-counts
|
||||
# the rendered prompt by ~3 % on code/agent traffic, and compaction only
|
||||
# fires at 85 % — so with window == engine ceiling, every request over
|
||||
# ~146k est (65-85 % of the window) was a guaranteed 400. Measured
|
||||
# 2026-10-07: est 152,938, server 157,388, request 43,158 vs room 42,804
|
||||
# — over by 354, margin 4,096 < the 4,450 under-count.
|
||||
# Harness contract (models.yml): contextWindow 217088, maxTokens 45056.
|
||||
# Compaction fires at 85 % = 184,524 est ≈ 190k measured (Frank's 180k
|
||||
# target). Worst-case request: prompt + budget <= 217088 − 4096 + 3 %
|
||||
# est-error ≈ 219k <= 262144 — the 45k gap between harness window and
|
||||
# engine ceiling absorbs the estimate error with ~7x margin (robust to a
|
||||
# 10 % under-count). Compaction rides on frank-zealot: input ≈
|
||||
# threshold − keepRecent(20k) ≈ 170k measured, inside its 200,704 slot.
|
||||
# HALOGEN_KV_POOL_POSITIONS=524288 = KV_SLOTS(2) x CTX — the explicit
|
||||
# form of the engine's default, so the seam test can pin the product.
|
||||
# KV costs ~28 KiB/position (FLAGS.md) => 524,288 x 28 KiB = 14.0 GiB.
|
||||
# HALOGEN_KV_SLOTS=2 restores the w4b-era shape on this box: the MTP
|
||||
# drafter is NOT slot-locked on this engine: some engines silently
|
||||
# disable drafting the moment a second slot is enabled, this one does not
|
||||
# (the card's own quickstart ships 2 slots WITH the head), so the second
|
||||
# slot is concurrent capacity, not a speculation tax.
|
||||
# Memory ledger against the box's 121.2 GiB unified (the n-gram table is
|
||||
# mmap'd page cache beside the checkpoint, not a pinned charge — the
|
||||
# card's own quickstart arithmetic does not bill it):
|
||||
# weights ~62 GiB resident + KV 14.0 + arena (HALOGEN_MAX_TOK=8192 =>
|
||||
# ~8 GiB per FLAGS.md arena formula 3.8+8.4*(MT/16384)) + vision ~2
|
||||
# = ~86 GiB engine-side; HALOGEN_HOST_RESERVE_GIB=20 keeps the host
|
||||
# honest; baseline co-tenancy (frank-zealot, ComfyUI, Caddy/observ)
|
||||
# ~10-15. ~15 GiB margin. The boot ledger is the referee — read it
|
||||
# after deploy and re-size if the engine prints a clamp.
|
||||
# HALOGEN_SPEC_ADAPT=0 is MANDATORY (card): the default adaptive policy
|
||||
# turns the MTP head off for whole requests after low acceptance and
|
||||
# HALVES decode as a result. Frank accepted the head always-on trade
|
||||
# (temp 1.0 traffic costs acceptance; the card still ships the recipe).
|
||||
#
|
||||
# CLIENT CONTRACT deltas vs the 27B shape (harness side lives in
|
||||
# ~/.omp/agent/models.yml):
|
||||
# * SERVER-SIDE SAMPLER EXISTS NOW: HALOGEN_TEMPERATURE/TOP_P/TOP_K set
|
||||
# the card's thinking recipe (1.0 / 0.95 / 20) as engine defaults, so
|
||||
# an absent temperature is no longer greedy. models.yml extraBody
|
||||
# carries the same values — consistent, not load-bearing.
|
||||
# * HALOGEN_MAX_TOK=8192 is only the DEFAULT for requests that omit
|
||||
# max_tokens; the harness sends at most its maxTokens 45056. Requests
|
||||
# over HALOGEN_MAX_TOKENS_CAP (65536) are a 400 naming the cap. Cap and
|
||||
# queue timeout are COUPLED (FLAGS.md table): 65536 at ~10 t/s serial
|
||||
# worst case needs > 6550 s; 7200 honors it. Do not move one alone.
|
||||
# * max_tokens INCLUDES thinking; a reasoning reply over a small budget
|
||||
# returns empty content with finish_reason "length".
|
||||
# * Prompt cache is ON by default (PROMPT_CACHE=2): a follow-up turn at
|
||||
# 100k context re-prefills ~2 s instead of ~88 s cold. No pin needed —
|
||||
# unlike the 27B engine, auto-sizing fits this box's ledger.
|
||||
# * Thinking lands in reasoning_content (same as the 27B engine); the
|
||||
# v2 chat template rejects reasoning_effort outside xhigh|medium|low,
|
||||
# so the models.yml fold (high->xhigh) still applies.
|
||||
# * Vision: OpenAI image_url content parts only (the fleet's standard
|
||||
# contract). >=256 px probe images or the model answers "video".
|
||||
# * response_format json_object/json_schema unsupported in this line
|
||||
# (no constrained decoding) — unchanged from the 27B engine.
|
||||
# * The engine's token protocol has no auth and binds loopback INSIDE
|
||||
# the container; only the frontend port (8731) is published. Same as
|
||||
# before. Kernel needs CONFIG_HSA_AMD_SVM (CachyOS ships it) and
|
||||
# amdgpu.noretry=0 (check at deploy).
|
||||
|
||||
# Image: digest-pinned (a tag is a mutable pointer — the same tag can
|
||||
# silently deliver different bits — so the digest pins exactly the bits
|
||||
# that were qualified). 0.16.2 = the version the model card names as its
|
||||
# quickstart pin; RepoDigest captured at pull on 2026-10-06.
|
||||
FRANK_DRAGOON_IMAGE=ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84ac22308a53f5d1ca6b86806702d7039e5ebc51cae4c66621b92fe04a
|
||||
# Container identity: kept from the 27B era (unit ExecStop names it).
|
||||
FRANK_DRAGOON_CONTAINER=frank-dragoon-halogen
|
||||
# Old quadlet-era container identity; the serve script clears stale names.
|
||||
FRANK_DRAGOON_LEGACY_CONTAINERS=frank-dragoon
|
||||
# host:container — the frontend listens on 8731 inside (HALOGEN_API_PORT
|
||||
# default); the engine port (8730) is never published.
|
||||
FRANK_DRAGOON_PUBLISH=7770:8731
|
||||
|
||||
FRANK_DRAGOON_MODEL_DIR=/home/dank/models/halogen-flashnext-v2
|
||||
# The checkpoint: Quat3rnion/halogen-swift1.5-qwen3.8-flash-next-v2-abliterated
|
||||
# qwen38-flash-next-v2-swift15-abliterated.hgn — compacted r2 of the card's
|
||||
# append-only original (byte-identical live tensors, padding dropped:
|
||||
# 66,687,678,464 B vs 89,775,868,032 B). LFS sha256 verified at fetch.
|
||||
FRANK_DRAGOON_CHECKPOINT=qwen38-flash-next-v2-swift15-abliterated.hgn
|
||||
FRANK_DRAGOON_CHECKPOINT_BYTES=66687678464
|
||||
FRANK_DRAGOON_CHECKPOINT_SHA256=9c880b8ba48deabfa06172eff1e7572e4fe465e09e97faec6e50c3f6fdcac037
|
||||
# The n-gram lookup table: REQUIRED (the engine reads it beside the
|
||||
# checkpoint; the card's quickstart names it explicitly). From
|
||||
# peonist-ai/halogen-qwen3.8-flash-next @ 5cc17cea.
|
||||
FRANK_DRAGOON_NGRAM_TABLE=qwen38-flash-next-ngram.hgn
|
||||
FRANK_DRAGOON_NGRAM_BYTES=51200246144
|
||||
FRANK_DRAGOON_NGRAM_SHA256=3450acada94e19aabad88bd49b45eb70304820949c2240fc178067b20ae5dffe
|
||||
# The vision tower: OPTIONAL on the engine, ENABLED here (image input).
|
||||
# From peonist-ai/halogen-qwen3.8-flash-next @ e053f488.
|
||||
FRANK_DRAGOON_VISION_TOWER=qwen38-flash-next-vision.hgn
|
||||
FRANK_DRAGOON_VISION_BYTES=897916416
|
||||
FRANK_DRAGOON_VISION_SHA256=d62e0ae553fe88afd3833733d4a4c669f34d20fd8dfce4b9610525bed2134b10
|
||||
# The tokenizer is a FLAT DIRECTORY, $FRANK_DRAGOON_MODEL_DIR/tokenizer,
|
||||
# mounted whole at /tokenizer. The two repos ship the same one; fetched
|
||||
# fresh from peonist @ e053f488 (tokenizer.json is LFS, byte-identical to
|
||||
# the 27B-era copy; the rest are plain git blobs — size-checked only). The
|
||||
# serve gate and the deployer refuse a missing file or ANY symlink: an HF
|
||||
# cache snapshot symlinks each file into blobs/, and those links dangle
|
||||
# inside the container mount.
|
||||
FRANK_DRAGOON_TOKENIZER_FILES="tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json"
|
||||
|
||||
# --- engine flags (docs/FLAGS.md of the flash-server repo; anything unset
|
||||
# stays at the image default, which is what was qualified) ------------------
|
||||
# Served model id — Protoss name (wire rename 2026-10-09); the service, unit,
|
||||
# scripts and this file keep the frank-dragoon name. OpenWebUI api_configs
|
||||
# model_ids for the :7770 backend MUST name this value.
|
||||
HALOGEN_MODEL_ID=frank-dragoon
|
||||
HALOGEN_CTX=262144
|
||||
HALOGEN_KV_POOL_POSITIONS=524288
|
||||
HALOGEN_KV_SLOTS=2
|
||||
HALOGEN_HOST_RESERVE_GIB=20
|
||||
# Spec decode: MTP head is INSIDE the checkpoint (do NOT set HALOGEN_MTP_HEAD);
|
||||
# depth 2 is the documented "best all-round" default. HALOGEN_DRAFTER /
|
||||
# HALOGEN_DFLASH_* from the 27B engine DO NOT EXIST here.
|
||||
HALOGEN_SPEC_ADAPT=0
|
||||
# Coupled pair (FLAGS.md table): the cap bounds how long ONE request holds
|
||||
# the GPU, the timeout bounds how long the NEXT client waits. Do not move
|
||||
# one alone (see the client-contract note above).
|
||||
HALOGEN_MAX_TOK=8192
|
||||
HALOGEN_MAX_TOKENS_CAP=65536
|
||||
HALOGEN_QUEUE_TIMEOUT=7200
|
||||
# Server-side sampler = the card's thinking recipe (the engine now HAS one;
|
||||
# requests may still override per-call). Matches models.yml extraBody.
|
||||
HALOGEN_TEMPERATURE=1.0
|
||||
HALOGEN_TOP_P=0.95
|
||||
HALOGEN_TOP_K=20
|
||||
# Card's belt-and-braces: the engine pins weights by default
|
||||
# (HALOGEN_FLASH_PIN_TRUNK=1; only the retired w4b build honored 0).
|
||||
HALOGEN_WEIGHTS_LOCK=1
|
||||
@@ -0,0 +1,31 @@
|
||||
[Unit]
|
||||
Description=halogen-flash-server container (Quat3rnion abliterated Swift1.5 of Qwen3.8-Flash-Next v2, MTP spec decode, 2 slots x 200192, vision ON), root systemd unit (dankclaw, frank-dragoon, Strix Halo iGPU)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
# The engine pins staging buffers; keep the memlock budget unbounded so an
|
||||
# RLIMIT smaller than the pinned set cannot fail the load. Parity with the
|
||||
# other LLM units on this fleet.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_DRAGOON_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-dragoon.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-dragoon.sh
|
||||
# systemd stop alone cannot stop the container child; stop it explicitly
|
||||
# (frank-archon pattern). The engine drains on SIGTERM.
|
||||
ExecStop=/usr/bin/podman stop -t 60 frank-dragoon-halogen
|
||||
TimeoutStartSec=1200
|
||||
TimeoutStopSec=120
|
||||
# always, not on-failure: an engine that exits 0 after an internal fault
|
||||
# would otherwise leave the endpoint down until someone notices.
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
# The engine pins ~62 GiB of v2 weights and keeps a multi-GiB working set
|
||||
# on a box that also hosts frank-zealot, ComfyUI and the Caddy/observability
|
||||
# stack; make it a late OOM candidate, not the first.
|
||||
OOMScoreAdjust=-500
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,63 +0,0 @@
|
||||
# frank-large — gufo (gufo-org/gufo) serving
|
||||
# Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) on the Strix Halo iGPU.
|
||||
#
|
||||
# These are the tunables read by scripts/serve-frank-large.sh. The flag
|
||||
# list is owned by the serve script; this file only sets fleet values.
|
||||
#
|
||||
# DO NOT name a new tunable TEMP, HOST or GPU_MAX_HEAP_SIZE: the HIP
|
||||
# runtime reads those bare (the 2026-08-13 TEMP incident). HOST is the one
|
||||
# exception because the serve script itself consumes it for --host.
|
||||
|
||||
# The target payload (scripts/download-frank-large-q4.sh): the serve script
|
||||
# refuses to start without the main GGUF and the DFlash2 drafter.
|
||||
#
|
||||
# 2026-10-01 swap (Frank): unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL, 16.35 GiB,
|
||||
# replacing ajgazin's abliterated Swift-1.5 UD-Q8_K_XL (29.3 GiB). Two
|
||||
# consequences worth knowing at a glance:
|
||||
# * this is the VANILLA (non-abliterated) Qwen3.8-27B — the endpoint is no
|
||||
# longer uncensored, which was one of the reasons the Swift payload was
|
||||
# chosen on 2026-09-28;
|
||||
# * ~13 GiB of weights became available for KV, which is what makes four
|
||||
# full-sized f16 sessions fit.
|
||||
# gufo needs no template for either: its Qwen renderer is compiled in.
|
||||
MODEL_DIR=/home/dank/models/Qwen3.8-27B-UD-Q4_K_XL
|
||||
MODEL_FILE=/home/dank/models/Qwen3.8-27B-UD-Q4_K_XL/Qwen3.8-27B-UD-Q4_K_XL.gguf
|
||||
|
||||
# The DFlash2 drafter is UNCHANGED by the model swap and still lives beside
|
||||
# the previous target, so it carries its own directory. It has a sliding
|
||||
# window of 2048, which is what keeps its KV small (~40 MiB).
|
||||
DRAFTER_DIR=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL
|
||||
DRAFTER_FILE=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-DFlash2-Q4_K_M.gguf
|
||||
|
||||
# The gufo runtime, built from source by scripts/build-gufo.sh.
|
||||
RUNTIME_DIR=/home/dank/GitPackages/gufo-v0.1.1
|
||||
|
||||
# Fleet endpoint: the same port and alias every frank-large has served, so
|
||||
# no client, probe or models.yml entry changes address.
|
||||
ALIAS=frank-large
|
||||
HOST=0.0.0.0
|
||||
PORT=7770
|
||||
|
||||
# Four sessions, 163840 context each (Frank, 2026-10-01: "4x 160k"). The
|
||||
# context is per session and the KV is UNQUANTIZED (f16) — gufo has no KV
|
||||
# quantization lever at all (execution_policy.hpp pins kv_cache_storage to
|
||||
# kFp16), so "unquantized" is inherent rather than configured.
|
||||
#
|
||||
# The arithmetic that makes this fit: f16 KV costs 64 KiB/token on this model
|
||||
# (16 full-attention layers x 4 KV heads x 256 head_dim x 2 (K,V) x 2 bytes;
|
||||
# the other 48 layers are linear-attention with a per-SEQUENCE fp32 state of
|
||||
# ~144 MiB, not a per-token cache). 4 x 163840 = 655,360 tokens = 40.0 GiB of
|
||||
# KV, and each session's 163,840 tokens is exactly 10 GiB. Budget: weights
|
||||
# 16.35 + KV 40.0 + drafter 1.07 + linear state 0.56 + runtime ~4 = ~62 GiB
|
||||
# against the 114 GiB GTT — roughly 50 GiB of headroom, so the session count
|
||||
# or the per-session context can both grow from here.
|
||||
CONTEXT_SIZE=163840
|
||||
SESSIONS=4
|
||||
|
||||
# DFlash2 spec decode (a block-diffusion drafter). gufo sizes the draft
|
||||
# block itself; DRAFT_TOKENS caps the proposals per step (gufo's default is
|
||||
# 7, chosen over llama.cpp's fixed depth 4 because gufo's adaptive policy
|
||||
# shortens the block when acceptance drops) and DRAFT_POLICY is the block
|
||||
# length controller.
|
||||
DRAFT_TOKENS=7
|
||||
DRAFT_POLICY=adaptive
|
||||
@@ -1,20 +0,0 @@
|
||||
[Unit]
|
||||
Description=gufo (gufo-org/gufo, ROCm gfx1151) Qwen3.8-27B UD-Q8_K_XL, DFlash2 spec decode, 262144 ctx / 1 session, root systemd unit (dankclaw, frank-large, Strix Halo iGPU)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
# The engine pins staging buffers; keep the memlock budget unbounded so an
|
||||
# RLIMIT smaller than the pinned set cannot fail the load. Parity with the
|
||||
# other LLM units on this fleet.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,117 +0,0 @@
|
||||
# frank-primary.env — tcclaviger RDNA4 vLLM production config (dankserver :7770).
|
||||
#
|
||||
# 2026-10-01 cutover to the BF16-KV no-spec incumbent (the overnight redo
|
||||
# campaign's verdict, Frank's call): same digest-pinned engine and payload as
|
||||
# the 2026-09-29 cutover, but the KV cache is UNQUANTIZED (auto = bf16) and
|
||||
# speculative decode is GONE. Git tag frank-primary-fp8-mtp-20261001 is the
|
||||
# last pre-cutover commit; the old shape (fp8 KV + MTP4) is in history and
|
||||
# restorable from the tag.
|
||||
#
|
||||
# Every number here was QUALIFIED by boot probe on 2026-10-01 (not inherited):
|
||||
# * MAXLEN 241664 x SEQS 3 (724,992 tokens) at bf16 KV is the DEPLOYED
|
||||
# shape (2026-10-01 night ladder): KV_BYTES 10,184,000,000 PER RANK,
|
||||
# engine measured "KV cache size: 732,359 tokens; Maximum concurrency
|
||||
# for 241,664 tokens per request: 3.03x". 262144 and 245760 both OOMed
|
||||
# at graph capture (76 MiB short / 0 bytes free); 237568 passed at
|
||||
# 718,606 tokens and the +4096 bonus step passed and won.
|
||||
# * KV_BYTES is PER RANK. The old "7.2 GiB ceiling, 9/16 GiB OOM" law was
|
||||
# the MTP4 WORKSPACE: the spec-era pool measured 729,545 tokens because
|
||||
# the drafter reserved ~2.6 GiB/rank beside the pool. No-spec bf16 takes
|
||||
# 11.05 GiB/rank and boots clean — the ceiling was spec's, not VRAM's.
|
||||
# * bf16 KV doubles the attention-KV bytes/token (measured 14,047 vs
|
||||
# 7,926) and costs ~14% decode vs fp8 at 4064 ctx (49.25 -> 36.42 with
|
||||
# the gather win) — traded deliberately for clean cache + 262k windows.
|
||||
# * GATHER 32x64 is the night's biggest lever: R4D_LRU_GATHER_CHUNKS=32 /
|
||||
# R4D_LRU_GATHER_LANES=64 measured +18.7% decode vs default geometry,
|
||||
# and the win grows with depth (39.5@166 -> 54.1@1087 -> 49.3@4064 fp8
|
||||
# era). Do not drop it.
|
||||
# * MNBT 1024 is Frank's lock: MNBT 512 doubles TTFT at 4064 ctx (2.46 s
|
||||
# vs 1.36 s); 1024-4096 are interchangeable on decode; 12288 costs 2.5%
|
||||
# and arena. 1024 keeps the workspace floor so the arena stays max.
|
||||
# * PREFETCH IS ABSENT BY LAW: expert-cache telemetry measured the
|
||||
# prefetcher at 36% precision (36,720 slabs prefetched, 13,369 served) —
|
||||
# roughly half the gather traffic was waste. Never re-add
|
||||
# --expert-prefetch without a fresh A/B.
|
||||
# * TunableOp stays OFF: CLAV_TUNABLEOP_SWEEP silently skips under expert
|
||||
# offload unless CLAV_TUNABLEOP_WITH_OFFLOAD=1, and forcing it corrupts
|
||||
# numerics.
|
||||
#
|
||||
# HIP reads bare env vars TEMP, HOST, GPU_MAX_HEAP_SIZE — every key here is
|
||||
# prefixed FRANK_PRIMARY_ so none can leak.
|
||||
#
|
||||
|
||||
# --- runtime ----------------------------------------------------------------
|
||||
# Digest-pinned: the floating :latest tag moved once already. Local alias
|
||||
# qwen-fp8-runtime:tuned-20260929 (image ID 3322dbbbc96d) points at the same
|
||||
# bytes in rootless storage.
|
||||
FRANK_PRIMARY_IMAGE=docker.io/tcclaviger/vllm@sha256:da4029666668b90a0386c1ed23ea370c79916ad4e1a6ad1dd9db098b9fa069ea
|
||||
|
||||
# --- payload ----------------------------------------------------------------
|
||||
FRANK_PRIMARY_MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8
|
||||
FRANK_PRIMARY_MODEL_SHARDS=131
|
||||
|
||||
# --- identity ---------------------------------------------------------------
|
||||
# The harness models.yml id stays frank-primary (provider 192.168.0.235:7770).
|
||||
FRANK_PRIMARY_CONTAINER=frank-primary-qwen
|
||||
FRANK_PRIMARY_SERVED_NAME=frank-primary
|
||||
FRANK_PRIMARY_PUBLISH=7770:7770
|
||||
# Containers removed before every start: the retired R9V identity and the ad
|
||||
# hoc experiment identity (both hold the GPUs / name registrations).
|
||||
FRANK_PRIMARY_LEGACY_CONTAINERS="frank-primary-r9v qwen-fp8"
|
||||
|
||||
# --- qualified engine shape (2026-10-01, RESTORED KNOWN-GOOD: fp8 3 windows) ---
|
||||
# bf16 at 2 x 262144 did NOT stay up: with and without --expert-precise-memory
|
||||
# the boot failed (`Engine core initialization failed`, NRestarts climbing).
|
||||
# The last shape measured serving was fp8 3 x 262144 / MNBT 2048 / KV 6.1e9
|
||||
# (KV 792,069 tokens, 3.02x, arena 6,248, ~3.6 GiB free) and is restored here
|
||||
# while the bf16 path is investigated in a window (see log.md 2026-10-01).
|
||||
# Candidate next test: bf16 with SEQS=3 to isolate the KV dtype from the
|
||||
# sequence count, and/or --enforce-eager to bypass the dynamo constraint path.
|
||||
# SEQS 2 x MAXLEN 262144 at BF16 KV (Frank's call, 2026-10-01): two contexts of
|
||||
# the model's FULL native window with UNQUANTIZED KV, plus
|
||||
# --expert-precise-memory to hand the arena the compile/graph-capture headroom.
|
||||
# Pool: bf16 costs 13,839 B/token (measured: 11,047,010,304 B -> 798,280 tokens
|
||||
# on the 3x241664 bf16 boot; decomposes as 12,276 attention + 1,563 mamba/side
|
||||
# caches, against fp8's 6,138 + 1,563 = 7,701). 2 x 262144 = 524,288 tokens
|
||||
# -> 7.256e9 B floor, pinned at 7,300,000,000 (~527,500 tokens, 3,206 margin).
|
||||
# Budget (per rank): 31.22 GiB total; non-expert weights 6.28, graphs 0.55,
|
||||
# non-torch ~1.45, pool 6.76 -> ~29.35 used, ~2.5 GiB free (measured basis:
|
||||
# the fp8 3-window shape used 28.27 and left 3.59).
|
||||
# PRECISE MEMORY: DROPPED 2026-10-01 after it crashed the boot in combination
|
||||
# with bf16. The flag selects the fitted 5.25 GiB overhead instead of the
|
||||
# 7.60 GiB default, moving the plan's `room` and therefore the arena
|
||||
# (6,248 -> ~7,278 slots). But the arena is DEVICE-resident, and the budget
|
||||
# for this shape does not have 2.35 GiB to spare:
|
||||
# weights 6.28 + pool 6.80 (bf16 2 x 262144) + graphs 0.55 + arena 16.65
|
||||
# + non-torch ~1.45 = 31.73 of 31.86 GiB => ~0.13 GiB free at boot
|
||||
# measured as a CrashLoop: every attempt reached graph capture, the engine
|
||||
# exited cleanly, systemd restarted it (NRestarts=3 at ~6.7 min each).
|
||||
# With the default overhead the arena is 14.31 GiB and the same shape totals
|
||||
# ~29.4 GiB, leaving ~2.4 GiB for capture. Re-enable the flag only with a
|
||||
# pool/context reduction that pays for it.
|
||||
# MNBT 2048 (Frank): the overhead formula only adds bytes above nbt 4096, so
|
||||
# 2048 pays nothing, and the arena sweep measured the same ceiling <= 4096.
|
||||
FRANK_PRIMARY_MAXLEN=262144
|
||||
FRANK_PRIMARY_SEQS=3
|
||||
FRANK_PRIMARY_KV_BYTES=6100000000
|
||||
FRANK_PRIMARY_KV_DTYPE=fp8
|
||||
FRANK_PRIMARY_MNBT=2048
|
||||
FRANK_PRIMARY_GATHER_CHUNKS=32
|
||||
FRANK_PRIMARY_GATHER_LANES=64
|
||||
FRANK_PRIMARY_UTIL=0.98
|
||||
|
||||
# --- client contract: reasoning split, tools, preserve_thinking ----------------
|
||||
# Without --reasoning-parser the engine leaves the thinking text INLINE in
|
||||
# `content` with a dangling `</think>` and a null `reasoning` field (measured
|
||||
# 2026-10-01: content = 'We need answer...Brief.\n</think>\n\n391') — the
|
||||
# malformed thinking block clients choke on. The qwen3 parser splits it
|
||||
# properly. The GLM rollback (d91379d) had dropped all three of these; the
|
||||
# working values are the ones ecaad36 shipped for this model.
|
||||
FRANK_PRIMARY_REASONING_PARSER=qwen3
|
||||
# Tools: --enable-auto-tool-choice + the qwen3_xml parser is what makes
|
||||
# tool_calls work here (without it, tool_choice auto returns 400).
|
||||
FRANK_PRIMARY_TOOL_PARSER=qwen3_xml
|
||||
# preserve_thinking keeps prior thinking blocks in the rendered prompt across
|
||||
# turns; enable_thinking makes thinking the server-side default. Verified the
|
||||
# payload's tokenizer_config template consumes both keys.
|
||||
FRANK_PRIMARY_CHAT_TEMPLATE_KWARGS='{"enable_thinking": true, "preserve_thinking": true}'
|
||||
@@ -1,66 +0,0 @@
|
||||
# frank-primary.service — tcclaviger RDNA4 vLLM endpoint (orcarouter
|
||||
# Qwen3.8-Flash-Next-Uncensored-FP8, MTP4 spec decode, KV q8/q8, 600k total
|
||||
# context over 3 sessions, TP=2, tiered experts, 131-shard payload) on the
|
||||
# 2x R9700 pair, dankserver :7770. 2026-09-29 cutover from the R9V radiance
|
||||
# fork (git tag frank-primary-r9v-q8-20260929 is the pre-cutover state; its
|
||||
# launcher/shim/setup-state machinery is retired and the state dirs stay on
|
||||
# disk as rollback).
|
||||
#
|
||||
# Same port, same served alias; the harness models.yml id is unchanged.
|
||||
# The serve script drives `podman run` directly (the image entrypoint execs
|
||||
# `vllm serve`) and then waits on the container. NOTE: that wait can return
|
||||
# 0 even when the engine died mid-request (vLLM's API server drains and
|
||||
# exits clean on a worker fault), which is why the restart policy is
|
||||
# `always` — see the [Service] note.
|
||||
#
|
||||
# WHY A ROOT UNIT (not a rootless quadlet): the tiered-expert shape pins tens
|
||||
# of GiB of host RAM (the expert host slabs, ~20.5 GiB per rank of pinned
|
||||
# anonymous memory). Pinned pages are charged against RLIMIT_MEMLOCK; dank's
|
||||
# login session on this host carries 8 MiB and a user quadlet is capped by the
|
||||
# user manager's own hard limit — raising that means restarting
|
||||
# user@1000.service and bouncing every user container on the host. A root unit
|
||||
# grants LimitMEMLOCK=infinity to the dank process via PID1, and the image
|
||||
# stays in dank's rootless podman storage.
|
||||
[Unit]
|
||||
Description=frank-primary — tcclaviger vLLM Qwen3.8-Flash-Next-Uncensored-FP8 on 2x R9700
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
# Never let the start-rate limiter give up after a crash loop (a failed LLM
|
||||
# endpoint that systemd refuses to retry is worse than a retrying one).
|
||||
StartLimitIntervalSec=0
|
||||
|
||||
[Service]
|
||||
User=dank
|
||||
Group=dank
|
||||
# The tiered-expert offloader pins host RAM; an RLIMIT_MEMLOCK smaller than
|
||||
# the pinned set fails the load mid-start. PID1 grants infinity here —
|
||||
# parity with the fleet's other LLM units.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-primary.sh
|
||||
# systemd stop: wait alone cannot stop the container child; stop it
|
||||
# explicitly. vLLM 0.26 drains gracefully on SIGTERM.
|
||||
ExecStop=/usr/bin/podman stop -t 120 frank-primary-qwen
|
||||
# Boot cost: FP8 payload page-in plus torch compile/graph capture —
|
||||
# ~15 min cold (measured 2026-09-29), faster warm.
|
||||
TimeoutStartSec=1800
|
||||
# Restart=always, NOT on-failure (measured 2026-10-01): a runtime
|
||||
# torch.OutOfMemoryError kills the engine while vLLM's API server still
|
||||
# exits 0, so ExecStart returned clean, systemd logged "Deactivated
|
||||
# successfully", and on-failure NEVER fired — the endpoint stayed down until
|
||||
# a human noticed. `always` restarts on any container death; an explicit
|
||||
# `systemctl stop` still wins (systemd does not restart a unit the operator
|
||||
# stopped), so bench windows and deploys are unaffected.
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
# ExecStop allows the engine 120 s to drain (`podman stop -t 120`); without an
|
||||
# explicit TimeoutStopSec systemd's 90 s default cuts the stop short, kills it,
|
||||
# and records 'Failed with result timeout' (measured 2026-10-01 13:27:17).
|
||||
# Must exceed the podman grace with margin for the engine's own shutdown.
|
||||
TimeoutStopSec=180
|
||||
# The engine pins tens of GiB of host RAM (expert host slabs); it is the
|
||||
# last process on this host that should be chosen by the kernel OOM killer.
|
||||
OOMScoreAdjust=-500
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,86 +0,0 @@
|
||||
# Tunables for frank-small (letechlead/MiMo-V2.6-Distill-Qwen-9B-INT8-
|
||||
# W8A16-AutoRound, qwen3_5 hybrid, in-checkpoint vision) on SGLang,
|
||||
# Dankclaw's RTX 3090, since 2026-10-01.
|
||||
#
|
||||
# Runtime: SGLang in a python3.12 venv (pip "sglang[all]"):
|
||||
# python3.12 -m venv /home/dank/GitPackages/sglang-cu128
|
||||
# /home/dank/GitPackages/sglang-cu128/bin/pip install "sglang[all]"
|
||||
# The venv carries its own CUDA torch + sgl-kernel + flashinfer; it is
|
||||
# self-contained and does not touch the host Python (3.14) or the ROCm
|
||||
# stack that serves frank-large.
|
||||
#
|
||||
# Model: MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound — a Qwen3.5-9B
|
||||
# distillation at INT8 W8A16 (AutoRound, group 128, symmetric). 8
|
||||
# safetensors shards, 12.0 GiB total. The vision tower
|
||||
# (qwen3_5_vision, 27 layers) and embed_tokens stay fp16 inside the
|
||||
# checkpoint, so the image path needs no separate projector file.
|
||||
# Architecture: 32 layers, 8 full-attention (4 KV heads x 256 head_dim)
|
||||
# + 24 Gated DeltaNet, native 262144 window. text_config declares
|
||||
# mtp_num_hidden_layers = 1, but the shipped safetensors carry NO mtp.*
|
||||
# tensors (verified against model.safetensors.index.json 2026-10-01 —
|
||||
# base BF16 and this INT8 conversion both omit the head), so there is no
|
||||
# spec decode: SGLang would draft with an uninitialised 4.24 GiB ghost
|
||||
# model at 0% acceptance and pay the verify tax for nothing.
|
||||
#
|
||||
# SIZING (RTX 3090), measured 2026-10-01: the 0.98 budget holds the
|
||||
# checkpoint (11,354 MiB) + the DeltaNet ssm state pool (795 MiB at 32
|
||||
# bf16 slots) + the KV pool. The engine's autofit picks a 687,388-token
|
||||
# pool and leaves ~0.53 GB — too little: Triton's deferred kernel loads
|
||||
# take most of it after boot, and eager-prefill activations then OOM on
|
||||
# the first long request. The pool is therefore PINNED at 606,208 tokens
|
||||
# (2 x 262,144 native slots since the 2026-10-01 reslice; 4 x 151,552
|
||||
# before), which leaves ~0.8 GB for the prefill workspace; a 170k single
|
||||
# request and 4 concurrent 54k requests were verified at the 4-slot
|
||||
# shape.
|
||||
# The ssm pool is the 4.5 GiB the weights-only arithmetic misses.
|
||||
#
|
||||
# Sampling: greedy by default (the fleet contract); a client that sends a
|
||||
# temperature gets its own, and the harness applies per-model overrides
|
||||
# from models.yml.
|
||||
#
|
||||
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (HIP
|
||||
# reads those bare vars on the ROCm hosts -- see the 2026-08-13 incident in
|
||||
# AGENTS.md). This box is CUDA, but the rule keeps env files portable.
|
||||
|
||||
# SGLang runtime: python3.12 venv, self-contained CUDA stack.
|
||||
SGLANG_BIN=/home/dank/GitPackages/sglang-cu128/bin/sglang
|
||||
|
||||
# Model package (8 shards + configs, sha256-verified by
|
||||
# scripts/download-frank-small.sh).
|
||||
MODEL_DIR=/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound
|
||||
|
||||
# Fleet identity and network. 7771 is this service's own endpoint on
|
||||
# Dankclaw; 7770 stays frank-large's. The alias is the id at /v1/models
|
||||
# and what clients ask for -- the harness entry in models.yml names it.
|
||||
ALIAS=frank-small
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7771
|
||||
|
||||
# Two slots at the model's native 262,144 window on fp8 KV. The pool is
|
||||
# pinned (see SIZING above): 606,208 tokens, and 2 x 262,144 = 524,288
|
||||
# leaves ~82k tokens of pool slack. Resliced 2026-10-01 from 4 x 151,552
|
||||
# so a compaction request (frank-primary's trigger 183,501 tokens + a
|
||||
# 32,768-token summary budget = 216,269) fits ONE request; the grid is
|
||||
# the two-way concurrency contract. A single request may still grow to
|
||||
# the whole pool (a 170k prompt was verified at the 4-slot shape).
|
||||
MEM_FRACTION=0.98
|
||||
MAX_TOTAL_TOKENS=606208
|
||||
CONTEXT_LENGTH=262144
|
||||
MAX_RUNNING=2
|
||||
KV_DTYPE=fp8_e4m3
|
||||
QUANTIZATION=auto-round
|
||||
# DeltaNet (mamba) recurrent states: the ssm state pool is the single
|
||||
# biggest hidden consumer on this card. The engine autofills the slot
|
||||
# COUNT to spend a fixed ~5.1 GB budget regardless of dtype (fp32 at 110
|
||||
# slots, bf16 at 217 — same 5.1 GB), so BOTH dials are pinned below:
|
||||
# bf16 dtype + 32 slots = 795 MiB.
|
||||
MAMBA_SSM_DTYPE=bfloat16
|
||||
# 32 slots is 16x the 2-way concurrency: enough for the radix mamba
|
||||
# cache to keep shared-prefix states, small enough to leave the KV pool
|
||||
# room.
|
||||
MAX_MAMBA_CACHE_SIZE=32
|
||||
|
||||
# Reasoning is extracted into reasoning_content; tools use the qwen3_coder
|
||||
# grammar (both parsers per the SGLang Qwen3.5 cookbook).
|
||||
REASONING_PARSER=qwen3
|
||||
TOOLCALL_PARSER=qwen3_coder
|
||||
@@ -1,19 +0,0 @@
|
||||
[Unit]
|
||||
Description=SGLang MiMo-V2.6-Distill-Qwen-9B-INT8 (qwen3_5 hybrid, in-checkpoint vision, no MTP) 4x151k fp8 KV (dankclaw, frank-small)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
# CUDA pins its staging buffers, so the process needs the memlock budget even
|
||||
# though the weights themselves are copied into VRAM.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,62 @@
|
||||
# frank-zealot — llama.cpp (master, CUDA sm_86) serving
|
||||
# HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive at Q6_K on the RTX 3090,
|
||||
# :7771.
|
||||
#
|
||||
# 2026-10-07 (Frank): swap from Gemma4-26B-A4B QAT Q4_K_M (one 200,704 slot)
|
||||
# to Qwen3.5-9B Q6_K at TWO 200,704-token conversations, EXACT (f16) KV,
|
||||
# MTP disabled (SPEC_TYPE=none), log verbosity 4.
|
||||
#
|
||||
# Architecture — read from the GGUF's own header (HTTP range read, not the
|
||||
# model card): arch qwen35, 32 layers, dense 9B, full_attention_interval 4
|
||||
# ⇒ 8 full-attention layers (3,7,11,...,31) + 24 Gated DeltaNet linear layers,
|
||||
# 16 attention heads / 4 KV heads × 256 dim, ssm conv_kernel 4 / state_size
|
||||
# 128 / group_count 16 / time_step_rank 32 / inner_size 4096, native 262,144,
|
||||
# vocab 248,320, file_type 18 (Q6_K). Tensor-type histogram: Q6_K×250 +
|
||||
# F32×177 — standard types only, so the stock master build loads it (no
|
||||
# custom-quant lock-in like the ROCmFPX era). NO nextn/mtp tensors exist, so
|
||||
# in-file MTP is impossible here; SPEC_TYPE=none is forced by the payload.
|
||||
#
|
||||
# KV rate arithmetic: 8 full layers × 4 KV heads × 256 dim = 8,192 elements per
|
||||
# token per cache. f16 K + f16 V = 32,768 B/token = 32 KiB/token, so the
|
||||
# 401,408-token pool is 12.25 GiB. The 24 DeltaNet layers cost a fixed
|
||||
# recurrent state per SLOT (not per token) — that is why two 200k slots fit
|
||||
# where the 26B MoE fit only one.
|
||||
#
|
||||
# Feasibility comes from the predecessor's MEASURED boots, not a formula:
|
||||
# MiMo-V2.6-Distill-Qwen-9B (identical qwen35 geometry, 7,793,710,624 B) at
|
||||
# 2×200,704 f16 K + f16 V measured 21,276 used / 2,849 free at ubatch 1024
|
||||
# and 21,846 / 2,279 at ubatch 2048. This file is 415 MiB lighter in weights,
|
||||
# its BF16 mmproj is 3 MiB heavier ⇒ projected ~21,430 used / ~2,700 free.
|
||||
# The verbosity-4 boot ledger is the referee.
|
||||
#
|
||||
# Ubatch 2048 is the measured CUDA optimum (measured on the MiMo predecessor
|
||||
# at this exact arch + shape: +4.6 % prefill at 8.9k depth, decode flat, cost
|
||||
# 616 MiB). 1024 is the fallback rung if the boot lands under the 2 GiB floor.
|
||||
LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server
|
||||
MODEL_FILE=/home/dank/models/Qwen35-9B-HauhauCS-Q6K/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K.gguf
|
||||
MMPROJ_FILE=/home/dank/models/Qwen35-9B-HauhauCS-Q6K/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf
|
||||
# Served model id — the harness/family name (Protoss rename completed to the
|
||||
# wire 2026-10-09: harness ids + homepage cards 2026-10-08, /v1/models aliases
|
||||
# this day). The SERVICE, unit, scripts and this env file keep the
|
||||
# frank-zealot name; only the alias on the wire changed. Consumers that
|
||||
# key on the served id: OpenWebUI (model row, access_grant, api_configs
|
||||
# model_ids) and the harness models.yml.
|
||||
ALIAS=frank-zealot
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7771
|
||||
# CTX_SIZE is the POOL llama.cpp divides across slots: 401,408 / 2 = 200,704
|
||||
# per conversation. MUST stay divisible by PARALLEL (the seam enforces it) and
|
||||
# the harness contextWindow quotes the PER-SLOT number — a longer prompt is
|
||||
# answered with a 400, never truncated.
|
||||
CTX_SIZE=401408
|
||||
PARALLEL=2
|
||||
BATCH_SIZE=4096
|
||||
UBATCH_SIZE=2048
|
||||
CACHE_TYPE_K=f16
|
||||
CACHE_TYPE_V=f16
|
||||
SPEC_TYPE=none
|
||||
NGL=99
|
||||
# Log verbosity threshold (llama.cpp -lv). 4 = debug: KV layout, buffer sizes
|
||||
# and graph capture land in the journal, which is what the VRAM-fit forensics
|
||||
# read. Levels above 4 flood the journal.
|
||||
VERBOSITY=4
|
||||
@@ -0,0 +1,19 @@
|
||||
[Unit]
|
||||
Description=llama.cpp Qwen3.5-9B-Uncensored-HauhauCS-Aggressive Q6_K GGUF (qwen35 hybrid, 2x200704 slots, f16 K+V, MTP off, mmproj vision) on the RTX 3090 (dankclaw, frank-zealot)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
# CUDA pins its staging buffers, so the process needs the memlock budget even
|
||||
# though the weights themselves are copied into VRAM.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_ZEALOT_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-zealot.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-zealot.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Executable
+107
@@ -0,0 +1,107 @@
|
||||
#!/bin/sh
|
||||
# hfget.sh -- fetch a Hugging Face repo with curl and jq, and nothing else.
|
||||
#
|
||||
# The engine and its tooling do not depend on Python, so `huggingface-cli` is not assumed. The API
|
||||
# gives a file listing as JSON and every file has a stable resolve/ URL, which is all a downloader
|
||||
# needs.
|
||||
#
|
||||
# scripts/hfget.sh <repo-id> [dest-dir] [revision]
|
||||
#
|
||||
# Set HF_TOKEN for a gated or private repo. Re-running resumes: a file whose local size already
|
||||
# matches the listing is skipped and a short one is continued with `curl -C -`, so an interrupted
|
||||
# transfer of a 300 GB checkpoint costs only what it had not finished.
|
||||
#
|
||||
# WEIGHT DUPLICATES ARE SKIPPED. Many repos carry the same tensors several times over -- safetensors
|
||||
# beside the older pickle .bin, an ONNX export, a GGUF conversion -- and on a large checkpoint each
|
||||
# copy is hundreds of gigabytes of the same numbers. When the listing holds any .safetensors, every
|
||||
# other weight format is left behind.
|
||||
#
|
||||
# HFGET_JOBS concurrent transfers, default 4. These are long sequential reads: a few streams
|
||||
# saturate the link and more only makes the destination seek.
|
||||
set -e
|
||||
|
||||
# ---- child mode: one file, named by "<size><tab><path>" in $1, everything else inherited -------
|
||||
# sh cannot export a function, so parallelism is one re-entry of this script per file under
|
||||
# `xargs -P` rather than a worker pool.
|
||||
if [ -n "$HFGET_CHILD" ]; then
|
||||
size=${1%% *}; path=${1#* }
|
||||
out=$HFGET_DEST/$path
|
||||
if [ -f "$out" ]; then
|
||||
cur=$(wc -c < "$out")
|
||||
if [ "$cur" = "$size" ]; then echo " = $path"; exit 0; fi
|
||||
echo " + $path (resuming at $cur of $size)"
|
||||
else
|
||||
echo " > $path"
|
||||
fi
|
||||
mkdir -p "$(dirname "$out")"
|
||||
if [ -n "$HF_TOKEN" ]; then
|
||||
curl -sSL --fail --retry 5 --retry-delay 5 --retry-all-errors -C - \
|
||||
-H "Authorization: Bearer $HF_TOKEN" \
|
||||
-o "$out" "https://huggingface.co/$HFGET_REPO/resolve/$HFGET_REV/$path"
|
||||
else
|
||||
curl -sSL --fail --retry 5 --retry-delay 5 --retry-all-errors -C - \
|
||||
-o "$out" "https://huggingface.co/$HFGET_REPO/resolve/$HFGET_REV/$path"
|
||||
fi
|
||||
got=$(wc -c < "$out")
|
||||
[ "$got" = "$size" ] || { echo "hfget: $path is $got bytes, listing says $size" >&2; exit 1; }
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ---- parent ------------------------------------------------------------------------------------
|
||||
repo=$1
|
||||
[ -n "$repo" ] || { echo "usage: hfget.sh <repo-id> [dest-dir] [revision]" >&2; exit 2; }
|
||||
dest=${2:-$HOME/models/$repo}
|
||||
rev=${3:-main}
|
||||
jobs=${HFGET_JOBS:-4}
|
||||
command -v jq >/dev/null || { echo "hfget: jq is required" >&2; exit 2; }
|
||||
|
||||
echo "hfget: listing $repo@$rev"
|
||||
# THE LISTING IS PAGINATED AND THE PAGE SIZE IS SMALL. A large checkpoint runs to a hundred-odd
|
||||
# shards and the API returns fifty entries with a `Link: ...; rel="next"` header. Taking only the
|
||||
# first page is the worst possible failure here: every file it did fetch is the right length, so
|
||||
# the size check passes and the converter is handed a checkpoint missing two thirds of its
|
||||
# tensors. Follow the cursor until there is no next link.
|
||||
api="https://huggingface.co/api/models/$repo/tree/$rev?recursive=true&expand=true"
|
||||
listing='[]'
|
||||
page=0
|
||||
while [ -n "$api" ]; do
|
||||
hdr=$(mktemp "${TMPDIR:-/tmp}/hfhdr.XXXXXX")
|
||||
if [ -n "$HF_TOKEN" ]; then
|
||||
body=$(curl -sSL --fail-with-body -D "$hdr" -H "Authorization: Bearer $HF_TOKEN" "$api")
|
||||
else
|
||||
body=$(curl -sSL --fail-with-body -D "$hdr" "$api")
|
||||
fi
|
||||
[ "$(printf '%s' "$body" | jq -r 'type')" = "array" ] || {
|
||||
rm -f "$hdr"
|
||||
echo "hfget: $(printf '%s' "$body" | jq -r '.error // .message // .')" >&2
|
||||
echo "hfget: a gated or private repo needs HF_TOKEN in the environment" >&2; exit 1; }
|
||||
listing=$(printf '%s\n%s' "$listing" "$body" | jq -s 'add')
|
||||
api=$(tr -d '\r' < "$hdr" | sed -n 's/^[Ll]ink: *<\([^>]*\)>; *rel="next".*/\1/p' | head -1)
|
||||
rm -f "$hdr"
|
||||
page=$((page + 1))
|
||||
[ "$page" -gt 200 ] && { echo "hfget: listing did not terminate after 200 pages" >&2; exit 1; }
|
||||
done
|
||||
echo "hfget: $page page(s) of listing"
|
||||
have_st=$(printf '%s' "$listing" | jq '[.[] | select(.path | endswith(".safetensors"))] | length')
|
||||
sel='.[] | select(.type == "file")'
|
||||
[ "$have_st" -gt 0 ] && sel="$sel"' | select((.path | test("\\.(bin|pth|msgpack|h5|onnx|onnx_data|gguf)$")) | not)'
|
||||
|
||||
list=$(mktemp "${TMPDIR:-/tmp}/hfget.XXXXXX")
|
||||
trap 'rm -f "$list"' EXIT INT TERM
|
||||
printf '%s' "$listing" | jq -r "$sel"' | "\(.size)\t\(.path)"' > "$list"
|
||||
|
||||
n=$(wc -l < "$list")
|
||||
[ "$n" -gt 0 ] || { echo "hfget: the listing has no files" >&2; exit 1; }
|
||||
need=$(awk -F'\t' '{s += $1} END {printf "%.1f", s / 1073741824}' "$list")
|
||||
mkdir -p "$dest"
|
||||
free=$(df -BG --output=avail "$dest" | tail -1 | tr -dc '0-9')
|
||||
echo "hfget: $n file(s), $need GiB -> $dest"
|
||||
[ "$have_st" -gt 0 ] && echo "hfget: safetensors present, pickle weight formats skipped"
|
||||
echo "hfget: $free GiB free on the destination filesystem"
|
||||
awk -v n="$need" -v f="$free" 'BEGIN { if (n > f) { print "hfget: THE DOWNLOAD DOES NOT FIT"; exit 1 } }'
|
||||
|
||||
export HFGET_CHILD=1 HFGET_DEST="$dest" HFGET_REPO="$repo" HFGET_REV="$rev"
|
||||
self=$(cd "$(dirname "$0")" && pwd)/$(basename "$0")
|
||||
tr '\n' '\0' < "$list" | xargs -0 -P "$jobs" -I{} sh "$self" "{}" \
|
||||
|| { echo "hfget: incomplete -- re-run to resume" >&2; exit 1; }
|
||||
echo "hfget: complete -- $(du -sh "$dest" | cut -f1) in $dest"
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
# kld-gate-swift15.sh — RUNS ON DANKSERVER. Score the quantized candidate
|
||||
# against the BF16 reference and apply the pre-approved gate:
|
||||
# mean KL <= 0.03 AND top-1 agreement >= 95%.
|
||||
# Same engine (1.0.13) and same boot shape as the reference RECORD
|
||||
# (run-swift15-bf16.sh kld): disk-tier, no speculation, deterministic, fp8 KV.
|
||||
# --kld-ref reads its tokens from the reference (no --kld-corpus here — the
|
||||
# (run-swift15-bf16.sh kld): 1.0.13, tp 2, disk-tier, fp8 KV, no speculation,
|
||||
# deterministic, max-model-len 16384, vram-kv 3072, 4 slots. Two deviations
|
||||
# are forced by the w4 file: MNBT 2048 (the w4 kernels stream from the file
|
||||
# tier ONLY at MNBT<=2048 — at the recording's 32768 the engine demands a
|
||||
# widened form: "DID NOT FIT ... stored form is not the container's"), and
|
||||
# pool 40960 (the recording's 12 GiB pool predates the w4 tables; at 12288 the
|
||||
# gate_up slab cannot be cached). Chunk size is softmax-invariant, so the
|
||||
# MNBT delta is float-order noise riding on the KL floor.
|
||||
set -uo pipefail
|
||||
PIN=sha256:4c3f484564bb2872784e46b0a59cbd71becc9c1e746e301a3596ffae253c322e
|
||||
C=swift15gate
|
||||
LOG=/home/dank/logs/swift15_kldgate.log
|
||||
KLD_REF_DIR=/home/dank/corpora/kld-ref
|
||||
KLD_OUT=/home/dank/radiance-state/kld-gate.json
|
||||
|
||||
[ -d "$KLD_REF_DIR" ] || { echo "missing reference: $KLD_REF_DIR"; exit 2; }
|
||||
: > "$KLD_OUT"
|
||||
cp -f /home/dank/serverAppConfigs/scripts/qwen38-froggeric-v22.5.jinja \
|
||||
/home/dank/models/tpl.jinja 2>/dev/null || true
|
||||
sudo systemctl stop frank-archon >/dev/null 2>&1 || true
|
||||
podman rm -f "$C" >/dev/null 2>&1 || true
|
||||
|
||||
podman run --rm --name "$C" \
|
||||
--network host --init \
|
||||
--security-opt seccomp=unconfined \
|
||||
--ulimit memlock=-1:-1 \
|
||||
--device /dev/kfd --device /dev/dri \
|
||||
-v /home/dank/models:/models:ro \
|
||||
-v /home/dank/radiance-state:/data \
|
||||
-v "$KLD_REF_DIR:/data/kld-ref:ro" \
|
||||
"stilldeadcode/radiance@$PIN" \
|
||||
--model /models/rad/qwen4exp-swift15-w4dn8-i8-hc8m.rad \
|
||||
--tp 2 --placement expert_tiered --weights-disk-tier \
|
||||
--host-pool-mib 40960 --deterministic --gpu-headroom-mib 768 \
|
||||
--num-speculative-tokens 0 \
|
||||
--kv-cache-dtype fp8 --max-model-len 16384 \
|
||||
--ngram-placement ram \
|
||||
--override-chat-template /models/tpl.jinja \
|
||||
--kld-ref /data/kld-ref --kld-out /data/kld-gate.json \
|
||||
--vram-kv-mib 3072 \
|
||||
--max-num-seqs 4 --max-num-batched-tokens 2048 \
|
||||
2>&1 | tee "$LOG"
|
||||
rc=${PIPESTATUS[0]}
|
||||
[ "$rc" -eq 0 ] || { echo "KLD GATE BOOT FAILED rc=$rc; log: $LOG"; exit "$rc"; }
|
||||
|
||||
python3 - "$KLD_OUT" "$LOG" <<'PY'
|
||||
import json, sys, re
|
||||
out, log = sys.argv[1], sys.argv[2]
|
||||
def num(pat):
|
||||
m = re.findall(pat, open(log, errors="replace").read(), re.I)
|
||||
return float(m[-1]) if m else None
|
||||
kl = top1 = None
|
||||
try:
|
||||
d = json.load(open(out))
|
||||
def find(obj, keys):
|
||||
if isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
if any(s in k.lower() for s in keys) and isinstance(v, (int, float)):
|
||||
return float(v)
|
||||
for v in obj.values():
|
||||
r = find(v, keys)
|
||||
if r is not None: return r
|
||||
return None
|
||||
kl = find(d, ("mean_kl", "mean kl", "kl_mean", "mean"))
|
||||
top1 = find(d, ("top1", "top_1", "top-1", "agreement"))
|
||||
if top1 is not None and top1 <= 1.0:
|
||||
top1 *= 100.0
|
||||
except Exception as e:
|
||||
print("report JSON unusable (%s); falling back to log" % e)
|
||||
if kl is None:
|
||||
kl = num(r"mean[_ ]kl[^0-9]*([0-9]*\.?[0-9]+(?:e[-+][0-9]+)?)")
|
||||
if top1 is None:
|
||||
top1 = num(r"top[-_ ]?1[^0-9]*([0-9]*\.?[0-9]+)")
|
||||
if kl is None or top1 is None:
|
||||
print("KLD GATE PARSE FAILED: mean_kl=%s top1=%s — inspect %s / %s" % (kl, top1, out, log))
|
||||
sys.exit(2)
|
||||
print("KLD GATE: mean KL = %.4f (max 0.03) | top-1 = %.2f%% (min 95.0)" % (kl, top1))
|
||||
ok = (kl <= 0.03) and (top1 >= 95.0)
|
||||
print("KLD GATE " + ("PASS" if ok else "FAIL"))
|
||||
sys.exit(0 if ok else 1)
|
||||
PY
|
||||
gate=$?
|
||||
echo "kld-gate complete rc=$gate; log: $LOG report: $KLD_OUT"
|
||||
exit "$gate"
|
||||
Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/bin/bash
|
||||
# kld-then-calib.sh — RUNS ON DANKSERVER. Campaign chain: wait for the KLD
|
||||
# record to finish, verify it, then boot the 1.2.0 calibration server and
|
||||
# push the 10M-token corpus. One GPU, so the phases serialize; prod is
|
||||
# restored by each run script's trap.
|
||||
set -u
|
||||
KLOG=/home/dank/logs/swift15-kld-driver.log
|
||||
CLOG=/home/dank/logs/swift15-calib-driver.log
|
||||
S=/home/dank/serverAppConfigs-swift15/scripts
|
||||
# Phase 1: KLD record (SKIP_KLD=1 when the record is already complete —
|
||||
# kld.json's rows count is the receipt; a mid-run script swap can kill the
|
||||
# wrapper with rc=127 after the engine exited clean).
|
||||
if [ "${SKIP_KLD:-0}" != 1 ]; then
|
||||
while ! grep -q "exited rc=" "$KLOG" 2>/dev/null; do sleep 60; done
|
||||
RC=$(grep -o "exited rc=[0-9]*" "$KLOG" | tail -1)
|
||||
if [ "$RC" != "exited rc=0" ]; then echo "KLD FAILED ($RC); calib NOT started"; exit 1; fi
|
||||
fi
|
||||
N=$(find /home/dank/corpora/kld-ref -type f | wc -l)
|
||||
SZ=$(du -sm /home/dank/corpora/kld-ref | cut -f1)
|
||||
echo "KLD OK files=$N size_mib=$SZ"
|
||||
[ "$N" -ge 5 ] && [ "$SZ" -ge 1000 ] || { echo "kld-ref looks thin; calib NOT started"; exit 1; }
|
||||
grep -q '"rows": 50235' /home/dank/corpora/kld-ref/kld.json || {
|
||||
echo "kld.json rows != 50235; calib NOT started"; exit 1; }
|
||||
|
||||
# Phase 2: calibration serve (1.2.0) + corpus push.
|
||||
nohup bash "$S/run-swift15-bf16.sh" calib > "$CLOG" 2>&1 &
|
||||
CP=$!
|
||||
for _ in $(seq 1 160); do
|
||||
sleep 15
|
||||
curl -sf localhost:7770/health >/dev/null 2>&1 && break
|
||||
done
|
||||
if ! curl -sf localhost:7770/health >/dev/null 2>&1; then
|
||||
echo "CALIB BOOT FAILED; see /home/dank/logs/swift15_bf16.log"; kill "$CP" 2>/dev/null; exit 1
|
||||
fi
|
||||
echo "CALIB UP, pushing corpus"
|
||||
python3 "$S/calib-push.py" --port 7770 --parallel 16 --container swift15ref --max-rounds 6
|
||||
PUSH=$?
|
||||
kill "$CP" 2>/dev/null
|
||||
sleep 10
|
||||
echo "CALIB_CHAIN_DONE push=$PUSH grams=$(find /home/dank/corpora/calib -name 'gram.r*.bin' 2>/dev/null | wc -l)"
|
||||
Executable
+317
@@ -0,0 +1,317 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the two Swift1.5 campaign corpora (calibration + KLD).
|
||||
|
||||
calib.jsonl — 10M tokens of agent-shaped RAW text (45% code, 30% prose,
|
||||
25% tool-call JSON), one {"prompt": text} per line, pushed to
|
||||
/v1/completions with max_tokens=1 during the calibration run (the Hessian
|
||||
tap is of the prompt tokens; docs/MOE-W4.md step 2). Code and prose mix
|
||||
the repo checkout with HF dataset parquets (Frank's call 2026-10-06:
|
||||
local material is only ~1.6M unique tokens; MOE-W4 wants 10M):
|
||||
nampdn-ai/tiny-codes (code) and Salesforce/wikitext wikitext-103-raw
|
||||
(prose). Tool calls are synthesized from commands harvested from the
|
||||
repo's own scripts plus a fleet-ops set, with argument-shaped variants.
|
||||
|
||||
kld.jsonl — 1.8M tokens, one {"prompt", "score_from", "source"} per
|
||||
line; each prompt is a multi-turn conversation rendered through the
|
||||
Froggeric v22.5 template with a >= --kld-min-prompt (8192) token target,
|
||||
and the last 256 tokens (the tail of the assistant answer) are the
|
||||
scored span: score_from = n - 256. At 8k-token prompts the 1.8M corpus
|
||||
is ~220 prompts = ~56k scored positions ~= 25 GB of record, matching
|
||||
the published Flash-Next KLD scale (83k positions, MOE-W4.md). The KLD
|
||||
reference run and every candidate run consume the SAME file (TOOLS.md
|
||||
--kld-record / --kld-ref). The KLD corpus is LOCAL material only, so the
|
||||
file-level disjointness rule (fnv1a64(path) % 5 == 0 -> KLD pool) is
|
||||
untouched by the HF additions.
|
||||
|
||||
Deps: tokenizers, jinja2, pyarrow (venv /home/dank/venvs/radcorpus).
|
||||
Usage: make-calib-corpus.py --repo ROOT --tokenizer-json PATH
|
||||
--template PATH --out-dir DIR [--hf-code-parquet F,F]
|
||||
[--hf-prose-parquet F,F] [--calib-tokens 10000000]
|
||||
[--kld-tokens 1800000]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
from tokenizers import Tokenizer
|
||||
|
||||
CODE_EXT = (".sh", ".py", ".jinja", ".recipe", ".env", ".example",
|
||||
".yaml", ".yml", ".bats", ".service", ".container", ".toml")
|
||||
JSONL_EXTS = (".jsonl", ".json")
|
||||
DOC_DIR = "serverAppConfigs-docs"
|
||||
|
||||
FLEET_CMDS = [
|
||||
"systemctl status frank-archon --no-pager",
|
||||
"journalctl -u frank-archon -n 100 --no-pager",
|
||||
"podman ps --format '{{.Names}} {{.Status}}'",
|
||||
"podman logs --tail 50 frank-archon",
|
||||
"curl -s http://192.168.0.235:7770/health",
|
||||
"curl -s http://192.168.0.235:7770/metrics | grep radiance_",
|
||||
"df -h /home/dank | tail -1",
|
||||
"free -g",
|
||||
"sudo systemctl restart frank-archon",
|
||||
"cd /home/dank/serverAppConfigs && git pull --ff-only",
|
||||
"./scripts/deploy.sh dankserver --service frank-archon",
|
||||
"./tests/run.sh 2>&1 | tail -5",
|
||||
"jq . /home/dank/models/rad/index.json | head -40",
|
||||
"python3 -c \"import json; print(json.load(open('config.json'))['num_hidden_layers'])\"",
|
||||
"ip neigh show 192.168.0.235",
|
||||
"amdgpu_top --device-info 2>/dev/null | head -20",
|
||||
"grep -E 'MemAvailable' /proc/meminfo",
|
||||
"sha256sum model-00001-of-00131.safetensors",
|
||||
"git log --oneline -10",
|
||||
"awk '{s+=$1} END {print s}' shard_sizes.txt",
|
||||
]
|
||||
TOOL_NAMES = ["bash", "read", "grep", "web_search", "edit", "write"]
|
||||
WORKDIRS = ["/home/dank", "/home/dank/serverAppConfigs", "/home/dank/models",
|
||||
"/etc/systemd/system", "/var/log"]
|
||||
DESCS = ["check service state", "tail the engine ledger", "list containers",
|
||||
"query health", "read metrics", "check disk", "check memory",
|
||||
"restart the endpoint", "sync the checkout", "run deployer",
|
||||
"run the test seams", "inspect the index", "probe the peer"]
|
||||
|
||||
|
||||
def fnv(s: str) -> int:
|
||||
h = 0xcbf29ce484222325
|
||||
for b in s.encode():
|
||||
h = ((h ^ b) * 0x100000001b3) & 0xFFFFFFFFFFFFFFFF
|
||||
return h
|
||||
|
||||
|
||||
def harvest_cmds(repo: str, rng: random.Random) -> list[str]:
|
||||
cmds = list(FLEET_CMDS)
|
||||
pat = re.compile(r"^\s*(?:sudo\s+)?([a-z][\w./-]*(?:\s+\S+){1,6})\s*$")
|
||||
for root, dirs, files in os.walk(repo):
|
||||
dirs[:] = [d for d in dirs if not d.startswith(".")]
|
||||
for f in files:
|
||||
if not f.endswith(".sh"):
|
||||
continue
|
||||
try:
|
||||
for line in open(os.path.join(root, f), encoding="utf-8",
|
||||
errors="replace"):
|
||||
m = pat.match(line.rstrip())
|
||||
if m and len(m.group(1)) > 12:
|
||||
cmds.append("ssh dank@192.168.0.235 '"
|
||||
+ m.group(1)[:90] + "'")
|
||||
except OSError:
|
||||
pass
|
||||
rng.shuffle(cmds)
|
||||
return cmds[:4000]
|
||||
|
||||
|
||||
def tool_call(rng: random.Random, cmds: list[str]) -> str:
|
||||
"""One agent turn: 2-4 tool calls with argument-shaped JSON."""
|
||||
parts = []
|
||||
for _ in range(rng.randint(2, 4)):
|
||||
args: dict = {"command": rng.choice(cmds)}
|
||||
if rng.random() < 0.4:
|
||||
args["cwd"] = rng.choice(WORKDIRS)
|
||||
if rng.random() < 0.3:
|
||||
args["timeout"] = rng.choice([10, 30, 60, 300, 600])
|
||||
if rng.random() < 0.5:
|
||||
args["description"] = rng.choice(DESCS)
|
||||
parts.append('<tool_call>{"name": "%s", "arguments": %s}<|im_end|>'
|
||||
% (rng.choice(TOOL_NAMES),
|
||||
json.dumps(args, ensure_ascii=False)))
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def load_parquet_texts(path: str, kind: str, rng: random.Random,
|
||||
cap: int) -> list[str]:
|
||||
import pyarrow.parquet as pq
|
||||
d = pq.read_table(path).to_pydict()
|
||||
n = len(next(iter(d.values())))
|
||||
idx = range(n) if n <= cap else rng.sample(range(n), cap)
|
||||
out = []
|
||||
for i in idx:
|
||||
if kind == "code":
|
||||
if "content" in d:
|
||||
frag = d["content"][i]
|
||||
else:
|
||||
frag = ("// %s\n### Instruction:\n%s\n### Response:\n%s"
|
||||
% (d.get("file_name", [""])[i], d["prompt"][i],
|
||||
d["code"][i] or d["response"][i]))
|
||||
else:
|
||||
frag = d["text"][i]
|
||||
if frag and frag.strip():
|
||||
out.append(frag[:8000])
|
||||
rng.shuffle(out)
|
||||
return out[:cap]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--repo", required=True)
|
||||
ap.add_argument("--tokenizer-json", required=True)
|
||||
ap.add_argument("--template", required=True)
|
||||
ap.add_argument("--out-dir", required=True)
|
||||
ap.add_argument("--calib-tokens", type=int, default=10_000_000)
|
||||
ap.add_argument("--kld-tokens", type=int, default=1_800_000)
|
||||
ap.add_argument("--chunk-tokens", type=int, default=8192)
|
||||
ap.add_argument("--score-tail", type=int, default=256)
|
||||
ap.add_argument("--kld-min-prompt", type=int, default=8192)
|
||||
ap.add_argument("--hf-code-parquet", default="")
|
||||
ap.add_argument("--hf-prose-parquet", default="")
|
||||
ap.add_argument("--seed", type=int, default=20261006)
|
||||
a = ap.parse_args()
|
||||
|
||||
rng = random.Random(a.seed)
|
||||
tokenizer = Tokenizer.from_file(a.tokenizer_json)
|
||||
env = Environment(loader=FileSystemLoader(os.path.dirname(a.template)),
|
||||
trim_blocks=True, lstrip_blocks=True)
|
||||
env.filters["tojson"] = lambda v: json.dumps(v, ensure_ascii=False)
|
||||
env.filters["raise_exception"] = lambda m: (_ for _ in ()).throw(
|
||||
ValueError(m))
|
||||
tname = os.path.basename(a.template)
|
||||
|
||||
code, prose, jsonl = [], [], []
|
||||
for root, dirs, files in os.walk(a.repo):
|
||||
dirs[:] = [d for d in dirs if not d.startswith(".")]
|
||||
for f in sorted(files):
|
||||
p = os.path.relpath(os.path.join(root, f), a.repo)
|
||||
if DOC_DIR in p and f.endswith(".md"):
|
||||
prose.append(p)
|
||||
elif p.startswith(DOC_DIR):
|
||||
pass
|
||||
elif f.endswith(CODE_EXT):
|
||||
code.append(p)
|
||||
elif f.endswith(JSONL_EXTS) and f != "package-lock.json":
|
||||
jsonl.append(p)
|
||||
# file-level split: bucket 0 -> KLD pool, 1..4 -> calibration pool
|
||||
pools: dict[str, tuple[list, list]] = {}
|
||||
for cls, lst in (("code", code), ("prose", prose), ("tools", jsonl)):
|
||||
pools[cls] = ([p for p in lst if fnv(p) % 5 == 0],
|
||||
[p for p in lst if fnv(p) % 5 != 0])
|
||||
|
||||
def text_of(p: str) -> str:
|
||||
path = os.path.join(a.repo, p)
|
||||
raw = open(path, encoding="utf-8", errors="replace").read()
|
||||
if p.endswith(JSONL_EXTS):
|
||||
lines = [json.loads(l)["content"] for l in raw.splitlines()
|
||||
if l.strip() and "content" in l]
|
||||
raw = "\n".join(lines)
|
||||
toks = tokenizer.encode(raw, add_special_tokens=False).ids
|
||||
return tokenizer.decode(toks[:2400], skip_special_tokens=True)
|
||||
|
||||
cmds = harvest_cmds(a.repo, rng)
|
||||
# fragment pools per class, corpus-split first
|
||||
frags: dict[str, dict[str, list[str]]] = {}
|
||||
for cls in pools:
|
||||
kld_files, cal_files = pools[cls]
|
||||
fr = {}
|
||||
src = {"kld": kld_files, "cal": cal_files}
|
||||
if cls == "tools":
|
||||
fr["kld"] = [tool_call(rng, cmds) for _ in range(600)]
|
||||
fr["cal"] = [tool_call(rng, cmds) for _ in range(40000)]
|
||||
else:
|
||||
fr["kld"] = [text_of(p) for p in src["kld"]]
|
||||
fr["cal"] = [text_of(p) for p in src["cal"]]
|
||||
frags[cls] = fr
|
||||
# per-class fragment caps: prose rows are short (~66 tok) so need many;
|
||||
# codeparrot files are long (~2.6k tok) so a small cap suffices
|
||||
hfcap = {"code": 40000, "prose": 60000, "tools": 0}
|
||||
for cls, spec in (("code", a.hf_code_parquet),
|
||||
("prose", a.hf_prose_parquet)):
|
||||
for path in [p for p in spec.split(",") if p]:
|
||||
frags[cls]["cal"].extend(
|
||||
load_parquet_texts(path, cls, rng, cap=hfcap[cls]))
|
||||
for cls in frags:
|
||||
rng.shuffle(frags[cls]["cal"])
|
||||
|
||||
def toks(s: str) -> int:
|
||||
return len(tokenizer.encode(s, add_special_tokens=False).ids)
|
||||
|
||||
def emit_calib() -> tuple[int, list[dict]]:
|
||||
targets = {"code": .45, "prose": .30, "tools": .25}
|
||||
rows: list[dict] = []
|
||||
for cls, frac in targets.items():
|
||||
want = int(a.calib_tokens * frac)
|
||||
pool = frags[cls]["cal"]
|
||||
if not pool:
|
||||
continue
|
||||
got, i, buf, nbuf = 0, 0, [], 0
|
||||
while got < want:
|
||||
frag = pool[i % len(pool)]
|
||||
i += 1
|
||||
n = toks(frag)
|
||||
if nbuf + n > a.chunk_tokens and buf:
|
||||
rows.append({"prompt": "\n".join(buf), "cls": cls})
|
||||
got += nbuf
|
||||
buf, nbuf = [], 0
|
||||
if got >= want:
|
||||
break
|
||||
buf.append(frag)
|
||||
nbuf += n
|
||||
if i > len(pool) * 40:
|
||||
break # pool exhausted with repeats; stop
|
||||
if buf and got < want:
|
||||
rows.append({"prompt": "\n".join(buf), "cls": cls})
|
||||
got += nbuf
|
||||
print(f"calib {cls}: {got} tokens, {i} fragments")
|
||||
rng.shuffle(rows)
|
||||
return len(rows), [{k: v for k, v in r.items() if k == "prompt"}
|
||||
for r in rows]
|
||||
|
||||
def emit_kld() -> tuple[int, list[dict]]:
|
||||
rows, total = [], 0
|
||||
classes = ["code", "prose", "tools"]
|
||||
i = {c: 0 for c in classes}
|
||||
guard = 0
|
||||
while total < a.kld_tokens and guard < 5000:
|
||||
guard += 1
|
||||
msgs, srcs = [], []
|
||||
for t in range(rng.choice([2, 3, 4])):
|
||||
cls = classes[(len(rows) + t) % 3]
|
||||
pool = frags[cls]["kld"]
|
||||
if not pool:
|
||||
continue
|
||||
frag = pool[i[cls] % len(pool)]
|
||||
i[cls] += 2
|
||||
msgs.append({"role": "user",
|
||||
"content": "Summarize and explain this "
|
||||
f"engineering material:\n{frag}"})
|
||||
msgs.append({"role": "assistant",
|
||||
"content": frag[:4000]})
|
||||
srcs.append(cls)
|
||||
rendered = env.get_template(tname).render(
|
||||
messages=msgs, add_generation_prompt=False)
|
||||
n = toks(rendered)
|
||||
if n < a.kld_min_prompt:
|
||||
continue
|
||||
rows.append({"prompt": rendered,
|
||||
"score_from": n - a.score_tail,
|
||||
"source": "+".join(sorted(set(srcs)))})
|
||||
total += n
|
||||
return len(rows), rows
|
||||
|
||||
os.makedirs(a.out_dir, exist_ok=True)
|
||||
nc, rows = emit_calib()
|
||||
with open(os.path.join(a.out_dir, "calib.jsonl"), "w") as f:
|
||||
for r in rows:
|
||||
f.write(json.dumps(r) + "\n")
|
||||
nk, krows = emit_kld()
|
||||
with open(os.path.join(a.out_dir, "kld.jsonl"), "w") as f:
|
||||
for r in krows:
|
||||
f.write(json.dumps(r) + "\n")
|
||||
scored = sum(len(tokenizer.encode(r["prompt"], add_special_tokens=False).ids)
|
||||
- r["score_from"] for r in krows)
|
||||
manifest = {
|
||||
"seed": a.seed, "template": tname,
|
||||
"tokenizer": os.path.basename(a.tokenizer_json),
|
||||
"calib_prompts": nc, "kld_prompts": nk,
|
||||
"kld_scored_positions": scored,
|
||||
"kld_score_tail": a.score_tail,
|
||||
"disjoint_file_pools": "local KLD material only; HF code feeds calib",
|
||||
"calib_mix": {"code": .45, "prose": .30, "tools": .25},
|
||||
"hf_code": a.hf_code_parquet, "hf_prose": a.hf_prose_parquet,
|
||||
}
|
||||
with open(os.path.join(a.out_dir, "manifest.json"), "w") as f:
|
||||
json.dump(manifest, f, indent=1)
|
||||
print(json.dumps(manifest, indent=1))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# STATUS: WIP — first of three patches in the "compile the GLM trunk" changelist. The
|
||||
# set does NOT yet boot the engine (see 03 for the next blocker), so it is NOT part of
|
||||
# the deployed frank-primary shape. Nothing in scripts/ mounts these files.
|
||||
# the deployed frank-archon shape. Nothing in scripts/ mounts these files.
|
||||
#
|
||||
# WHY THIS IS THE RIGHT LEVER (measured 2026-09-30, not inferred):
|
||||
#
|
||||
|
||||
@@ -29,7 +29,8 @@
|
||||
# its base SupportsEagleBase (has_own_lm_head, has_own_embed_tokens) -- and
|
||||
# getattr_static()s each one off the instance, breaking on the first miss.
|
||||
# So BOTH wrappers need all three attributes.
|
||||
# * MRO law: INHERITING SupportsEagle3 as a base dies at class creation with
|
||||
# * Method-resolution order forbids the obvious inheritance: adding
|
||||
# SupportsEagle3 as a base dies at class creation with
|
||||
# "Cannot create a consistent method resolution order (MRO) for bases ..."
|
||||
# (GLM's wrappers already carry four/five Protocol-derived bases). The
|
||||
# attribute route avoids the MRO entirely. Also proven by simulation:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# gpu_worker-kv-cap.patch — fork-local fix for the expert-offload KV cap.
|
||||
#
|
||||
# STATUS: NOT part of the deployed frank-primary shape. It exists because the
|
||||
# STATUS: NOT part of the deployed frank-archon shape. It exists because the
|
||||
# fork's plan under-models this model's KV, and it is the enabler for a future
|
||||
# MTP attempt (the production GLM shape runs without MTP and boots fine on the
|
||||
# plan's own reserve).
|
||||
|
||||
Executable
+202
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env python3
|
||||
"""probe-frank-archon-kv.py — verify the seqs x max-model-len KV contract on a
|
||||
frank-archon (radiance) service. Run it ON dankserver: it talks to
|
||||
127.0.0.1:7770 and builds its prompts from the repo checkout there.
|
||||
No scratch container, no mask: the service under test is the service.
|
||||
|
||||
ssh dankserver 'python3 /home/dank/serverAppConfigs/scripts/probe-frank-archon-kv.py'
|
||||
|
||||
Four probes, in increasing cost:
|
||||
A decode at depth — one ~51k-token chat completion, 512 tokens, greedy.
|
||||
Reports the engine's own `timings` (prefill, decode, draft acceptance).
|
||||
Measurement discipline: generate at least 128 tokens (a short run
|
||||
measures first-token noise, not the server), greedy, real text.
|
||||
B the capacity contract — as many CONCURRENT chats of ~197k prompt tokens as
|
||||
the deployed `FRANK_ARCHON_MAX_SEQS` (read from the repo env file on this
|
||||
host), each carrying a unique marker at its head. All must answer their own
|
||||
marker. This is what the endpoint owes: N full-height conversations
|
||||
resident at once, exact retrieval in every one of them.
|
||||
C prefix-cache restore — probe A's cold prompt again, so the disk tier has
|
||||
to give the blocks back (cache_n > 0, TTFT collapses).
|
||||
D concurrency decode — `seqs` CONCURRENT 512-token generations at ~25 % of
|
||||
the window. This is the cost a slot count imposes on DECODE, which no
|
||||
prefill number shows: the fused-step schedule is unfair, so every stream is
|
||||
printed plus the aggregate. Measured 2026-10-13 at 3 x 200,000 fp8:
|
||||
single-stream 99.0 t/s (43.5k) and 93.3 (197k) — decode is nearly
|
||||
window-free — against 3-stream 113.9/43.3/10.9 t/s = 18.4 t/s aggregate.
|
||||
Concurrency, not the slab, is what decode pays for slots. The 2026-10-13
|
||||
shape hunt missed this for a whole day because leg B generates 24 tokens.
|
||||
PROBE LAW (learned the hard way, twice): radiance routes the thinking block to
|
||||
`reasoning_content`, so a chat request whose budget is spent inside
|
||||
`<think>` returns EMPTY `content` with `finish_reason: stop`. A probe that
|
||||
reads `content` alone scores 0/2 on a healthy server. Every request here sets
|
||||
chat_template_kwargs.enable_thinking=false and reads both fields.
|
||||
|
||||
A/B method for a shape change (no scratch container): write a variant env file
|
||||
on the host, point a transient systemd drop-in at it
|
||||
(`Environment=FRANK_ARCHON_ENV_FILE=...` — drop-ins are read AFTER the unit's
|
||||
own Environment= lines, so they win), restart, probe, delete the drop-in,
|
||||
daemon-reload, restart. `systemctl mask` is not available: the unit file in
|
||||
/etc/systemd/system is a relink symlink into the checkout.
|
||||
"""
|
||||
import json
|
||||
import glob
|
||||
import time
|
||||
import urllib.request
|
||||
import concurrent.futures
|
||||
|
||||
BASE = "http://127.0.0.1:7770"
|
||||
MODEL = "frank-archon"
|
||||
REPO = "/home/dank/serverAppConfigs"
|
||||
GLOBS = [f"{REPO}/scripts/*.sh", f"{REPO}/serverAppConfigs-docs/**/*.md"]
|
||||
|
||||
|
||||
def post(payload, timeout=1800):
|
||||
req = urllib.request.Request(
|
||||
BASE + "/v1/chat/completions",
|
||||
data=json.dumps(payload).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
t0 = time.time()
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
body = json.loads(r.read().decode())
|
||||
hdr = dict(r.headers.items())
|
||||
return body, time.time() - t0, hdr
|
||||
|
||||
|
||||
def get(path, timeout=60):
|
||||
with urllib.request.urlopen(BASE + path, timeout=timeout) as r:
|
||||
return json.loads(r.read().decode())
|
||||
|
||||
|
||||
def corpus(min_chars):
|
||||
files = []
|
||||
for g in GLOBS:
|
||||
files.extend(sorted(glob.glob(g, recursive=True)))
|
||||
out, total = [], 0
|
||||
for f in files:
|
||||
try:
|
||||
t = open(f, encoding="utf-8", errors="replace").read()
|
||||
except OSError:
|
||||
continue
|
||||
out.append(t)
|
||||
total += len(t)
|
||||
if total > min_chars:
|
||||
break
|
||||
return "".join(text for text in out)
|
||||
|
||||
|
||||
def prompt(text, chars, salt, fact):
|
||||
"""Real repo text, salted so the prefix cache cannot answer, marker planted
|
||||
near the head, question at the tail."""
|
||||
head = f"\n[SALT {salt}] confidential marker: {fact}\n"
|
||||
body = text + text
|
||||
p = (head + body * max(1, chars // len(body)))[:chars]
|
||||
return p + "\n\nQuestion: what is the confidential marker in the " \
|
||||
"SALT block near the top of this document? Answer with the marker " \
|
||||
"alone, no other words.\n"
|
||||
|
||||
|
||||
def chat(text, chars, salt, fact, max_tokens):
|
||||
return {"model": MODEL,
|
||||
"messages": [{"role": "user",
|
||||
"content": prompt(text, chars, salt, fact)}],
|
||||
"max_tokens": max_tokens, "temperature": 0.0,
|
||||
"chat_template_kwargs": {"enable_thinking": False}}
|
||||
|
||||
|
||||
def show(tag, j, wall, expect=None):
|
||||
tim, u = j.get("timings") or {}, j.get("usage", {})
|
||||
ch = (j.get("choices") or [{}])[0]
|
||||
msg = ch.get("message") or {}
|
||||
txt = (msg.get("content") or msg.get("reasoning_content") or "").strip()
|
||||
acc = (tim["draft_n_accepted"] / tim["draft_n"]) if tim.get("draft_n") else None
|
||||
line = (f"[{tag}] prompt={u.get('prompt_tokens')} gen={u.get('completion_tokens')} "
|
||||
f"wall={wall:.1f}s prefill={tim.get('prompt_per_second', 0):.0f} t/s "
|
||||
f"decode={tim.get('predicted_per_second', 0):.1f} t/s "
|
||||
f"draft={tim.get('draft_n')}/{tim.get('draft_n_accepted')} "
|
||||
f"accept={acc if acc is None else round(acc, 4)} "
|
||||
f"cache_n={tim.get('cache_n')} finish={ch.get('finish_reason')}")
|
||||
if expect:
|
||||
hit = expect.split("-")[0] in txt.lower()
|
||||
line += f" expected={expect} answered={txt[:80]!r} HIT={hit}"
|
||||
print(line)
|
||||
|
||||
|
||||
def deployed_shape():
|
||||
"""The contract is the deployed env, not a constant: slot count and window
|
||||
come from the checkout on this host, so the probe cannot drift from the
|
||||
service it is testing (the 2026-10-13 shape is 4 x 200,000; before that
|
||||
2 x 200,000, and the fp8 era 3 x 200,000)."""
|
||||
shape = {}
|
||||
with open(f"{REPO}/scripts/frank-archon.env") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if line.startswith("FRANK_ARCHON_") and "=" in line:
|
||||
key, _, val = line.partition("=")
|
||||
shape[key] = val.strip().strip('"')
|
||||
return (int(shape["FRANK_ARCHON_MAX_SEQS"]),
|
||||
int(shape["FRANK_ARCHON_MAX_MODEL_LEN"]),
|
||||
shape["FRANK_ARCHON_KV_DTYPE"])
|
||||
|
||||
|
||||
def main():
|
||||
seqs, maxlen, dtype = deployed_shape()
|
||||
print(f"shape: {seqs} x {maxlen:,} tokens, kv {dtype}")
|
||||
print("health:", json.dumps(get("/health")))
|
||||
print("models:", [m.get("id") for m in get("/v1/models").get("data", [])])
|
||||
deep = corpus(160_000)
|
||||
wide = corpus(1_100_000)
|
||||
|
||||
# 3.28 chars/token is the measured ratio of this corpus through the
|
||||
# froggeric template (646,000 chars -> 196,919 tokens at the 2026-10-11
|
||||
# deploy); 98.5 % of the window leaves the response room under the ceiling.
|
||||
chars = int(0.985 * maxlen * 3.28)
|
||||
# Each marker's FIRST hyphen-separated word is unique, because show()
|
||||
# scores the hit on that word — a shared prefix would score every slot HIT
|
||||
# on any answer that says the word.
|
||||
words = ["cobalt", "quartz", "basalt", "ember", "willow", "cobweb", "flint"]
|
||||
facts = {f"B{i+1}": f"{words[i % len(words)]}-flag-{(i + 1) * 137}"
|
||||
for i in range(seqs)}
|
||||
bodies = {s: chat(wide, chars, s, f, 24) for s, f in facts.items()}
|
||||
print(f"[B] {len(bodies['B1']['messages'][0]['content']):,} chars per "
|
||||
f"request, x{seqs} concurrent")
|
||||
|
||||
# A + C first (cheap), B last (the expensive set).
|
||||
for tag, salt in (("A-cold", "A1-9137"), ):
|
||||
j, wall, _ = post(chat(deep, 140_000, salt, "zephyr-lantern-7", 512))
|
||||
show(tag, j, wall, "zephyr-lantern-7")
|
||||
t0 = time.time()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=seqs) as ex:
|
||||
futs = {s: ex.submit(post, b) for s, b in bodies.items()}
|
||||
for s, fut in futs.items():
|
||||
try:
|
||||
j, wall, _ = fut.result()
|
||||
show(f"B-{s}", j, wall, facts[s])
|
||||
except Exception as e: # a preemption/timeout lands here
|
||||
print(f"[B-{s}] FAILED after {time.time() - t0:.1f}s: {e!r}")
|
||||
print(f"[B] total {time.time() - t0:.1f}s")
|
||||
# D — concurrency decode: seqs streams, 512 tokens each, ~25 % depth.
|
||||
dchars = int(0.25 * maxlen * 3.28)
|
||||
dbodies = [chat(deep, dchars, f"D{i + 1}-9137", f"flint-marker-{i}", 512)
|
||||
for i in range(seqs)]
|
||||
print(f"[D] {seqs} concurrent streams x 512 tokens, "
|
||||
f"{dchars:,} chars each")
|
||||
t0 = time.time()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=seqs) as ex:
|
||||
futs = [ex.submit(post, b) for b in dbodies]
|
||||
for i, fut in enumerate(futs, 1):
|
||||
try:
|
||||
j, wall, _ = fut.result()
|
||||
show(f"D-s{i}", j, wall)
|
||||
except Exception as e: # a preemption/timeout lands here
|
||||
print(f"[D-s{i}] FAILED after {time.time() - t0:.1f}s: {e!r}")
|
||||
tot = time.time() - t0
|
||||
print(f"[D] {seqs} x 512 tokens in {tot:.1f}s = "
|
||||
f"{seqs * 512 / tot:.1f} t/s aggregate decode")
|
||||
j, wall, _ = post(chat(deep, 140_000, "A1-9137", "zephyr-lantern-7", 512))
|
||||
show("C-prefix-hit", j, wall, "zephyr-lantern-7")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,460 @@
|
||||
{%- set template_version = "qwen3.8-froggeric-v22.5" %}
|
||||
{%- set _tool_format = tool_call_format if tool_call_format is defined else 'xml' %}
|
||||
{%- set image_count = namespace(value=0) %}
|
||||
{%- set video_count = namespace(value=0) %}
|
||||
{%- set add_vision_id = add_vision_id if add_vision_id is defined else false %}
|
||||
{%- set enable_thinking = enable_thinking if enable_thinking is defined else true %}
|
||||
{%- set auto_disable_thinking_with_tools = auto_disable_thinking_with_tools if auto_disable_thinking_with_tools is defined else false %}
|
||||
{%- if preserve_reasoning is defined and preserve_reasoning is not none %}
|
||||
{%- set _preserve_thinking = preserve_reasoning %}
|
||||
{%- elif preserve_thinking is defined and preserve_thinking is not none %}
|
||||
{%- set _preserve_thinking = preserve_thinking %}
|
||||
{%- else %}
|
||||
{%- set _preserve_thinking = true %}
|
||||
{%- endif %}
|
||||
{%- set max_tool_arg_chars = max_tool_arg_chars if max_tool_arg_chars is defined else 0 %}
|
||||
{%- set max_tool_response_chars = max_tool_response_chars if max_tool_response_chars is defined else 0 %}
|
||||
{%- set _default_reasoning_effort = 'medium' %}
|
||||
{%- set _has_tools = (tools is defined and tools and tools is iterable and tools is not mapping) %}
|
||||
{%- set _effort_raw = (reasoning_effort | string | lower) if reasoning_effort is defined and reasoning_effort is not none else _default_reasoning_effort %}
|
||||
{%- set _initial_thinking = enable_thinking %}
|
||||
{%- if _effort_raw in ('none', 'off') %}
|
||||
{%- set _initial_thinking = false %}
|
||||
{%- set _initial_effort = 'medium' %}
|
||||
{%- elif _effort_raw in ('minimal', 'low') %}
|
||||
{%- set _initial_effort = 'low' %}
|
||||
{%- elif _effort_raw in ('high', 'xhigh', 'max', 'ultracode', 'extreme') %}
|
||||
{%- set _initial_effort = 'xhigh' %}
|
||||
{%- else %}
|
||||
{%- set _initial_effort = 'medium' %}
|
||||
{%- endif %}
|
||||
{%- set ns_state = namespace(thinking=_initial_thinking, effort=_initial_effort) %}
|
||||
{%- if auto_disable_thinking_with_tools and _has_tools %}
|
||||
{%- set ns_state.thinking = false %}
|
||||
{%- endif %}
|
||||
{%- for msg in messages %}
|
||||
{%- if msg.role == 'system' or msg.role == 'developer' or msg.role == 'user' %}
|
||||
{%- if msg.content is string %}
|
||||
{%- if '<|think_off|>' in msg.content %}
|
||||
{%- set ns_state.thinking = false %}
|
||||
{%- elif '<|think_on|>' in msg.content %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- elif '<|think_xhigh|>' in msg.content or '<|think_high|>' in msg.content or '<|think_ultracode|>' in msg.content or '<|think_extreme|>' in msg.content or '<|think_max|>' in msg.content %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'xhigh' %}
|
||||
{%- elif '<|think_low|>' in msg.content or '<|think_minimal|>' in msg.content %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'low' %}
|
||||
{%- elif '<|think_medium|>' in msg.content %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'medium' %}
|
||||
{%- endif %}
|
||||
{%- elif msg.content is iterable and msg.content is not mapping %}
|
||||
{%- for item in msg.content %}
|
||||
{%- if item is string %}
|
||||
{%- set _item_text = item %}
|
||||
{%- elif item is mapping and 'text' in item and item.text is string %}
|
||||
{%- set _item_text = item.text %}
|
||||
{%- else %}
|
||||
{%- set _item_text = '' %}
|
||||
{%- endif %}
|
||||
{%- if _item_text %}
|
||||
{%- if '<|think_off|>' in _item_text %}
|
||||
{%- set ns_state.thinking = false %}
|
||||
{%- elif '<|think_on|>' in _item_text %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- elif '<|think_xhigh|>' in _item_text or '<|think_high|>' in _item_text or '<|think_ultracode|>' in _item_text or '<|think_extreme|>' in _item_text or '<|think_max|>' in _item_text %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'xhigh' %}
|
||||
{%- elif '<|think_low|>' in _item_text or '<|think_minimal|>' in _item_text %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'low' %}
|
||||
{%- elif '<|think_medium|>' in _item_text %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'medium' %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- set reasoning_instructions = '' %}
|
||||
{%- if ns_state.thinking %}
|
||||
{%- if ns_state.effort == 'xhigh' %}
|
||||
{%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}
|
||||
{%- elif ns_state.effort == 'low' %}
|
||||
{%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- macro render_content(content, do_vision_count, is_system_content=false) %}
|
||||
{%- if content is string %}
|
||||
{{- content }}
|
||||
{%- elif content is iterable and content is not mapping %}
|
||||
{%- for item in content %}
|
||||
{%- if item is mapping %}
|
||||
{%- if item.type == 'image' or 'image' in item or 'image_url' in item %}
|
||||
{%- if is_system_content %}
|
||||
{{- raise_exception('System message cannot contain images.') }}
|
||||
{%- endif %}
|
||||
{%- if do_vision_count %}
|
||||
{%- set image_count.value = image_count.value + 1 %}
|
||||
{%- endif %}
|
||||
{%- if add_vision_id %}
|
||||
{{- 'Picture ' ~ image_count.value ~ ': ' }}
|
||||
{%- endif %}
|
||||
{{- '<|vision_start|><|image_pad|><|vision_end|>' }}
|
||||
{%- elif item.type == 'video' or 'video' in item or 'video_url' in item %}
|
||||
{%- if is_system_content %}
|
||||
{{- raise_exception('System message cannot contain videos.') }}
|
||||
{%- endif %}
|
||||
{%- if do_vision_count %}
|
||||
{%- set video_count.value = video_count.value + 1 %}
|
||||
{%- endif %}
|
||||
{%- if add_vision_id %}
|
||||
{{- 'Video ' ~ video_count.value ~ ': ' }}
|
||||
{%- endif %}
|
||||
{{- '<|vision_start|><|video_pad|><|vision_end|>' }}
|
||||
{%- elif 'text' in item %}
|
||||
{{- item.text }}
|
||||
{%- else %}
|
||||
{{- raise_exception('Unexpected item type in content.') }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- item | string }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- elif content is none or content is undefined %}
|
||||
{{- '' }}
|
||||
{%- else %}
|
||||
{{- raise_exception('Unexpected content type.') }}
|
||||
{%- endif %}
|
||||
{%- endmacro %}
|
||||
{%- if not messages %}
|
||||
{{- raise_exception('No messages provided.') }}
|
||||
{%- endif %}
|
||||
{%- set head = namespace(count=0, seen_non_system=false) %}
|
||||
{%- for message in messages %}
|
||||
{%- set _is_sys = (message.role == 'system' or message.role == 'developer') %}
|
||||
{%- if _is_sys and not head.seen_non_system %}
|
||||
{%- set head.count = head.count + 1 %}
|
||||
{%- else %}
|
||||
{%- set head.seen_non_system = true %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- set sys_state = namespace(content='') %}
|
||||
{%- for message in messages[:head.count] %}
|
||||
{%- set _part = render_content(message.content, false, true) | trim %}
|
||||
{%- if '<|think_off|>' in _part %}{%- set _part = _part.split('<|think_off|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_on|>' in _part %}{%- set _part = _part.split('<|think_on|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_xhigh|>' in _part %}{%- set _part = _part.split('<|think_xhigh|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_high|>' in _part %}{%- set _part = _part.split('<|think_high|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_ultracode|>' in _part %}{%- set _part = _part.split('<|think_ultracode|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_extreme|>' in _part %}{%- set _part = _part.split('<|think_extreme|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_max|>' in _part %}{%- set _part = _part.split('<|think_max|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_medium|>' in _part %}{%- set _part = _part.split('<|think_medium|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_low|>' in _part %}{%- set _part = _part.split('<|think_low|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_minimal|>' in _part %}{%- set _part = _part.split('<|think_minimal|>') | join('') | trim %}{%- endif %}
|
||||
{%- if _part %}
|
||||
{%- if sys_state.content %}
|
||||
{%- set sys_state.content = sys_state.content ~ '\n\n' ~ _part %}
|
||||
{%- else %}
|
||||
{%- set sys_state.content = _part %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- set _sc = sys_state.content %}
|
||||
{%- set _msgs = messages[head.count:] %}
|
||||
{%- if _has_tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if reasoning_instructions %}
|
||||
{{- reasoning_instructions + '\n\n' }}
|
||||
{%- endif %}
|
||||
{{- '# Tools\n\nYou have access to the following functions:\n\n<tools>' }}
|
||||
{%- for tool in tools %}
|
||||
{{- '\n' }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- '\n</tools>' }}
|
||||
{%- if _tool_format == 'json' %}
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n' }}
|
||||
{%- if ns_state.thinking %}
|
||||
{{- '<think>\nBrief explanation of tool call\n</think>\n' }}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n{"name": "example_function_name", "arguments": {"example_parameter_1": "value_1", "example_parameter_2": "This is the value for the second parameter"}}\n</tool_call>\n\n<IMPORTANT>\nReminder:\n' }}
|
||||
{%- if ns_state.thinking %}
|
||||
{{- '- You can use the <think></think> block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the <think></think> block.\n' }}
|
||||
{%- endif %}
|
||||
{{- '- Function calls MUST follow the specified format: a single JSON object with "name" and "arguments" keys inside <tool_call></tool_call> XML tags.\n' }}
|
||||
{%- if ns_state.thinking %}
|
||||
{{- '- If you choose to call a tool, you MUST output the <tool_call> block IMMEDIATELY after thinking, with NO conversational text before it.\n' }}
|
||||
{%- else %}
|
||||
{{- '- If you choose to call a tool, you MUST output the <tool_call> block IMMEDIATELY, with NO conversational text before it.\n' }}
|
||||
{%- endif %}
|
||||
{{- '- The <tool_call> tag MUST be at the very beginning of a new line, with NO spaces or indentation before it.\n- To call multiple functions, output a separate, completely closed <tool_call></tool_call> block for EACH function. Do NOT nest <tool_call> blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n</IMPORTANT>' }}
|
||||
{%- else %}
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n' }}
|
||||
{%- if ns_state.thinking %}
|
||||
{{- '<think>\nBrief explanation of tool call\n</think>\n' }}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n' }}
|
||||
{%- if ns_state.thinking %}
|
||||
{{- '- You can use the <think></think> block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the <think></think> block.\n' }}
|
||||
{%- endif %}
|
||||
{{- '- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags.\n' }}
|
||||
{%- if ns_state.thinking %}
|
||||
{{- '- If you choose to call a tool, you MUST output the <tool_call> block IMMEDIATELY after thinking, with NO conversational text before it.\n' }}
|
||||
{%- else %}
|
||||
{{- '- If you choose to call a tool, you MUST output the <tool_call> block IMMEDIATELY, with NO conversational text before it.\n' }}
|
||||
{%- endif %}
|
||||
{{- '- The <tool_call> and <function> tags MUST be at the very beginning of a new line, with NO spaces or indentation before them.\n- To call multiple functions, output a separate, completely closed <tool_call></tool_call> block for EACH function. Do NOT nest <tool_call> blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n</IMPORTANT>' }}
|
||||
{%- endif %}
|
||||
{%- if _sc %}
|
||||
{{- '\n\n' + _sc }}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- if _sc %}
|
||||
{{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '') + _sc + '<|im_end|>\n' }}
|
||||
{%- elif reasoning_instructions %}
|
||||
{{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set _last_idx = _msgs | length - 1 %}
|
||||
{%- set ns = namespace(multi_step_tool=true, last_query_index=_last_idx) %}
|
||||
{%- for message in _msgs[::-1] %}
|
||||
{%- set index = (_msgs | length - 1) - loop.index0 %}
|
||||
{%- if ns.multi_step_tool and message.role == 'user' %}
|
||||
{%- set _rc = render_content(message.content, false) | trim %}
|
||||
{%- if not (_rc.startswith('<tool_response>') and _rc.endswith('</tool_response>')) %}
|
||||
{%- set ns.multi_step_tool = false %}
|
||||
{%- set ns.last_query_index = index %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if ns.multi_step_tool %}
|
||||
{%- if _last_idx > 50 %}
|
||||
{%- set ns.last_query_index = _last_idx %}
|
||||
{%- else %}
|
||||
{%- set ns.last_query_index = 0 %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set ns2 = namespace(prev_role='', consecutive_failures=0) %}
|
||||
{%- for message in _msgs %}
|
||||
{%- set is_system = (message.role == "system" or message.role == "developer") %}
|
||||
{%- set content = render_content(message.content, true, is_system) | trim %}
|
||||
{%- if is_system or message.role == 'user' %}
|
||||
{%- if '<|think_off|>' in content %}{%- set content = content.split('<|think_off|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_on|>' in content %}{%- set content = content.split('<|think_on|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_xhigh|>' in content %}{%- set content = content.split('<|think_xhigh|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_high|>' in content %}{%- set content = content.split('<|think_high|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_ultracode|>' in content %}{%- set content = content.split('<|think_ultracode|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_extreme|>' in content %}{%- set content = content.split('<|think_extreme|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_max|>' in content %}{%- set content = content.split('<|think_max|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_medium|>' in content %}{%- set content = content.split('<|think_medium|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_low|>' in content %}{%- set content = content.split('<|think_low|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_minimal|>' in content %}{%- set content = content.split('<|think_minimal|>') | join('') | trim %}{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if is_system %}
|
||||
{{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
|
||||
{%- elif message.role == 'user' %}
|
||||
{%- set ns2.consecutive_failures = 0 %}
|
||||
{{- '<|im_start|>user\n' + content + '<|im_end|>\n' }}
|
||||
{%- elif message.role == 'assistant' %}
|
||||
{%- set reasoning_content = '' %}
|
||||
{%- set _explicit_reasoning = '' %}
|
||||
{%- if message.reasoning_content is defined and message.reasoning_content is not none %}
|
||||
{%- if message.reasoning_content is string %}
|
||||
{%- set _explicit_reasoning = message.reasoning_content %}
|
||||
{%- else %}
|
||||
{%- set _explicit_reasoning = message.reasoning_content | string %}
|
||||
{%- endif %}
|
||||
{%- elif message.thinking is defined and message.thinking is not none %}
|
||||
{%- if message.thinking is string %}
|
||||
{%- set _explicit_reasoning = message.thinking %}
|
||||
{%- else %}
|
||||
{%- set _explicit_reasoning = message.thinking | string %}
|
||||
{%- endif %}
|
||||
{%- elif message.reasoning is defined and message.reasoning is not none %}
|
||||
{%- if message.reasoning is string %}
|
||||
{%- set _explicit_reasoning = message.reasoning %}
|
||||
{%- else %}
|
||||
{%- set _explicit_reasoning = message.reasoning | string %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if _explicit_reasoning %}
|
||||
{%- set _lead_end = '' %}
|
||||
{%- if content.startswith('<think>') and '</think>' in content %}
|
||||
{%- set _lead_end = '</think>' %}
|
||||
{%- elif content.startswith('<thinking>') and '</thinking>' in content %}
|
||||
{%- set _lead_end = '</thinking>' %}
|
||||
{%- elif content.startswith('</think>') %}
|
||||
{%- set _lead_end = '</think>' %}
|
||||
{%- elif content.startswith('</thinking>') %}
|
||||
{%- set _lead_end = '</thinking>' %}
|
||||
{%- endif %}
|
||||
{%- if _lead_end %}
|
||||
{%- set content = content.split(_lead_end)[-1].lstrip('\n') %}
|
||||
{%- endif %}
|
||||
{%- set reasoning_content = _explicit_reasoning %}
|
||||
{%- else %}
|
||||
{%- set _think_end = '' %}
|
||||
{%- if content.startswith('</think>') %}
|
||||
{%- set _think_end = '</think>' %}
|
||||
{%- elif content.startswith('</thinking>') %}
|
||||
{%- set _think_end = '</thinking>' %}
|
||||
{%- elif '\n</think>' in content %}
|
||||
{%- set _think_end = '\n</think>' %}
|
||||
{%- elif '\n</thinking>' in content %}
|
||||
{%- set _think_end = '\n</thinking>' %}
|
||||
{%- elif '\n</ think>' in content %}
|
||||
{%- set _think_end = '\n</ think>' %}
|
||||
{%- elif '\n</think >' in content %}
|
||||
{%- set _think_end = '\n</think >' %}
|
||||
{%- elif content.startswith('<think>') and '</think>' in content %}
|
||||
{%- set _think_end = '</think>' %}
|
||||
{%- elif content.startswith('<thinking>') and '</thinking>' in content %}
|
||||
{%- set _think_end = '</thinking>' %}
|
||||
{%- endif %}
|
||||
{%- if _think_end %}
|
||||
{%- if 'thinking' in _think_end %}
|
||||
{%- set _think_start = '<thinking>' %}
|
||||
{%- else %}
|
||||
{%- set _think_start = '<think>' %}
|
||||
{%- endif %}
|
||||
{%- set reasoning_content = content.split(_think_end)[0].rstrip('\n') %}
|
||||
{%- if _think_start in reasoning_content %}
|
||||
{%- set reasoning_content = reasoning_content.split(_think_start)[-1].lstrip('\n') %}
|
||||
{%- endif %}
|
||||
{%- set content = content.split(_think_end)[-1].lstrip('\n') %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set reasoning_content = reasoning_content | trim %}
|
||||
{%- if (_preserve_thinking or loop.index0 > ns.last_query_index) %}
|
||||
{{- '<|im_start|>assistant\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n' + content }}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls is defined and message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined and tool_call.function is not none %}
|
||||
{%- set tc = tool_call.function %}
|
||||
{%- else %}
|
||||
{%- set tc = tool_call %}
|
||||
{%- endif %}
|
||||
{%- set tc_name = tc.name if (tc.name is defined and tc.name is not none) else '' %}
|
||||
{%- if _tool_format == 'json' %}
|
||||
{%- if loop.first %}
|
||||
{%- if content | trim %}
|
||||
{{- '\n\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '\n' }}
|
||||
{%- endif %}
|
||||
{%- set _args = '{}' %}
|
||||
{%- if tc.arguments is defined and tc.arguments is not none %}
|
||||
{%- if tc.arguments is mapping %}
|
||||
{%- set _args = tc.arguments | tojson %}
|
||||
{%- elif tc.arguments is string %}
|
||||
{%- if tc.arguments %}
|
||||
{%- set _args = tc.arguments %}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{%- set _args = tc.arguments | tojson %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n{"name": ' }}{{- tc_name | tojson }}{{- ', "arguments": ' }}{{- _args }}{{- '}\n</tool_call>' }}
|
||||
{%- else %}
|
||||
{%- if loop.first %}
|
||||
{%- if content | trim %}
|
||||
{{- '\n\n<tool_call>\n<function=' + tc_name + '>\n' }}
|
||||
{%- else %}
|
||||
{{- '<tool_call>\n<function=' + tc_name + '>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '\n<tool_call>\n<function=' + tc_name + '>\n' }}
|
||||
{%- endif %}
|
||||
{%- if tc.arguments is defined and tc.arguments is not none %}
|
||||
{%- if tc.arguments is mapping %}
|
||||
{%- for args_name, args_value in tc.arguments.items() %}
|
||||
{{- '<parameter=' + args_name + '>\n' }}
|
||||
{%- if args_value is string %}
|
||||
{%- set _av = args_value %}
|
||||
{%- else %}
|
||||
{%- set _av = args_value | tojson %}
|
||||
{%- endif %}
|
||||
{%- if max_tool_arg_chars > 0 and _av | length > max_tool_arg_chars %}
|
||||
{{- _av[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_av | length | string) ~ ' chars]' }}
|
||||
{%- else %}
|
||||
{{- _av }}
|
||||
{%- endif %}
|
||||
{{- '\n</parameter>\n' }}
|
||||
{%- endfor %}
|
||||
{%- else %}
|
||||
{%- if tc.arguments is string %}
|
||||
{%- set _raw_args = tc.arguments %}
|
||||
{%- else %}
|
||||
{%- set _raw_args = tc.arguments | tojson %}
|
||||
{%- endif %}
|
||||
{%- if _raw_args %}
|
||||
{%- if max_tool_arg_chars > 0 and _raw_args | length > max_tool_arg_chars %}
|
||||
{{- _raw_args[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_raw_args | length | string) ~ ' chars]' }}
|
||||
{%- else %}
|
||||
{{- _raw_args }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{{- '</function>\n</tool_call>' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == 'tool' %}
|
||||
{%- set _content_lower = content | lower %}
|
||||
{%- set _content_head = _content_lower[:120] %}
|
||||
{%- set _is_code_or_grep = ('throw new ' in _content_lower or 'throw error' in _content_lower or 'console.error' in _content_lower or 'logger.error' in _content_lower or 'logging.error' in _content_lower or 'import ' in _content_head or 'def ' in _content_head or 'function ' in _content_head) %}
|
||||
{%- set _exit_code_zero = ('exit code: 0' in _content_head or 'process exited with code 0' in _content_head) %}
|
||||
{%- set _error_field_ok = ('"error": null' in _content_head or '"error":null' in _content_head or '"error": false' in _content_head or '"error":false' in _content_head or '"error": ""' in _content_head or '"error":""' in _content_head) %}
|
||||
{%- set _strong_error = (('"error":' in _content_head and not _error_field_ok) or '"status": "error"' in _content_head or '"status":"error"' in _content_head or 'traceback (most recent call last):' in _content_head or 'command not found' in _content_head or 'invalid syntax' in _content_head or 'fatal:' in _content_head or (('exit code: ' in _content_head or 'process exited with code' in _content_head) and not _exit_code_zero) or _content_head.startswith('exception:') or _content_head.startswith('failed to ')) %}
|
||||
{%- set _weak_error = ('error:' in _content_head or 'err!' in _content_head) %}
|
||||
{%- set _weak_suppressed = ('$ ' in _content_head or 'took ' in _content_head or content | length >= 600) %}
|
||||
{%- if not _is_code_or_grep and (_strong_error or (_weak_error and not _weak_suppressed)) %}
|
||||
{%- set ns2.consecutive_failures = ns2.consecutive_failures + 1 %}
|
||||
{%- else %}
|
||||
{%- set ns2.consecutive_failures = 0 %}
|
||||
{%- endif %}
|
||||
{%- if ns2.prev_role != 'tool' %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{%- set _is_json_payload = (_tool_format == 'json' and content | trim | length > 0 and (content | trim)[:1] in ('{', '[')) %}
|
||||
{%- if not _is_json_payload and max_tool_response_chars > 0 and content | length > max_tool_response_chars %}
|
||||
{%- set content = content[:max_tool_response_chars] + '\n[TRUNCATED - original length ' ~ (content | length | string) ~ ' chars]' %}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' + content }}
|
||||
{%- if ns2.consecutive_failures >= 2 %}
|
||||
{{- '\n\n⚠️ SYSTEM WARNING: ' ~ ns2.consecutive_failures ~ ' consecutive tool errors detected. Your previous approach is incorrect. You MUST use a fundamentally different approach or corrected arguments.' }}
|
||||
{%- elif ns2.consecutive_failures == 1 %}
|
||||
{{- '\n\n⚠️ SYSTEM WARNING: The previous tool call returned an error. Diagnose the failure and retry with completely corrected arguments.' }}
|
||||
{%- endif %}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- set _next_role = _msgs[loop.index0 + 1].role %}
|
||||
{%- if _next_role != 'tool' %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>user\n[' + message.role + ']: ' + content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- set ns2.prev_role = message.role %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- if not ns_state.thinking %}
|
||||
{{- '<think>\n\n</think>\n\n' }}
|
||||
{%- else %}
|
||||
{{- '<think>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
@@ -0,0 +1,105 @@
|
||||
# qwen4exp-swift15-w4dn8-i8-hc8m.recipe -- the Swift 1.5 finetune of Qwen3.8-Flash-Next
|
||||
# (ukisai/Swift1.5-Qwen3.8-Flash-Next, BF16 source, 335.3 GiB, key set identical to the official
|
||||
# base) as a radiance container for frank-archon: EVERY routed expert -- gate_up AND down --
|
||||
# through GPTQ into the w4nl64a8h table. The format assigns exactly ONE encoding per table
|
||||
# ("one table of experts takes one encoding, and only a whole expert kept plain bf16 is served
|
||||
# beside it", qwen4exp_fp8 rad_arch_declare — so the original split tier (down block-fp8 +
|
||||
# gate_up w4): the served kernel is w4nl64a8h for the whole expert set, bf16 experts beside it.
|
||||
# The rest is qwen4exp-w5dn64-i8-hc8m.recipe verbatim: int8 trunk linears, block-fp8 hc mixers
|
||||
# and MTP fc, the fixed-scale n-gram table, the 2-bit draft head.
|
||||
#
|
||||
# Why down is the calibrated half: the calibration tap rides the down projection, so its GPTQ
|
||||
# rounding propagates against real down-input Hessians; gate_up shares the same w4 table
|
||||
# uncalibrated. The published w5dn64 comparison (mean KL 0.0789 -> 0.0717, top-1 92.85% ->
|
||||
# 93.34%, decode +4%, prefill -19%) is the 5-bit upper bound; this tier trades part of that
|
||||
# quality back for prefill.
|
||||
#
|
||||
# CALIB=DIR rad-convert <Swift1.5-Qwen3.8-Flash-Next checkpoint> \
|
||||
# --recipe scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe -o qwen4exp-swift15-w4dn8-i8-hc8m.rad
|
||||
#
|
||||
# DIR is the calibration directory the bf16 container wrote (RADIANCE_CALIB_DIR, 10M tokens of
|
||||
# traffic-shaped corpus). Everything not named here is served as the checkpoint holds it (bf16:
|
||||
# embeddings, the vision tower, the QSA indexer); the first rule that matches a weight decides it.
|
||||
# The bf16 container is NOT passed with --reuse: the peak-disk arithmetic does not fit (checkpoint +
|
||||
# container + container + output > purge-freed space), and --reuse only saves a re-read.
|
||||
|
||||
# The protected experts, cast to bf16 ahead of both expert rules. THE NAMES BELOW ARE THE BASE
|
||||
# CHECKPOINT'S (from its 1M-token recording, qwen4exp-w5dn64-i8-hc8m.recipe) -- a warm start only.
|
||||
# The Swift 1.5 finetune's routing distribution is not assumed equal: the derivation step rewrites
|
||||
# this section from Swift's OWN calibration recording before this recipe is ever run, by the
|
||||
# protection rule -- an expert whose share of its layer's down-input energy is at least 3% and at least three
|
||||
# times its share of the routing, an even count on each rank's half of the layer (the four-bit GEMM
|
||||
# takes the rest as two equal tables by parity) by adding that half's next most energetic expert.
|
||||
# PROTECTED-BEGIN (rewritten from Swift's OWN grams by derive-protected-experts.py)
|
||||
blk.1.ffn_*_exps.271.weight cast dtype=bf16
|
||||
blk.1.ffn_*_exps.355.weight cast dtype=bf16
|
||||
blk.2.ffn_*_exps.279.weight cast dtype=bf16
|
||||
blk.2.ffn_*_exps.413.weight cast dtype=bf16
|
||||
blk.34.ffn_*_exps.146.weight cast dtype=bf16
|
||||
blk.34.ffn_*_exps.153.weight cast dtype=bf16
|
||||
blk.34.ffn_*_exps.407.weight cast dtype=bf16
|
||||
blk.34.ffn_*_exps.453.weight cast dtype=bf16
|
||||
blk.37.ffn_*_exps.288.weight cast dtype=bf16
|
||||
blk.37.ffn_*_exps.377.weight cast dtype=bf16
|
||||
blk.38.ffn_*_exps.406.weight cast dtype=bf16
|
||||
blk.38.ffn_*_exps.420.weight cast dtype=bf16
|
||||
blk.42.ffn_*_exps.37.weight cast dtype=bf16
|
||||
blk.42.ffn_*_exps.90.weight cast dtype=bf16
|
||||
blk.42.ffn_*_exps.302.weight cast dtype=bf16
|
||||
blk.42.ffn_*_exps.388.weight cast dtype=bf16
|
||||
blk.44.ffn_*_exps.315.weight cast dtype=bf16
|
||||
blk.44.ffn_*_exps.350.weight cast dtype=bf16
|
||||
blk.46.ffn_*_exps.290.weight cast dtype=bf16
|
||||
blk.46.ffn_*_exps.419.weight cast dtype=bf16
|
||||
blk.47.ffn_*_exps.119.weight cast dtype=bf16
|
||||
blk.47.ffn_*_exps.143.weight cast dtype=bf16
|
||||
blk.47.ffn_*_exps.399.weight cast dtype=bf16
|
||||
blk.47.ffn_*_exps.445.weight cast dtype=bf16
|
||||
# PROTECTED-END
|
||||
# (derive-protected-experts.py rewrote this section from Swift's own grams)
|
||||
|
||||
# Every other routed expert, trunk and MTP layer alike: u4 codes into the w4nl table, rotated by
|
||||
# the 128-wide Hadamard, an E4M3 scale a group of 64 under the fixed 2^-13 searched for least
|
||||
# error as stored, chosen by GPTQ against $CALIB and swept three times by coordinate descent.
|
||||
# The calibration Grams are down-input Hessians, so down is the calibrated half of the table;
|
||||
# gate_up has no recording of its own and quantises uncalibrated (said on stderr, as the base
|
||||
# recipe says). The MTP layer has no recording either. Expert mass: gate_up 150.00 GiB -> ~37.6,
|
||||
# down 75.00 GiB -> ~18.8: ~56 GiB of experts, ~28 GiB a rank at TP=2 -- well inside the pool.
|
||||
blk.*.ffn_*_exps.*.weight gptq table=w4nl group=64 scale=fp8_e4m3 scale2=f32 block2=*x* scale2_value=0.0001220703125 transform=fwht128 rule=search cd=3 calib=$CALIB
|
||||
|
||||
# Every connection's two mixing matrices -- the 96 trunk connections, the final mixer and the MTP
|
||||
# head's three -- as E4M3 rows with an f32 scale a group of each row. A 128x128 block cannot cover
|
||||
# them: lowrank is 320, 2.5 blocks. Down is [320, 10240] and takes a scale every 128 columns; up is
|
||||
# [10240, 320] and takes four scales a row, 80 columns each.
|
||||
*_hc_down.weight rtn codes=fp8_e4m3 group=128 scale=f32
|
||||
*_hc_up.weight rtn codes=fp8_e4m3 group=80 scale=f32
|
||||
|
||||
# The MTP head's fc_hidden and fc_embedding, the same rows. They only ever change the drafts.
|
||||
mtp.fc_hidden.weight rtn codes=fp8_e4m3 group=128 scale=f32
|
||||
mtp.fc_embedding.weight rtn codes=fp8_e4m3 group=128 scale=f32
|
||||
|
||||
# The trunk's block linears as int8, a scale per 128 columns of a row searched for least error.
|
||||
# The QSA indexer's projection is not one of them: its top-k is a hard choice between blocks, and
|
||||
# keeping it as the checkpoint holds it costs 20 MB.
|
||||
blk.*.attn_qg.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
blk.*.attn_k.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
blk.*.attn_v.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
blk.*.attn_output.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
blk.*.ssm_inz.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
blk.*.ssm_out.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
blk.*.ffn_gate_up_shexp.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
blk.*.ffn_down_shexp.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
|
||||
# The lm_head, int8 with a bf16 scale a row per 128 columns (logits_gemm_i8): at E4M3 it costs
|
||||
# ~0.6 points of top-1 on its own.
|
||||
output.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
|
||||
# The n-gram table at one fixed E4M3 scale; qwen4exp-w4.recipe says where the value comes from.
|
||||
# The value sits on a three-octave plateau (2^-13..2^-10, rel_l2 0.02662) with 1.7x headroom over
|
||||
# the base table's absmax; the prep phase verifies Swift's PLE absmax lands inside the plateau
|
||||
# before this rule is trusted.
|
||||
blk.*.ple_ngram.weight rtn codes=fp8_e4m3 block=*x* scale=bf16 rule=fixed scale_value=1.99317932128906e-4
|
||||
|
||||
# The MTP head's 2-bit draft head. INERT for this checkpoint: the HF weight map has no
|
||||
# mtp.draft_head.weight (same as the base), kept for shape parity with the family.
|
||||
mtp.draft_head.weight rtn codes=u2 zero=u8 group=128 scale=f16
|
||||
Executable
+116
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env bash
|
||||
# run-swift15-bf16.sh — RUNS ON DANKSERVER. Boot the no-recipe bf16 Swift1.5
|
||||
# container for the two offline campaign phases. The engine owns the GPUs, so
|
||||
# the production unit must be stopped first (this script does it and restores
|
||||
# on exit, autoresearch pattern: stop, never mask — relinked unit).
|
||||
#
|
||||
# kld --kld-record the reference logits from the bf16 model
|
||||
# (runs the corpus and EXITS; engine writes DIR and leaves)
|
||||
# calib serve with the calibration tap; a separate driver
|
||||
# (scripts/calib-push.py) pushes the corpus; the engine writes the
|
||||
# Hessians at RADIANCE_CALIB_TOKENS and the tap switches itself off.
|
||||
# Stop this script after calib-push prints TAP_DONE.
|
||||
#
|
||||
# Shape = the vendor's bf16 calibration lines (GUIDE.md §4.3:528 + MOE-W4.md
|
||||
# "calibrate the checkpoint as it ships"): expert_tiered + weights-disk-tier,
|
||||
# no speculation, deterministic. Large prefill steps because a disk-tier step
|
||||
# costs the same at 1 token as at 32K. The container declared --max-ctx 4096;
|
||||
# --max-model-len 16384 exceeds it on purpose (the KLD prompts run to 12,007
|
||||
# tokens, calib to 8,242) — the boot ledger arbitrates the QSA buffers; if
|
||||
# declare refuses, re-convert with a wider --max-ctx.
|
||||
set -uo pipefail
|
||||
|
||||
MODE="${1:?usage: run-swift15-bf16.sh kld|calib}"
|
||||
PIN_1013=sha256:4c3f484564bb2872784e46b0a59cbd71becc9c1e746e301a3596ffae253c322e
|
||||
PIN_120=sha256:665775ed1d90de6dbb25558e07fc60d20328f1346f96f3c6e00845eb763f01fa
|
||||
RAD=/home/dank/models/rad/qwen4exp-swift15-bf16.rad
|
||||
C=swift15ref
|
||||
LOG=/home/dank/logs/swift15_bf16.log
|
||||
KLD_REF_DIR=/home/dank/corpora/kld-ref
|
||||
CALIB_DIR=/home/dank/corpora/calib
|
||||
KLD_CORPUS=/home/dank/corpora/out/kld.jsonl
|
||||
|
||||
restore() { sudo systemctl start frank-archon >/dev/null 2>&1 || true; }
|
||||
trap restore EXIT
|
||||
sudo systemctl stop frank-archon >/dev/null 2>&1 || true
|
||||
podman rm -f "$C" >/dev/null 2>&1 || true
|
||||
|
||||
ARGS=(--model /models/rad/qwen4exp-swift15-bf16.rad
|
||||
--tp 2 --placement expert_tiered --weights-disk-tier
|
||||
--host-pool-mib 12288 --deterministic --gpu-headroom-mib 768
|
||||
--num-speculative-tokens 0
|
||||
--kv-cache-dtype fp8 --max-model-len 16384
|
||||
--ngram-placement ram
|
||||
--override-chat-template /models/tpl.jinja)
|
||||
|
||||
case "$MODE" in
|
||||
kld)
|
||||
# 1.0.13: the engine that will serve the candidate — the reference is
|
||||
# recorded on the same engine that scores it. The weights slab DERIVES as
|
||||
# card - headroom - kv, and the disk tier refuses under 9.57 GiB (4.21 of
|
||||
# layer buffers + 5.36 fixed), so kv is explicit SMALL: recording is
|
||||
# prefill-only, self-batched at MNBT 32768 = two 16k prompts at a time;
|
||||
# 4 slots x 16k fp8 (29 KiB/pos, 1.7 GiB) + headroom fits 3072.
|
||||
IMAGE="stilldeadcode/radiance@$PIN_1013"
|
||||
mkdir -p "$KLD_REF_DIR"
|
||||
ARGS+=(--kld-record /data/kld-ref --kld-corpus /data/kld.jsonl
|
||||
--vram-kv-mib 3072
|
||||
--max-num-seqs 4 --max-num-batched-tokens 32768)
|
||||
;;
|
||||
calib)
|
||||
# 1.2.0: the calibration tap (RADIANCE_CALIB_DIR + RADIANCE_CALIB_TOKENS)
|
||||
# lives in the 1.2.0 qwen4exp plugin; 1.0.13 has only the DIR half with an
|
||||
# unparameterisable threshold. Grams are bf16 activations — engine-invariant.
|
||||
# Same KV ceiling as kld: the disk-tier weights floor (9.57 GiB) plus
|
||||
# ~18 GiB of fixed per-rank claims cap the pool near 3.5 GiB on a 32 GiB
|
||||
# card — pool minus the 2.17 GiB fixed holds ~237k fp8 tokens a rank, so
|
||||
# the vendor's big-step calibration shape is affordable: MOE-W4.md:244
|
||||
# measures 640 tok/s at a 32k step against 930 at 55k, and the doc's own
|
||||
# run drives MNBT 65536. 8 slots x 16k = 131k resident tokens fits the
|
||||
# pool; the step batches 65k. KLD's observed rate confirmed step size,
|
||||
# not slot count, is the throughput lever.
|
||||
IMAGE="stilldeadcode/radiance@$PIN_120"
|
||||
mkdir -p "$CALIB_DIR"
|
||||
ARGS+=(--port 7770 --vram-kv-mib 3072
|
||||
--max-num-seqs 8 --max-num-batched-tokens 65536
|
||||
--prefix-cache-dir /data/calib-kvcache)
|
||||
;;
|
||||
probe)
|
||||
IMAGE="stilldeadcode/radiance@$PIN_1013"
|
||||
ARGS+=(--port 7770 --max-num-seqs 8 --max-num-batched-tokens 4096)
|
||||
;;
|
||||
*) echo "unknown mode: $MODE" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
# The prod template copy, readable from the container: mount the models tree
|
||||
# ro plus the state dir rw (KLD record + prefix tier land there).
|
||||
cp -f /home/dank/serverAppConfigs/scripts/qwen38-froggeric-v22.5.jinja \
|
||||
/home/dank/models/tpl.jinja 2>/dev/null || true
|
||||
|
||||
KLD_MOUNT=()
|
||||
if [ "$MODE" = kld ]; then
|
||||
# Nested under the /data mount, so these must come after it (line order
|
||||
# in podman run below): the corpus read-only, the record dir writable.
|
||||
KLD_MOUNT=(-v "$KLD_CORPUS:/data/kld.jsonl:ro"
|
||||
-v "$KLD_REF_DIR:/data/kld-ref")
|
||||
fi
|
||||
EXTRA_ENV=()
|
||||
if [ "$MODE" = calib ]; then
|
||||
EXTRA_ENV=(-v "$CALIB_DIR:/data/calib"
|
||||
-e RADIANCE_CALIB_DIR=/data/calib
|
||||
-e RADIANCE_CALIB_TOKENS=10000000)
|
||||
fi
|
||||
|
||||
podman run --rm --name "$C" \
|
||||
--network host --init \
|
||||
--security-opt seccomp=unconfined \
|
||||
--ulimit memlock=-1:-1 \
|
||||
--device /dev/kfd --device /dev/dri \
|
||||
-v /home/dank/models:/models:ro \
|
||||
-v /home/dank/radiance-state:/data \
|
||||
"${KLD_MOUNT[@]}" \
|
||||
"${EXTRA_ENV[@]}" \
|
||||
"$IMAGE" "${ARGS[@]}" 2>&1 | tee "$LOG"
|
||||
rc=${PIPESTATUS[0]}
|
||||
echo "swift15-bf16 $MODE exited rc=$rc; log: $LOG"
|
||||
exit "$rc"
|
||||
Executable
+196
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env bash
|
||||
# serve-frank-archon.sh — frank-archon endpoint: radiance (StillDeadcode,
|
||||
# Apache-2.0) serving the OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 checkpoint at
|
||||
# NATIVE block-FP8 precision from a rad-convert container
|
||||
# (/home/dank/models/rad/qwen4exp-fp8-native.rad), on BOTH R9700s (TP=2,
|
||||
# wht6 wire), dankserver :7770. 2026-10-05 cutover from FreeToken (PR #587,
|
||||
# one card; git tag frank-archon-freetoken-pr587-20261005 is the
|
||||
# pre-cutover state).
|
||||
#
|
||||
# The run line below IS the engine argv: the image's entrypoint is the
|
||||
# engine and its tools sit beside it on PATH. Unlike every predecessor
|
||||
# there is no per-boot VRAM planner to second-guess: the boot ledger in the
|
||||
# log states every budget before anything is allocated (weights vs KV
|
||||
# split, the expert host pool's coverage of the worst case, the PLE pin,
|
||||
# the P2P answer per pair) — read it after every resize.
|
||||
# Deployed shape (2026-10-13): THREE decode slots x 200,000 tokens, fp8
|
||||
# (quantized) KV in a 5,120 MiB pool (contract 3 x 200,000 + 1.3 %) and a
|
||||
# 25,600 MiB resident expert slab — 30,720 MiB of the 30,969 MiB the engine's
|
||||
# own budget table makes claimable, spent on the slab because prefill is
|
||||
# slab-limited. 44 GiB/rank pinned expert pool, MTP depth 2, prefix cache host
|
||||
# 8 GiB + disk 64 GiB, PLE pinned in RAM.
|
||||
#
|
||||
# The root unit (scripts/frank-archon.service, User=dank,
|
||||
# LimitMEMLOCK=infinity) grants the pin budget: 2 x 44 GiB expert pools +
|
||||
# the 47.7 GiB PLE table + 8 GiB prefix host tier — ~147 GiB of pinned
|
||||
# pages, which is ALSO why this script refuses to start when ttm's
|
||||
# pages_limit/pages_max cap is under FRANK_ARCHON_TTM_MIN_MIB (see below).
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_ARCHON_ENV_FILE:-$SCRIPT_DIR/frank-archon.env}"
|
||||
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
echo "ERROR: frank-archon env file absent: $ENV_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
|
||||
: "${FRANK_ARCHON_IMAGE:?FRANK_ARCHON_IMAGE must name the radiance image}"
|
||||
: "${FRANK_ARCHON_MODEL_DIR:?FRANK_ARCHON_MODEL_DIR must name the .rad dir}"
|
||||
: "${FRANK_ARCHON_MODEL_FILE:?FRANK_ARCHON_MODEL_FILE must name the .rad file}"
|
||||
: "${FRANK_ARCHON_STATE_DIR:?FRANK_ARCHON_STATE_DIR must name the /data state dir}"
|
||||
: "${FRANK_ARCHON_CONTAINER:?FRANK_ARCHON_CONTAINER must name the container}"
|
||||
: "${FRANK_ARCHON_SERVED_NAME:?FRANK_ARCHON_SERVED_NAME must name the alias}"
|
||||
: "${FRANK_ARCHON_PORT:?FRANK_ARCHON_PORT must be the port}"
|
||||
: "${FRANK_ARCHON_TEMPLATE:?FRANK_ARCHON_TEMPLATE must name the chat template}"
|
||||
: "${FRANK_ARCHON_TP:?FRANK_ARCHON_TP must be set}"
|
||||
: "${FRANK_ARCHON_TP_WIRE:?FRANK_ARCHON_TP_WIRE must be set}"
|
||||
: "${FRANK_ARCHON_MAX_SEQS:?FRANK_ARCHON_MAX_SEQS must be set}"
|
||||
: "${FRANK_ARCHON_MAX_MODEL_LEN:?FRANK_ARCHON_MAX_MODEL_LEN must be set}"
|
||||
: "${FRANK_ARCHON_PLACEMENT:?FRANK_ARCHON_PLACEMENT must be set}"
|
||||
: "${FRANK_ARCHON_HOST_POOL_MIB:?FRANK_ARCHON_HOST_POOL_MIB must be set}"
|
||||
: "${FRANK_ARCHON_GPU_HEADROOM_MIB:?FRANK_ARCHON_GPU_HEADROOM_MIB must be set}"
|
||||
: "${FRANK_ARCHON_VRAM_WEIGHTS_MIB:?FRANK_ARCHON_VRAM_WEIGHTS_MIB must be set}"
|
||||
: "${FRANK_ARCHON_VRAM_KV_MIB:?FRANK_ARCHON_VRAM_KV_MIB must be set}"
|
||||
: "${FRANK_ARCHON_KV_DTYPE:?FRANK_ARCHON_KV_DTYPE must be set}"
|
||||
: "${FRANK_ARCHON_SPEC_TOKENS:?FRANK_ARCHON_SPEC_TOKENS must be set}"
|
||||
: "${FRANK_ARCHON_MAX_BATCHED_TOKENS:?FRANK_ARCHON_MAX_BATCHED_TOKENS must be set}"
|
||||
: "${FRANK_ARCHON_PREFIX_HOST_MIB:?FRANK_ARCHON_PREFIX_HOST_MIB must be set}"
|
||||
: "${FRANK_ARCHON_PREFIX_DISK_MIB:?FRANK_ARCHON_PREFIX_DISK_MIB must be set}"
|
||||
: "${FRANK_ARCHON_NGRAM_PLACEMENT:?FRANK_ARCHON_NGRAM_PLACEMENT must be set}"
|
||||
: "${FRANK_ARCHON_TTM_MIN_MIB:?FRANK_ARCHON_TTM_MIN_MIB must be set}"
|
||||
: "${FRANK_ARCHON_TTM_TARGET_MIB:?FRANK_ARCHON_TTM_TARGET_MIB must be set}"
|
||||
: "${FRANK_ARCHON_RAD_MIN_MIB:?FRANK_ARCHON_RAD_MIN_MIB must be set}"
|
||||
|
||||
# The image gate is the digest-pinned ref itself: a name@sha256 ref exists
|
||||
# in storage only if THAT manifest was pulled — a retagged look-alike or a
|
||||
# newer latest does not satisfy it. `podman image exists` takes a
|
||||
# name@digest ref in one command — no pipefail/grep race (the 2026-09-30
|
||||
# lesson). Runs as dank: the unit's process uses dank's ROOTLESS podman
|
||||
# storage (root unit, User=dank — the fleet's frank-archon shape).
|
||||
if ! podman image exists "$FRANK_ARCHON_IMAGE"; then
|
||||
echo "ERROR: radiance image absent: $FRANK_ARCHON_IMAGE" >&2
|
||||
echo "frank-archon: podman pull docker://stilldeadcode/radiance:latest (as dank)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The payload gate: the .rad container file, sized. A truncated convert
|
||||
# would otherwise buy a guaranteed long-timeout boot instead of this fast
|
||||
# failure. The staging dir / raw checkpoint are NOT this script's concern
|
||||
# (scripts/build-frank-archon-rad.sh built the file; its rad-info output
|
||||
# is the record).
|
||||
RAD="$FRANK_ARCHON_MODEL_DIR/$FRANK_ARCHON_MODEL_FILE"
|
||||
if [ ! -f "$RAD" ]; then
|
||||
echo "ERROR: .rad container missing: $RAD" >&2
|
||||
echo "frank-archon: run scripts/build-frank-archon-rad.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
RAD_MIB=$(( $(stat -c %s "$RAD") / 1048576 ))
|
||||
if [ "$RAD_MIB" -lt "$FRANK_ARCHON_RAD_MIN_MIB" ]; then
|
||||
echo "ERROR: .rad container too small: $RAD is ${RAD_MIB} MiB, want >= $FRANK_ARCHON_RAD_MIN_MIB" >&2
|
||||
echo "frank-archon: the convert was truncated — rebuild it" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$FRANK_ARCHON_TEMPLATE" ]; then
|
||||
echo "ERROR: chat template missing: $FRANK_ARCHON_TEMPLATE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$FRANK_ARCHON_STATE_DIR/kvcache/frank-archon"
|
||||
# The engine's create is NOT recursive: with only the state dir mounted, its
|
||||
# "cannot create /data/kvcache/frank-archon: No such file or directory"
|
||||
# left the SSD prefix tier off (measured 2026-10-06 boot ledger). Pre-create
|
||||
# the full path on the host so the tier opens on first boot.
|
||||
|
||||
mkdir -p "$FRANK_ARCHON_STATE_DIR/tune"
|
||||
# rad-tune's kernel-selection cache (see the runbook's "Tuning levers"):
|
||||
# the engine's selector reads $RADIANCE_HOME/tune/<machine>.tune at boot;
|
||||
# $RADIANCE_HOME defaults to /opt/radiance, which lives in the image, so
|
||||
# the cache must ride a host mount or it dies with the --rm container
|
||||
# that wrote it. The dir is pre-created for the same reason as kvcache
|
||||
# (the engine's create is not recursive).
|
||||
|
||||
if [ ! -e /dev/kfd ] || [ ! -d /dev/dri ]; then
|
||||
echo "ERROR: GPU device nodes absent (/dev/kfd, /dev/dri)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# TTM preflight: the pinned ledger (2 x FRANK_ARCHON_HOST_POOL_MIB + the
|
||||
# 47.7 GiB PLE table + the prefix host tier, ~147 GiB) rides TTM's shared
|
||||
# pages, capped by ttm's pages_limit (the current kernel exposes NO
|
||||
# pages_max; the cap function takes the min over the knobs that exist).
|
||||
# Below FRANK_ARCHON_TTM_MIN_MIB the load dies mid-pin with an allocator
|
||||
# error, so the gate refuses first — and RAISES the knob before refusing:
|
||||
# the cap is a runtime knob whose default is half of RAM, so a power failure
|
||||
# resets it (measured 2026-10-12 23:17 on this host: 200 GiB -> 125,988 MiB,
|
||||
# and Restart=always re-ran the refusal 6,919 times over ~20 h).
|
||||
# FRANK_ARCHON_TTM_TARGET_MIB is the recorded host ceiling, not the floor.
|
||||
# shellcheck source=/dev/null
|
||||
source "$SCRIPT_DIR/ttm-preflight.sh"
|
||||
ttm_ensure_cap || exit 1
|
||||
|
||||
# A crashed/killed unit can leave the container name registered (systemd
|
||||
# SIGKILLs conmon before cleanup runs). Clear OUR name plus the legacy
|
||||
# identities (the FreeToken-era frank-archon-qwen, the retired R9V
|
||||
# container and the ad hoc experiment containers all hold this GPU); all
|
||||
# idempotent.
|
||||
for CTR in $FRANK_ARCHON_LEGACY_CONTAINERS "$FRANK_ARCHON_CONTAINER"; do
|
||||
podman rm -f "$CTR" >/dev/null 2>&1 || true
|
||||
done
|
||||
|
||||
LOG=/home/dank/logs/frank_archon.log
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
|
||||
# Publisher-vetted container contract (docs/DOCKER.md, compose/common.yaml):
|
||||
# --network host (the server binds the host's addresses directly, no NAT),
|
||||
# --init (the engine releases the cards on SIGTERM), seccomp=unconfined
|
||||
# (Docker/podman's default profile refuses io_uring, which the n-gram
|
||||
# table's reads go through, and the NUMA binding calls the ROCm runtime
|
||||
# makes for host memory), memlock unbounded (the pinned ledger — legal
|
||||
# because the root unit grants LimitMEMLOCK=infinity to this dank process),
|
||||
# /dev/kfd + /dev/dri (render/video groups are moot for a root container).
|
||||
# The template rides a single-file mount; the .rad dir is ro; /data holds
|
||||
# the prefix-cache disk tier.
|
||||
podman run -d --name "$FRANK_ARCHON_CONTAINER" \
|
||||
--network host \
|
||||
--init \
|
||||
--security-opt seccomp=unconfined \
|
||||
--ulimit memlock=-1:-1 \
|
||||
--device /dev/kfd --device /dev/dri \
|
||||
-v "$FRANK_ARCHON_MODEL_DIR":/models:ro \
|
||||
-v "$FRANK_ARCHON_STATE_DIR":/data \
|
||||
-v "$FRANK_ARCHON_STATE_DIR/tune":/opt/radiance/share/radiance/tune \
|
||||
-v "$FRANK_ARCHON_TEMPLATE":/templates/template.jinja:ro \
|
||||
"$FRANK_ARCHON_IMAGE" \
|
||||
--model "/models/$FRANK_ARCHON_MODEL_FILE" \
|
||||
--host :: --port "$FRANK_ARCHON_PORT" \
|
||||
--served-model-name "$FRANK_ARCHON_SERVED_NAME" \
|
||||
--override-chat-template /templates/template.jinja \
|
||||
--tp "$FRANK_ARCHON_TP" --tp-wire "$FRANK_ARCHON_TP_WIRE" \
|
||||
--max-num-seqs "$FRANK_ARCHON_MAX_SEQS" \
|
||||
--max-model-len "$FRANK_ARCHON_MAX_MODEL_LEN" \
|
||||
--placement "$FRANK_ARCHON_PLACEMENT" \
|
||||
--host-pool-mib "$FRANK_ARCHON_HOST_POOL_MIB" \
|
||||
--gpu-headroom-mib "$FRANK_ARCHON_GPU_HEADROOM_MIB" \
|
||||
--vram-weights-mib "$FRANK_ARCHON_VRAM_WEIGHTS_MIB" \
|
||||
--vram-kv-mib "$FRANK_ARCHON_VRAM_KV_MIB" \
|
||||
--kv-cache-dtype "$FRANK_ARCHON_KV_DTYPE" \
|
||||
--prefix-cache-host-mib "$FRANK_ARCHON_PREFIX_HOST_MIB" \
|
||||
--prefix-cache-dir /data/kvcache/frank-archon \
|
||||
--prefix-cache-disk-mib "$FRANK_ARCHON_PREFIX_DISK_MIB" \
|
||||
--ngram-placement "$FRANK_ARCHON_NGRAM_PLACEMENT" \
|
||||
--num-speculative-tokens "$FRANK_ARCHON_SPEC_TOKENS" \
|
||||
--max-num-batched-tokens "$FRANK_ARCHON_MAX_BATCHED_TOKENS" \
|
||||
${FRANK_ARCHON_EXTRA_ARGS:+$FRANK_ARCHON_EXTRA_ARGS} \
|
||||
> "$LOG" 2>&1
|
||||
|
||||
# From here on, this script's lifetime IS the container's: wait returns the
|
||||
# container's exit code, so a crash is a nonzero ExecStart and the unit's
|
||||
# Restart=always is real. (systemd stop goes through ExecStop in the unit —
|
||||
# podman wait alone cannot stop the container child.)
|
||||
exec podman wait "$FRANK_ARCHON_CONTAINER"
|
||||
Executable
+202
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env bash
|
||||
# serve-frank-dragoon.sh — frank-dragoon endpoint: halogen-flash-server 0.16.2
|
||||
# (closed-source engine, image ghcr.io/peonist-ai/halogen-flash-server,
|
||||
# digest-pinned in frank-dragoon.env) serving the Quat3rnion abliterated
|
||||
# Swift1.5 finetune of peonist's Qwen3.8-Flash-Next v2 checkpoint on
|
||||
# Dankclaw's Strix Halo iGPU, host :7770. 2026-10-06 cutover from the
|
||||
# halogen 27B-dense shape — revert the swap commit's trio to roll back
|
||||
# (the 27B payload stays on disk).
|
||||
#
|
||||
# The engine is driven by `podman run` DIRECTLY (the fleet's frank-archon
|
||||
# pattern): the image starts engine + OpenAI frontend in one container.
|
||||
# The frontend listens on 8731 inside (HALOGEN_API_PORT default) and is
|
||||
# published as host 7770; the engine's own token port (8730, no auth)
|
||||
# stays unpublished on the container's loopback. HALOGEN_CHECKPOINT
|
||||
# selects the file inside the /models mount; the n-gram lookup table
|
||||
# (HALOGEN_NGRAM_TABLE, the card sets it explicitly though the documented
|
||||
# fallback is "beside the checkpoint") and the vision tower
|
||||
# (HALOGEN_VISION_TOWER) ride in the same directory.
|
||||
#
|
||||
# Gates mirror the deployer EXACTLY (the 2026-10-01 crash-loop lesson —
|
||||
# serve gate and deploy gate disagreeing = green deploy, red unit):
|
||||
# checkpoint/n-gram/vision byte-exact, tokenizer directory present with
|
||||
# every named file a FLAT REGULAR FILE (a HF cache snapshot's symlinks
|
||||
# into blobs/ dangle inside the mount), GPU device nodes present.
|
||||
#
|
||||
# Why containers instead of a bare binary: the payload is .hgn and only
|
||||
# the flash-server line reads it; it ships as a container; the root unit
|
||||
# keeps LimitMEMLOCK=infinity for parity and to outlive the login session,
|
||||
# exactly like frank-archon's.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}"
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
echo "frank-dragoon: env file not found: $ENV_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
|
||||
: "${FRANK_DRAGOON_IMAGE:?FRANK_DRAGOON_IMAGE must name the digest-pinned image}"
|
||||
: "${FRANK_DRAGOON_CONTAINER:?FRANK_DRAGOON_CONTAINER must name the container}"
|
||||
: "${FRANK_DRAGOON_PUBLISH:?FRANK_DRAGOON_PUBLISH must be host:container}"
|
||||
: "${FRANK_DRAGOON_MODEL_DIR:?FRANK_DRAGOON_MODEL_DIR must name the payload dir}"
|
||||
: "${FRANK_DRAGOON_CHECKPOINT:?FRANK_DRAGOON_CHECKPOINT must name the .hgn inside it}"
|
||||
: "${FRANK_DRAGOON_CHECKPOINT_BYTES:?FRANK_DRAGOON_CHECKPOINT_BYTES must pin the size}"
|
||||
: "${FRANK_DRAGOON_NGRAM_TABLE:?FRANK_DRAGOON_NGRAM_TABLE must name the lookup table}"
|
||||
: "${FRANK_DRAGOON_NGRAM_BYTES:?FRANK_DRAGOON_NGRAM_BYTES must pin the table size}"
|
||||
: "${FRANK_DRAGOON_VISION_TOWER:?FRANK_DRAGOON_VISION_TOWER must name the tower}"
|
||||
: "${FRANK_DRAGOON_VISION_BYTES:?FRANK_DRAGOON_VISION_BYTES must pin the tower size}"
|
||||
: "${FRANK_DRAGOON_TOKENIZER_FILES:?FRANK_DRAGOON_TOKENIZER_FILES must list the tokenizer dir contents}"
|
||||
: "${HALOGEN_MODEL_ID:?HALOGEN_MODEL_ID must be the served alias (frank-dragoon)}"
|
||||
: "${HALOGEN_CTX:?HALOGEN_CTX must be set (per-slot window)}"
|
||||
: "${HALOGEN_KV_POOL_POSITIONS:?HALOGEN_KV_POOL_POSITIONS must be set}"
|
||||
: "${HALOGEN_KV_SLOTS:?HALOGEN_KV_SLOTS must be set}"
|
||||
: "${HALOGEN_SPEC_ADAPT:?HALOGEN_SPEC_ADAPT must be set (0 = head always on)}"
|
||||
|
||||
# Capture podman's output BEFORE grepping: under `set -o pipefail` the
|
||||
# `podman images | grep -q` pattern races (grep exits on first match,
|
||||
# podman dies SIGPIPE 141, pipefail turns the 141 into "no match" — the
|
||||
# 2026-09-30 deploy refusal).
|
||||
DIGESTS="$(podman images --format '{{.RepoDigests}}')"
|
||||
if ! grep -qF "${FRANK_DRAGOON_IMAGE#*@}" <<<"$DIGESTS"; then
|
||||
echo "frank-dragoon: image $FRANK_DRAGOON_IMAGE not in podman storage;" >&2
|
||||
echo " pull it: podman pull ${FRANK_DRAGOON_IMAGE%@*}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Size-exact gates on all three payload files: a truncated curl exits 0 on
|
||||
# some failures and a partial .hgn loads far enough to look alive, so the
|
||||
# byte counts from the Hub tree API are the contract: curl exits 0 on an
|
||||
# HTTP error and writes the error body as the file, so a truncated download
|
||||
# is caught by size here and never boots.
|
||||
ck() {
|
||||
# ck <env-name> <path> <expected-bytes>
|
||||
if [ ! -f "$2" ]; then
|
||||
echo "frank-dragoon: $1 $2 missing;" >&2
|
||||
echo " fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
local sz
|
||||
sz="$(stat -c %s "$2")"
|
||||
if [ "$sz" != "$3" ]; then
|
||||
echo "frank-dragoon: $1 is $sz bytes, expected $3 (partial download?)" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
ck checkpoint "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_CHECKPOINT" "$FRANK_DRAGOON_CHECKPOINT_BYTES"
|
||||
ck n-gram-table "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_NGRAM_TABLE" "$FRANK_DRAGOON_NGRAM_BYTES"
|
||||
ck vision-tower "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_VISION_TOWER" "$FRANK_DRAGOON_VISION_BYTES"
|
||||
|
||||
# The payload ships the tokenizer as a FLAT DIRECTORY. A HuggingFace cache
|
||||
# snapshot symlinks each file into a sibling blobs/, and those symlinks
|
||||
# dangle inside the container mount — so the directory must be real and
|
||||
# every named file must be a regular file, not a link.
|
||||
TOK_DIR="$FRANK_DRAGOON_MODEL_DIR/tokenizer"
|
||||
if [ -L "$TOK_DIR" ] || [ ! -d "$TOK_DIR" ]; then
|
||||
echo "frank-dragoon: tokenizer dir $TOK_DIR missing or a symlink (HF cache snapshot?);" >&2
|
||||
echo " fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
for TF in $FRANK_DRAGOON_TOKENIZER_FILES; do
|
||||
if [ ! -f "$TOK_DIR/$TF" ] || [ -L "$TOK_DIR/$TF" ]; then
|
||||
echo "frank-dragoon: tokenizer file $TOK_DIR/$TF missing or a symlink;" >&2
|
||||
echo " fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# GPU access is mandatory but the node paths are variables so the seam can
|
||||
# drive the gate on a box without an AMD GPU.
|
||||
DEV_KFD="${FRANK_DRAGOON_DEV_KFD:-/dev/kfd}"
|
||||
DEV_DRI="${FRANK_DRAGOON_DEV_DRI:-/dev/dri}"
|
||||
if [ ! -e "$DEV_KFD" ] || [ ! -d "$DEV_DRI" ]; then
|
||||
echo "frank-dragoon: $DEV_KFD or $DEV_DRI missing — no gfx1151 to serve on" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Host-RAM preflight — the engine's pin gate, checked BEFORE podman run.
|
||||
# The engine refuses to pin when MemAvailable is below the checkpoint plus
|
||||
# a 16 GiB floor: pinned pages cannot be swapped, and under the floor the
|
||||
# n-gram lookup pages in from disk on every long prompt. A RAM-short host
|
||||
# (a co-tenant spike, a sibling engine dying) crash-loops the unit — each
|
||||
# boot dies in ~5 s at the pin gate — until the RAM clears. The 2026-10-06
|
||||
# cutover hit this: 8 boots, MemAvailable 74.77 GiB vs the 78.11 GiB needed,
|
||||
# self-resolved in ~2 min when the dying 27B engine freed RAM. Check here and
|
||||
# WAIT for the RAM to clear, so a transient dip boots once instead of
|
||||
# crash-looping; if it never clears, refuse loudly.
|
||||
MEMINFO="${FRANK_DRAGOON_MEMINFO:-/proc/meminfo}"
|
||||
PIN_KB=$(( FRANK_DRAGOON_CHECKPOINT_BYTES / 1024 ))
|
||||
NEED_KB=$(( PIN_KB + 16 * 1048576 ))
|
||||
ram_avail_kb() { awk '/^MemAvailable:/{print $2; exit}' "$MEMINFO"; }
|
||||
MEMAVAIL_KB="$(ram_avail_kb)" || MEMAVAIL_KB=""
|
||||
if [ -n "$MEMAVAIL_KB" ] && [ "$MEMAVAIL_KB" -lt "$NEED_KB" ]; then
|
||||
WAIT_S="${FRANK_DRAGOON_RAM_WAIT:-600}"
|
||||
GIB() { awk -v kb="$1" 'BEGIN{printf "%.2f", kb/1048576}'; }
|
||||
echo "frank-dragoon: host RAM below the pin gate — MemAvailable $(GIB "$MEMAVAIL_KB") GiB < needed $(GIB "$NEED_KB") GiB ($(GIB "$PIN_KB") GiB checkpoint + 16.00 GiB floor); waiting up to ${WAIT_S}s for host RAM to clear" >&2
|
||||
deadline=$(( $(date +%s) + WAIT_S ))
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
sleep 5
|
||||
MEMAVAIL_KB="$(ram_avail_kb)" || MEMAVAIL_KB=""
|
||||
if [ -n "$MEMAVAIL_KB" ] && [ "$MEMAVAIL_KB" -ge "$NEED_KB" ]; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ -z "$MEMAVAIL_KB" ] || [ "$MEMAVAIL_KB" -lt "$NEED_KB" ]; then
|
||||
echo "frank-dragoon: MemAvailable $(GIB "${MEMAVAIL_KB:-0}") GiB after ${WAIT_S}s; refusing to start (the engine would refuse the pin)" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "frank-dragoon: MemAvailable cleared to $(GIB "$MEMAVAIL_KB") GiB; starting" >&2
|
||||
fi
|
||||
for CTR in ${FRANK_DRAGOON_LEGACY_CONTAINERS:-} "$FRANK_DRAGOON_CONTAINER"; do
|
||||
podman rm -f "$CTR" >/dev/null 2>&1 || true
|
||||
done
|
||||
|
||||
LOG="${FRANK_DRAGOON_LOG:-/home/dank/logs/frank_dragoon.log}"
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
# --ipc=host for the udev-granted supplementary group, seccomp unconfined
|
||||
# for the ROCm ioctl surface, and unbounded memlock (the unit's
|
||||
# LimitMEMLOCK=infinity is inherited, so runc never has to RAISE it). NO
|
||||
# --ulimit nofile: Dankclaw's hard limit is 524288, rootless runc cannot
|
||||
# raise a hard limit and dies with "error setting rlimit type 7:
|
||||
# operation not permitted" (measured 2026-10-05 cutover); the model card
|
||||
# and both sibling serve scripts pass no nofile either. No comment lines
|
||||
# inside the backslash chain — a mid-chain # terminates the continuation
|
||||
# (the 2026-10-05 exit-127 crash loop on the sibling serve script).
|
||||
podman run --name "$FRANK_DRAGOON_CONTAINER" \
|
||||
--publish "$FRANK_DRAGOON_PUBLISH" \
|
||||
--ipc=host \
|
||||
--ulimit memlock=-1:-1 \
|
||||
--device "$DEV_KFD" --device "$DEV_DRI" \
|
||||
--group-add keep-groups \
|
||||
--security-opt seccomp=unconfined \
|
||||
-e HALOGEN_MODEL_ID="$HALOGEN_MODEL_ID" \
|
||||
-e HALOGEN_CHECKPOINT="/models/$FRANK_DRAGOON_CHECKPOINT" \
|
||||
-e HALOGEN_NGRAM_TABLE="/models/$FRANK_DRAGOON_NGRAM_TABLE" \
|
||||
-e HALOGEN_VISION_TOWER="/models/$FRANK_DRAGOON_VISION_TOWER" \
|
||||
-e HALOGEN_TOKENIZER=/tokenizer \
|
||||
-e HALOGEN_CTX="$HALOGEN_CTX" \
|
||||
-e HALOGEN_KV_POOL_POSITIONS="$HALOGEN_KV_POOL_POSITIONS" \
|
||||
-e HALOGEN_KV_SLOTS="$HALOGEN_KV_SLOTS" \
|
||||
-e HALOGEN_HOST_RESERVE_GIB="${HALOGEN_HOST_RESERVE_GIB:-20}" \
|
||||
-e HALOGEN_SPEC_ADAPT="$HALOGEN_SPEC_ADAPT" \
|
||||
-e HALOGEN_MAX_TOK="${HALOGEN_MAX_TOK:-8192}" \
|
||||
-e HALOGEN_MAX_TOKENS_CAP="${HALOGEN_MAX_TOKENS_CAP:-65536}" \
|
||||
-e HALOGEN_QUEUE_TIMEOUT="${HALOGEN_QUEUE_TIMEOUT:-7200}" \
|
||||
-e HALOGEN_TEMPERATURE="${HALOGEN_TEMPERATURE:-1.0}" \
|
||||
-e HALOGEN_TOP_P="${HALOGEN_TOP_P:-0.95}" \
|
||||
-e HALOGEN_TOP_K="${HALOGEN_TOP_K:-20}" \
|
||||
${HALOGEN_WEIGHTS_LOCK:+-e HALOGEN_WEIGHTS_LOCK="$HALOGEN_WEIGHTS_LOCK"} \
|
||||
-v "$FRANK_DRAGOON_MODEL_DIR:/models:ro" \
|
||||
-v "$FRANK_DRAGOON_MODEL_DIR/tokenizer:/tokenizer:ro" \
|
||||
"$FRANK_DRAGOON_IMAGE" \
|
||||
> "$LOG" 2>&1
|
||||
|
||||
# FOREGROUND `podman run`: the engine's stdout/stderr are this script's
|
||||
# output, so the boot ledger (kv pool, prompt cache, MTP head, any clamp)
|
||||
# lands in $LOG as documented. podman run returns the container's exit
|
||||
# code, so a crash is a nonzero ExecStart and the unit's Restart=always
|
||||
# stays real; systemd stop goes through the unit's ExecStop and the run
|
||||
# client forwards SIGTERM to the engine, which drains on it.
|
||||
@@ -1,121 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) as frank-large on
|
||||
# Dankclaw's Strix Halo iGPU (gfx1151), using gufo (gufo-org/gufo) — the
|
||||
# inference engine built specifically for this hardware.
|
||||
#
|
||||
# Runtime: /home/dank/GitPackages/gufo-v0.1.1/bin/gufo, built from source
|
||||
# against the host ROCm by scripts/build-gufo.sh. It replaced the stock
|
||||
# llama.cpp prebuilt (unsloth b11030) on 2026-09-28; port, alias and client
|
||||
# contract did not move.
|
||||
#
|
||||
# Payload: ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF's
|
||||
# UD-Q8_K_XL, fetched by scripts/download-frank-large-swift.sh. It replaced
|
||||
# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL on 2026-09-28. The two are structurally
|
||||
# interchangeable for this engine — same `qwen35` architecture, the same 866
|
||||
# tensors in the same order, the same per-tensor Q8_0/F32/BF16 layout, the
|
||||
# same MTP head at blk.64 — so the swap needed no engine change. The fine-tune
|
||||
# is abliterated (uncensored) and carries the SWIFT OPEN LICENSE 1.0, which
|
||||
# is not an OSI licence.
|
||||
#
|
||||
# Shape: one session, full 262144 context (the model's native window). The
|
||||
# 27B dense weights (16.35 GiB at UD-Q4_K_XL) + the DFlash2 drafter (1.07) + KV +
|
||||
# compute buffers fit the iGPU's GTT aperture (~114 GiB); a load at this
|
||||
# context measured 66.5 GiB of the 116.7 GiB device.
|
||||
#
|
||||
# Spec method: DFlash2 (a block-diffusion drafter), gufo's adaptive block
|
||||
# policy, capped at DRAFT_TOKENS proposals per step. The drafter is the
|
||||
# z-lab Qwen3.8-27B-DFlash2-Q4_K_M and it has a sliding window of 2048
|
||||
# (small draft KV). It is UNCHANGED by the model swap and still lives beside
|
||||
# the previous target, which is why DRAFTER_DIR is its own tunable.
|
||||
#
|
||||
# Sampling: greedy (--temperature 0) server-side, as every frank-large has
|
||||
# served. The harness overrides it per request via extraBody (1.0 / 0.95 /
|
||||
# top-k 20).
|
||||
#
|
||||
# Template: gufo compiles the official Qwen Jinja into the binary and takes
|
||||
# NO --chat-template-file — and it is not a Jinja engine at all
|
||||
# (src/models/qwen/chat_template.cpp is a hand-written C++ renderer), so the
|
||||
# fleet's scripts/qwen-sharp-v22.4.0.jinja cannot be supplied to it. That law
|
||||
# was given up when this endpoint moved to gufo on 2026-09-28, by decision:
|
||||
# the terse system-prompt injection, the in-band <|think_*|> control, the
|
||||
# json|xml tool_call_format switch and the tool-argument truncation are all
|
||||
# absent. gufo's own renderer keeps the effort instruction strings and
|
||||
# preserve_thinking, which are the parts the harness depends on. Accepted
|
||||
# 2026-09-28; recovering the terse prompt from the client side is the
|
||||
# cheapest partial restore.
|
||||
#
|
||||
# Text-only: gufo auto-discovers a mmproj beside the model and this payload
|
||||
# has none (the base repo ships one separately, unused), so the endpoint
|
||||
# stays text-only.
|
||||
#
|
||||
# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides
|
||||
# it; anything not set there falls through to the defaults below.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) ------------------
|
||||
: "${RUNTIME_DIR:=/home/dank/GitPackages/gufo-v0.1.1}"
|
||||
: "${MODEL_DIR:=/home/dank/models/Swift-1.5-Qwen3.8-27B-Uncensored-MTP-UD-Q8_K_XL}"
|
||||
: "${ALIAS:=frank-large}"
|
||||
: "${HOST:=0.0.0.0}"
|
||||
: "${PORT:=7770}"
|
||||
: "${CONTEXT_SIZE:=262144}"
|
||||
: "${SESSIONS:=1}"
|
||||
: "${DRAFT_TOKENS:=7}"
|
||||
: "${DRAFT_POLICY:=adaptive}"
|
||||
|
||||
GUFO_BIN="$RUNTIME_DIR/bin/gufo"
|
||||
|
||||
# The target GGUF and the drafter are named explicitly because they do not
|
||||
# share a directory any more: the target is a different repository's artifact
|
||||
# and keeps its own long filename, while the DFlash2 drafter is unchanged and
|
||||
# still lives beside the previous target. MODEL_FILE/DRAFTER_FILE default to
|
||||
# the deployed payload, so the env file only has to state them when it moves.
|
||||
: "${MODEL_FILE:=$MODEL_DIR/Qwen3.8-27B-UD-Q4_K_XL.gguf}"
|
||||
: "${DRAFTER_DIR:=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL}"
|
||||
: "${DRAFTER_FILE:=$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}"
|
||||
|
||||
if [[ ! -x "$GUFO_BIN" ]]; then
|
||||
echo "runtime not found at $GUFO_BIN" >&2
|
||||
echo "frank-large: build it: bash scripts/build-gufo.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
for f in "$MODEL_FILE" "$DRAFTER_FILE"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "missing asset $f (run the matching scripts/download-frank-large*.sh)" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# --- env -------------------------------------------------------------------
|
||||
# gufo links against the host ROCm through its RPATH (/opt/rocm ->
|
||||
# /etc/alternatives/rocm), so no LD_LIBRARY_PATH is needed. The device is
|
||||
# already gfx1151 — `gufo diagnose` reports it natively — so no
|
||||
# HSA_OVERRIDE_GFX_VERSION either; the override existed for the prebuilt
|
||||
# llama.cpp runtime and can only misreport the arch here.
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
|
||||
# --- the serve command ---------------------------------------------------
|
||||
# Server options precede the subcommand, model options follow it.
|
||||
exec "$GUFO_BIN" serve \
|
||||
--host "$HOST" \
|
||||
--port "$PORT" \
|
||||
--sessions "$SESSIONS" \
|
||||
llm \
|
||||
--model "$MODEL_FILE" \
|
||||
--served-model-name "$ALIAS" \
|
||||
--context "$CONTEXT_SIZE" \
|
||||
--speculative dflash2 \
|
||||
--dflash-model "$DRAFTER_FILE" \
|
||||
--draft-tokens "$DRAFT_TOKENS" \
|
||||
--draft-policy "$DRAFT_POLICY" \
|
||||
--temperature 0 \
|
||||
--think auto \
|
||||
--preserve-thinking on
|
||||
@@ -1,133 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# serve-frank-primary.sh — frank-primary endpoint (tcclaviger RDNA4 vLLM fork,
|
||||
# orcarouter Qwen3.8-Flash-Next-Uncensored-FP8 + MTP4 spec decode, 2x R9700,
|
||||
# dankserver :7770; 2026-09-29 cutover from the R9V radiance fork — git tag
|
||||
# frank-primary-r9v-q8-20260929 is the pre-cutover state).
|
||||
#
|
||||
# The engine is driven by `podman run` DIRECTLY: the image's entrypoint execs
|
||||
# `vllm serve "$@"`, so the run line below IS the engine argv. This replaces
|
||||
# the R9V launcher/shim machinery (that fork needed a docker->podman shim and
|
||||
# a 104-key on-host setup state; this stack is fully expressed by flags, and
|
||||
# every flag here was qualified by boot probe on 2026-09-29 — see
|
||||
# frank-primary.env for the qualification record).
|
||||
#
|
||||
# Shape (2026-10-01 bf16 cutover): MAXLEN 262144 x 3 sessions (786,432
|
||||
# tokens), KV UNQUANTIZED (--kv-cache-dtype auto = bf16; the fp8 q8/q8 shape
|
||||
# is retired to git tag frank-primary-fp8-mtp-20261001), KV_BYTES
|
||||
# 11,047,010,304 PER RANK (the engine measured 798,280 tokens of capacity /
|
||||
# 3.05x concurrency), NO speculative decode, --max-num-batched-tokens 1024
|
||||
# (Frank's lock), MoE gather geometry R4D_LRU_GATHER_CHUNKS=32 /
|
||||
# R4D_LRU_GATHER_LANES=64 (+18.7% decode, grows with depth), prefix caching,
|
||||
# expert prefetch OFF (36% precision measured), TunableOp off.
|
||||
#
|
||||
# The root unit (scripts/frank-primary.service, User=dank,
|
||||
# LimitMEMLOCK=infinity) grants the pin budget for the tiered-expert host
|
||||
# slabs (~20.5 GiB/rank of pinned anonymous memory). A plain shell session
|
||||
# also works, but the unit is the fleet-parity boot-persistent shape.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_PRIMARY_ENV_FILE:-$SCRIPT_DIR/frank-primary.env}"
|
||||
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
echo "serve-frank-primary: env file missing: $ENV_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
|
||||
: "${FRANK_PRIMARY_IMAGE:?FRANK_PRIMARY_IMAGE must name the digest-pinned image}"
|
||||
: "${FRANK_PRIMARY_MODEL_DIR:?FRANK_PRIMARY_MODEL_DIR must name the FP8 payload}"
|
||||
: "${FRANK_PRIMARY_MODEL_SHARDS:?FRANK_PRIMARY_MODEL_SHARDS must give the shard count}"
|
||||
: "${FRANK_PRIMARY_CONTAINER:?FRANK_PRIMARY_CONTAINER must name the container}"
|
||||
: "${FRANK_PRIMARY_SERVED_NAME:?FRANK_PRIMARY_SERVED_NAME must name the alias}"
|
||||
: "${FRANK_PRIMARY_PUBLISH:?FRANK_PRIMARY_PUBLISH must be host:container}"
|
||||
: "${FRANK_PRIMARY_MAXLEN:?FRANK_PRIMARY_MAXLEN must be set}"
|
||||
: "${FRANK_PRIMARY_SEQS:?FRANK_PRIMARY_SEQS must be set}"
|
||||
: "${FRANK_PRIMARY_KV_BYTES:?FRANK_PRIMARY_KV_BYTES must be set (PER RANK)}"
|
||||
: "${FRANK_PRIMARY_KV_DTYPE:?FRANK_PRIMARY_KV_DTYPE must be set (auto = bf16 KV)}"
|
||||
: "${FRANK_PRIMARY_REASONING_PARSER:?FRANK_PRIMARY_REASONING_PARSER must be set (qwen3)}"
|
||||
: "${FRANK_PRIMARY_TOOL_PARSER:?FRANK_PRIMARY_TOOL_PARSER must be set (qwen3_xml)}"
|
||||
: "${FRANK_PRIMARY_CHAT_TEMPLATE_KWARGS:?FRANK_PRIMARY_CHAT_TEMPLATE_KWARGS must be set}"
|
||||
# Capture podman's output BEFORE grepping: under `set -o pipefail`, the
|
||||
# classic `podman images | grep -q` pattern races — grep exits on its first
|
||||
# match, podman dies SIGPIPE (141) writing the remaining lines, and pipefail
|
||||
# turns the 141 into "no match" (the 2026-09-30 deploy refusal).
|
||||
DIGESTS="$(podman images --format '{{.RepoDigests}}')"
|
||||
if ! grep -qF "${FRANK_PRIMARY_IMAGE#*@}" <<<"$DIGESTS"; then
|
||||
echo "serve-frank-primary: pinned image absent from rootless storage: $FRANK_PRIMARY_IMAGE" >&2
|
||||
echo " fetch it: podman pull $FRANK_PRIMARY_IMAGE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$FRANK_PRIMARY_MODEL_DIR/config.json" ]; then
|
||||
echo "serve-frank-primary: payload config missing: $FRANK_PRIMARY_MODEL_DIR/config.json" >&2
|
||||
exit 1
|
||||
fi
|
||||
SHARDS="$(find "$FRANK_PRIMARY_MODEL_DIR" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l)"
|
||||
if [ "$SHARDS" != "$FRANK_PRIMARY_MODEL_SHARDS" ]; then
|
||||
echo "serve-frank-primary: payload shard count wrong: $FRANK_PRIMARY_MODEL_DIR" >&2
|
||||
echo " found $SHARDS safetensors, want $FRANK_PRIMARY_MODEL_SHARDS" >&2
|
||||
echo " re-fetch: scripts/download-frank-primary.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -f "$FRANK_PRIMARY_MODEL_DIR/model.safetensors.index.json" ]; then
|
||||
echo "serve-frank-primary: weight-map index missing: $FRANK_PRIMARY_MODEL_DIR/model.safetensors.index.json" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -e /dev/kfd ] || [ ! -d /dev/dri ]; then
|
||||
echo "serve-frank-primary: GPU device nodes absent (/dev/kfd, /dev/dri)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# A crashed/killed unit can leave the container name registered (systemd
|
||||
# SIGKILLs conmon before cleanup runs). Clear OUR name plus the legacy
|
||||
# identities (the retired R9V container and the ad hoc experiment container
|
||||
# both hold these GPUs); all idempotent.
|
||||
for CTR in $FRANK_PRIMARY_LEGACY_CONTAINERS "$FRANK_PRIMARY_CONTAINER"; do
|
||||
podman rm -f "$CTR" >/dev/null 2>&1 || true
|
||||
done
|
||||
|
||||
LOG=/home/dank/logs/frank_primary.log
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
|
||||
# --device /dev/dri is the WHOLE DIRECTORY (individual renderD nodes enumerate
|
||||
# without the card nodes and HIP refuses contexts — "No CUDA GPUs are
|
||||
# available" while device_count() says 2). --ipc=host for the engine's shared
|
||||
# memory; memlock unbounded for the pinned host slabs.
|
||||
# The entrypoint already execs `vllm serve "$@"`, so no `serve` here.
|
||||
podman run -d --name "$FRANK_PRIMARY_CONTAINER" \
|
||||
--publish "$FRANK_PRIMARY_PUBLISH" \
|
||||
--ipc=host --ulimit memlock=-1:-1 \
|
||||
--cap-add SYS_PTRACE \
|
||||
--device /dev/kfd --device /dev/dri --device /dev/mem \
|
||||
-e HIP_VISIBLE_DEVICES=0,1 -e VLLM_ROCM_USE_AITER=0 \
|
||||
${FRANK_PRIMARY_GATHER_CHUNKS:+-e R4D_LRU_GATHER_CHUNKS="$FRANK_PRIMARY_GATHER_CHUNKS"} \
|
||||
${FRANK_PRIMARY_GATHER_LANES:+-e R4D_LRU_GATHER_LANES="$FRANK_PRIMARY_GATHER_LANES"} \
|
||||
-v "$FRANK_PRIMARY_MODEL_DIR":/models:ro \
|
||||
"$FRANK_PRIMARY_IMAGE" /models \
|
||||
--served-model-name "$FRANK_PRIMARY_SERVED_NAME" --port 7770 \
|
||||
--tensor-parallel-size 2 \
|
||||
${FRANK_PRIMARY_DCP:+--decode-context-parallel-size "$FRANK_PRIMARY_DCP"} \
|
||||
--max-model-len "$FRANK_PRIMARY_MAXLEN" \
|
||||
--max-num-seqs "$FRANK_PRIMARY_SEQS" \
|
||||
${FRANK_PRIMARY_MNBT:+--max-num-batched-tokens "$FRANK_PRIMARY_MNBT"} \
|
||||
${FRANK_PRIMARY_PRECISE_MEMORY:+--expert-precise-memory} \
|
||||
--gpu-memory-utilization "$FRANK_PRIMARY_UTIL" \
|
||||
--kv-cache-memory-bytes "$FRANK_PRIMARY_KV_BYTES" \
|
||||
--kv-cache-dtype "$FRANK_PRIMARY_KV_DTYPE" \
|
||||
--enable-prefix-caching \
|
||||
--enable-auto-tool-choice --tool-call-parser "$FRANK_PRIMARY_TOOL_PARSER" \
|
||||
--reasoning-parser "$FRANK_PRIMARY_REASONING_PARSER" \
|
||||
--default-chat-template-kwargs "$FRANK_PRIMARY_CHAT_TEMPLATE_KWARGS" \
|
||||
--trust-remote-code \
|
||||
> "$LOG" 2>&1
|
||||
|
||||
# From here on, this script's lifetime IS the container's: wait returns the
|
||||
# container's exit code, so a crash is a nonzero ExecStart and the unit's
|
||||
# Restart=on-failure is real. (systemd stop goes through ExecStop in the
|
||||
# unit — podman wait alone cannot stop the container child.)
|
||||
exec podman wait "$FRANK_PRIMARY_CONTAINER"
|
||||
@@ -1,97 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve letechlead/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound as
|
||||
# frank-small on Dankclaw's RTX 3090 via SGLang.
|
||||
#
|
||||
# Topology: the whole checkpoint (12.0 GiB, INT8 W8A16 AutoRound) lives in
|
||||
# VRAM; the 3090's 24 GiB holds weights + a ~11.8 GiB fp8 KV pool at
|
||||
# --mem-fraction-static 0.98. The host CPU only tokenizes.
|
||||
#
|
||||
# Architecture: qwen3_5 hybrid — 32 layers, 8 full-attention + 24 Gated
|
||||
# DeltaNet, native 262144 window, and a vision tower (qwen3_5_vision)
|
||||
# baked into the safetensors: no separate projector file, the image path
|
||||
# is in the checkpoint. The config declares an MTP layer, but the shipped
|
||||
# weights carry none (no mtp.* tensors in the index), so this service runs
|
||||
# WITHOUT speculative decoding — a NEXTN draft would be an uninitialised
|
||||
# 4.24 GiB ghost model drafting at 0% acceptance.
|
||||
#
|
||||
# Context: CONTEXT_LENGTH x MAX_RUNNING slots must fit the fp8 KV pool,
|
||||
# which the engine autofits around the weights AND the DeltaNet ssm state
|
||||
# pool (see frank-small.env). The mamba pool defaults to ~5.1 GB, so it is
|
||||
# pinned (bf16, 32 slots). With the grid at its measured fit the card has
|
||||
# no headroom for prefill CUDA-graph capture, so prefill runs eager
|
||||
# (--disable-prefill-cuda-graph); decode graphs are untouched.
|
||||
#
|
||||
# Sampling: greedy by default (the fleet contract); clients that send their
|
||||
# own temperature get their own values.
|
||||
#
|
||||
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides it.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -------------------
|
||||
: "${SGLANG_BIN:=/home/dank/GitPackages/sglang-cu128/bin/sglang}"
|
||||
: "${MODEL_DIR:=/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound}"
|
||||
: "${ALIAS:=frank-small}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7771}"
|
||||
: "${MEM_FRACTION:=0.98}"
|
||||
: "${CONTEXT_LENGTH:=151552}"
|
||||
: "${MAX_TOTAL_TOKENS:=606208}"
|
||||
: "${MAX_RUNNING:=4}"
|
||||
: "${KV_DTYPE:=fp8_e4m3}"
|
||||
: "${QUANTIZATION:=auto-round}"
|
||||
: "${REASONING_PARSER:=qwen3}"
|
||||
: "${TOOLCALL_PARSER:=qwen3_coder}"
|
||||
: "${MAMBA_SSM_DTYPE:=bfloat16}"
|
||||
: "${MAX_MAMBA_CACHE_SIZE:=32}"
|
||||
|
||||
# Fail loudly rather than half-loading a model into VRAM: a missing file here
|
||||
# is a download that has not finished, and the unit's restart policy would
|
||||
# otherwise turn that into a crash loop that reads like a runtime fault.
|
||||
for path in "$SGLANG_BIN" "$MODEL_DIR/model.safetensors.index.json" \
|
||||
"$(cd -- "$(dirname -- "$SGLANG_BIN")/.." && pwd)/cuda-nvcc/usr/local/cuda-13.0/bin/nvcc"; do
|
||||
[[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
|
||||
done
|
||||
shard_count=$(find "$MODEL_DIR" -maxdepth 1 -name 'model-*.safetensors' | wc -l)
|
||||
if [[ "$shard_count" -ne 8 ]]; then
|
||||
echo "frank-small: expected 8 safetensors shards in $MODEL_DIR, found $shard_count" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
|
||||
# open after boot; make sure it exists before the server needs it.
|
||||
nvidia-modprobe -u -c=0 2>/dev/null || true
|
||||
|
||||
# FlashInfer JITs its attention kernels with nvcc at boot. The host's
|
||||
# /usr/bin/nvcc is CUDA 12.4 and dies on `--compress-mode` (a 12.8+
|
||||
# option), so the venv carries a matching CUDA 13.0 nvcc (extracted from
|
||||
# the cuda-nvcc-13-0 deb); it must shadow the system one on PATH.
|
||||
export PATH="$(cd -- "$(dirname -- "$SGLANG_BIN")/.." && pwd)/cuda-nvcc/usr/local/cuda-13.0/bin:$PATH"
|
||||
|
||||
# Sampling: SGLang has no server-wide sampling defaults; the harness sends
|
||||
# its per-model recipe from models.yml (compat.extraBody), and other clients
|
||||
# get the engine's own defaults.
|
||||
exec "$SGLANG_BIN" serve \
|
||||
--model-path "$MODEL_DIR" \
|
||||
--served-model-name "$ALIAS" \
|
||||
--host "$BIND_ADDR" \
|
||||
--port "$LISTEN_PORT" \
|
||||
--mem-fraction-static "$MEM_FRACTION" \
|
||||
--context-length "$CONTEXT_LENGTH" \
|
||||
--max-running-requests "$MAX_RUNNING" \
|
||||
--kv-cache-dtype "$KV_DTYPE" \
|
||||
--quantization "$QUANTIZATION" \
|
||||
--reasoning-parser "$REASONING_PARSER" \
|
||||
--tool-call-parser "$TOOLCALL_PARSER" \
|
||||
--mamba-ssm-dtype "$MAMBA_SSM_DTYPE" \
|
||||
--max-mamba-cache-size "$MAX_MAMBA_CACHE_SIZE" \
|
||||
--disable-prefill-cuda-graph \
|
||||
--max-total-tokens "$MAX_TOTAL_TOKENS"
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user