Compare commits
130
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
39b8396794 | ||
|
|
a9d45e7012 | ||
|
|
2581b8d4b6 | ||
|
|
09a0251d2d | ||
|
|
066813ea95 | ||
|
|
7daa886dd6 | ||
|
|
bae78e7dae | ||
|
|
a6f55f4cc6 | ||
|
|
a9da6589a6 | ||
|
|
1a6a78ef73 | ||
|
|
c52ba781e3 | ||
|
|
0bf26b0741 | ||
|
|
394fd8535b | ||
|
|
55f87fcdce | ||
|
|
5ced474eb9 | ||
|
|
2992a7ddd6 | ||
|
|
7c77326c33 | ||
|
|
65dec6a80e | ||
|
|
f99b698d25 | ||
|
|
757a90057f | ||
|
|
009e8a885d | ||
|
|
6b677139ad | ||
|
|
184a5c665a | ||
|
|
958b10545e | ||
|
|
c4dca68a03 | ||
|
|
f29dbdf7c7 | ||
|
|
98f11a19c1 | ||
|
|
a30ac9457a | ||
|
|
75695c312a | ||
|
|
c507f91438 | ||
|
|
b175f4afd1 | ||
|
|
971e23a25c | ||
|
|
3ca64e14fc | ||
|
|
f3dfea13ad | ||
|
|
02fa1e22ae | ||
|
|
1aef7418ad | ||
|
|
fa5a444b64 | ||
|
|
90f678582e | ||
|
|
cc0fdd15a1 | ||
|
|
f148ecab4a | ||
|
|
c2f3411342 | ||
|
|
8c5bde7034 | ||
|
|
22a7331100 | ||
|
|
c1e192df00 | ||
|
|
8375567de3 | ||
|
|
79c2e64c2c | ||
|
|
f20da6a6ff | ||
|
|
058ea6cca8 | ||
|
|
3f8531179e | ||
|
|
3c5d77d1fa | ||
|
|
203b0be145 | ||
|
|
74da746c51 | ||
|
|
7066accc25 | ||
|
|
76aaa2543b | ||
|
|
8220030ba1 | ||
|
|
5191c49967 | ||
|
|
f9330fa201 | ||
|
|
4b71c014e9 | ||
|
|
cee90eea52 | ||
|
|
73c37f4301 | ||
|
|
6632d0a2f6 | ||
|
|
caf5e1e0ca | ||
|
|
5278f5be26 | ||
|
|
ec6184de9a | ||
|
|
1d062f06a3 | ||
|
|
b51e078e36 | ||
|
|
68a525740d | ||
|
|
5b7b3ccb90 | ||
|
|
2137080d1c | ||
|
|
f747cb0dab | ||
|
|
db8faa68ff | ||
|
|
cf6c62d44a | ||
|
|
0aafad5f2c | ||
|
|
a3f45ce028 | ||
|
|
45de2fd47f | ||
|
|
fa858f4c10 | ||
|
|
07af1a9366 | ||
|
|
f0d0614ce6 | ||
|
|
9aed9987b6 | ||
|
|
4048dc8630 | ||
|
|
9bc40fd899 | ||
|
|
bb60e8524f | ||
|
|
319dd0f7da | ||
|
|
650dfff5a2 | ||
|
|
de70320cde | ||
|
|
910c54b6ff | ||
|
|
8c887e6fc9 | ||
|
|
49dd2be4cd | ||
|
|
a11f703c49 | ||
|
|
6cdf88f611 | ||
|
|
259fc5505d | ||
|
|
3e46d51df8 | ||
|
|
d062a376b2 | ||
|
|
95219cf4c1 | ||
|
|
fee66187a1 | ||
|
|
0258ae0897 | ||
|
|
9aee00f2b3 | ||
|
|
41d2f56269 | ||
|
|
d8da7a9eb7 | ||
|
|
44e7a8dfaf | ||
|
|
f2575a1018 | ||
|
|
87721ab0fe | ||
|
|
c71a3066c0 | ||
|
|
df6b5bb6e8 | ||
|
|
95923d3a98 | ||
|
|
81e6122891 | ||
|
|
2d89bf97cc | ||
|
|
20d7182f4b | ||
|
|
0f9faf05ea | ||
|
|
31b1ec397b | ||
|
|
7fef3cb907 | ||
|
|
bb3532dfec | ||
|
|
0f7274e55a | ||
|
|
da36fa32cf | ||
|
|
713b126639 | ||
|
|
2d9cecae6d | ||
|
|
fadad18da1 | ||
|
|
306ff32427 | ||
|
|
d39424ffba | ||
|
|
3eb4fa2b62 | ||
|
|
ba67ca5c9e | ||
|
|
8343397584 | ||
|
|
5e166543fa | ||
|
|
812d9ebff5 | ||
|
|
fb1a4fd150 | ||
|
|
90a08ab4b5 | ||
|
|
272acb4c7f | ||
|
|
13112a93f5 | ||
|
|
baf33d016f | ||
|
|
1919ad7f31 |
@@ -1,4 +1,4 @@
|
||||
# Tunables for frank-large (kingjones777/Qwen3.8-Flash-Next-ROCmFP4-
|
||||
# Tunables for frank-dragoon (kingjones777/Qwen3.8-Flash-Next-ROCmFP4-
|
||||
# STRIX-GGUF on the kingjones30/ROCmFPX llama.cpp runtime, Dankclaw,
|
||||
# Strix Halo gfx1151, unified memory).
|
||||
#
|
||||
@@ -56,7 +56,7 @@ CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
|
||||
CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true,"preserve_reasoning":true}'
|
||||
|
||||
# Alias + network.
|
||||
ALIAS=frank-large
|
||||
ALIAS=frank-dragoon
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7770
|
||||
|
||||
+3
-3
@@ -1,5 +1,5 @@
|
||||
[Unit]
|
||||
Description=ROCmFPX (kingjones30 fork) llama.cpp Qwen3.8-Flash-Next ROCmFP4 STRIX 177B MoE (dankclaw, frank-large)
|
||||
Description=ROCmFPX (kingjones30 fork) llama.cpp Qwen3.8-Flash-Next ROCmFP4 STRIX 177B MoE (dankclaw, frank-dragoon)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
@@ -7,8 +7,8 @@ Wants=network-online.target
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
|
||||
Environment="FRANK_DRAGOON_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-dragoon.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-dragoon.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
# is reachable on this port regardless of the boot tag. The NPU is a
|
||||
# single-lane device: generations serialize FIFO, and the first request after a
|
||||
# model swap pays the ~6-11 s load cost. The NPU draws from the unified RAM
|
||||
# pool shared with frank-large (iGPU).
|
||||
# pool shared with frank-dragoon (iGPU).
|
||||
FLM_MODEL=nuextract3:4b
|
||||
FLM_HOST=0.0.0.0
|
||||
FLM_PORT=7771
|
||||
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX-GGUF as
|
||||
# frank-large via the kingjones30/ROCmFPX llama.cpp runtime on
|
||||
# frank-dragoon via the kingjones30/ROCmFPX llama.cpp runtime on
|
||||
# Dankclaw (Strix Halo gfx1151, unified memory).
|
||||
#
|
||||
# Runtime: kingjones30/ROCmFPX (fork of charlie12345/ROCmFPX with the
|
||||
@@ -29,7 +29,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
@@ -44,7 +44,7 @@ fi
|
||||
: "${MMPROJ_PATH:=/home/dank/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST/mmproj/mmproj-Qwen3.8-Flash-Next-f16.gguf}"
|
||||
: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
|
||||
: "${CHAT_TEMPLATE_KWARGS:={}}"
|
||||
: "${ALIAS:=frank-large}"
|
||||
: "${ALIAS:=frank-dragoon}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7770}"
|
||||
: "${CTX_SIZE:=401408}"
|
||||
@@ -4,7 +4,7 @@
|
||||
# Runs `flm serve <tag>` on the XDNA2 NPU (50 TOPS, Dankclaw). The NPU is a
|
||||
# single-lane device: generations serialize in FIFO order, so this is a
|
||||
# background/triage endpoint, not an interactive multi-user chat. The NPU
|
||||
# draws from the unified RAM pool shared with frank-large (the iGPU
|
||||
# draws from the unified RAM pool shared with frank-dragoon (the iGPU
|
||||
# llama.cpp service); see serverAppConfigs-docs/LLM/frank-npu.md
|
||||
# for the memory budget and the model-selection tradeoff.
|
||||
#
|
||||
|
||||
@@ -9,11 +9,12 @@ services run from `/home/dank/serverAppConfigs` on each host (the 2026-08-22
|
||||
merge absorbed the former `vllmScripts` repo, history preserved).
|
||||
|
||||
- **Dankserver** (192.168.0.235) — media/library stack (RomM, Copyparty,
|
||||
Calibre-Web, Jellyfin, Plex) plus the `frank-primary` LLM service.
|
||||
Calibre-Web, Jellyfin, Plex) plus the `frank-archon` LLM service.
|
||||
- **Dankclaw** (192.168.0.143) — AI/search/dev/observability stack (SearXNG
|
||||
via Gluetun, ComfyUI, OpenWebUI, Hermes, Gitea, Grafana, Prometheus, Loki,
|
||||
Alloy, Pi-hole) plus TWO LLM services: `frank-large` (halogen container on
|
||||
the gfx1151 iGPU) and `frank-small` (llama.cpp CUDA on the RTX 3090). Caddy on
|
||||
Alloy, Pi-hole) plus TWO LLM services: `frank-dragoon` (flash-server
|
||||
container on the gfx1151 iGPU) and `frank-zealot` (llama.cpp CUDA on the
|
||||
RTX 3090). Caddy on
|
||||
Dankclaw reverse-proxies both hosts.
|
||||
- **FrankBook-Pro** (192.168.0.112) — macOS mini running the *arr stack
|
||||
(Prowlarr, Sonarr, Radarr, Lidarr, Readarr, qBittorrent) as quadlets inside
|
||||
@@ -24,9 +25,9 @@ merge absorbed the former `vllmScripts` repo, history preserved).
|
||||
|
||||
| Target | Service | Runtime | Model | Port |
|
||||
|--------|---------|---------|-------|------|
|
||||
| Dankserver | `frank-primary` | **FreeToken PR #587** (branch `rocm714_torch214`, head `77a7201`; local image `frank-primary-freetoken:pr587-77a7201` built on the host from `scripts/docker/frank-primary-freetoken.Dockerfile` — digest-pinned rocm/pytorch rocm7.14 torch-2.11.0 base, plain `podman run`, root systemd unit) | `orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8` (~172.8 GiB, 131 shards, modelopt block-FP8, loaded straight from safetensors), **ONE R9700 (second card idle)**, all experts in **pinned host RAM** (~120 GiB) behind a GPU LRU (`--moe-cache-rate 0.10`), PLE table 47.7 GiB pinned (root unit + `LimitMEMLOCK=infinity`), **KV pool `--num-tokens 299968`** (24 KiB/token, multiple-of-64 law of the `qsa_sparse` auto backend), 2 slots, `--max-output-tokens 36864`, **parsers AUTO** (thinking → `reasoning_content`, tools qwen3.8 dialect), vision tower host-streamed. Measured: boot ~2.3 min cold / 92 s warm, decode **17.1 tok/s** single-stream; **TP=2 does not boot on ROCm** (pynccl JIT); `/metrics` 404 (Prometheus scrape reads down). Deploy gate reads the `/health` BODY (`status:"ok"`) — FreeToken 200s while loading. Rollback tag `frank-primary-vllm-shapeG-20261003` |7770|
|
||||
| Dankclaw | `frank-large` | **halogen container** (`ghcr.io/peonist-ai/halogen@sha256:dc0a39a0…`, digest-pinned, plain `podman run` via `scripts/serve-frank-large.sh`, root systemd unit) | `peonist-ai/halogen-qwen3.8-27b` **`qwen3.8-27b-p1w4d-d2.hgn`** (35,865,565,184 B; the `.hgn` format loads in NO other engine — llama.cpp, vLLM and transformers all refuse it), **DFlash2 spec decode (`HALOGEN_DRAFTER=2`)**, **1 slot × 200,700 ctx** — `HALOGEN_KV_SLOTS=1` is load-bearing: batching XOR speculation, slots>1 silently disables the drafter (measured: 8 slots at serial ~6 t/s/stream), d2 KV (2× fp8, 64 KiB/token → 12.2 GiB pool, engine sizes it and prints a boot ledger naming any clamp), **TEXT-ONLY** (the engine ignores the vision encoder — vision is frank-small's job), **censored base** (the ROCmFPX AGENT rollback is the refusal escape hatch), **no server-side sampler: absent temperature is GREEDY**, so the card's thinking recipe (temp 1.0 / top_p 0.95) rides the wire in `models.yml`. Publisher card 31.7 t/s mean across eight text workloads vs the retired ROCmFPX endpoint's measured 27.6–28.1. Prompt over the slot AND `max_tokens` over `HALOGEN_MAX_TOKENS_CAP` are both 400s naming the limit, never truncations. Serve passes NO `--ulimit nofile` (rootless runc cannot raise Dankclaw's 524288 hard limit — see Key Lessons). Cutover 2026-10-05|7770|
|
||||
| Dankclaw | `frank-small` | llama.cpp master, CUDA sm_86 build (root systemd unit, relinked) | bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF **Q6_K** (9B dense qwen3_5 hybrid, 7.26 GiB GGUF, no MTP weights so no spec decode) + the repo's f16 mmproj for **vision**, on the RTX 3090, **2 × 200,704** (pool 401,408 — NOT the native window) with **fully unquantized KV (f16 K + f16 V)** under flash attention, **batch 4096 / ubatch 2048** (doubled 2026-10-05: measured +4.6 % prefill at 8,928 tokens for 616 MiB), 21,846 used / **2,279 free** of 24,576 MiB, 94.4 tok/s decode. Resized 2026-10-03 (3 × 180,224 + q8_0 V → 2 × 200,704 + f16 V → + Q6_K rung: the freed slots paid for exact values AND the higher quant; the rung costs a measured −10 % decode — weight-bandwidth-bound); rollback `MODEL_FILE=…/MiMo-Q5KM/…Q5_K_M.gguf` (buys the 10 % back) or `CTX_SIZE=540672 PARALLEL=3 CACHE_TYPE_V=q8_0` (Q5 only) | 7771 |
|
||||
| Dankserver | `frank-archon` | **radiance** (StillDeadcode, **pin 1.0.13** digest-pinned `stilldeadcode/radiance@sha256:4c3f484564…`, plain `podman run`, root systemd unit) | ukisai/Swift1.5-Qwen3.8-Flash-Next BF16 (335.3 GiB, 397/397 sha256-verified) converted by `scripts/convert-swift15-w4dn8.sh` to `qwen4exp-swift15-w4dn8-i8-hc8m.rad` (**116,416 MiB**): **all 49,152 experts GPTQ w4** (10M-token calibration on engine 1.2.0) + **24 whole bf16-protected experts/layer** (energy-share law), int8 trunk/lm_head, fp8 HC, PLE n-gram fp8 fixed-scale, on **BOTH R9700s TP=2** (`--tp-wire wht6`), experts in a **44 GiB/rank pinned host pool** + VRAM cache (`--placement expert_tiered`, explicit `--vram-weights-mib 25600 --vram-kv-mib 5120` = 30,720 of the **30,969 MiB** the engine's own budget table makes claimable: card 32,624 MiB, settle floor 209 MiB), **47.7 GiB n-gram copy pinned to host RAM** (`--ngram-placement ram`; root unit + `LimitMEMLOCK=infinity`), **fp8 KV** (pool 5,120 MiB = the 3-slot contract + 1.3 % — the only dtype that funds 3 × 200,000 without gutting the slab, since bf16 bills 13.8 KiB/token), **MTP spec decode depth 2**, `--max-num-seqs 3` (the capacity contract is 3 × 200,000; 3 × (1+2) = 9 all-reduce rows, inside the 25-row wht6 exactness bound), `--max-model-len 200000`, MNBT 2048, Froggeric v22.5 template. KLD-gated (mean KL 0.0131, top-1 97.92 % vs BF16); decode ~190 t/s live (gate bar 30.8); prefill is **slab-limited and pays for it** — 22,208 → 3,397, 24,000 → 3,290, 25,600 → **3,584** t/s at 43.5k single-stream, 3-way 197k aggregate 3,051 → **3,589** t/s, with a cliff at ~26 GiB (26,071 → 4,379). Cutover 2026-10-08 from fp8-native (payload on disk = rollback); 3-slot fp8-KV shape 2026-10-13, re-cut twice in the slab's favour (22208/6912 at 4 slots → 24000/5120 → 25600/5120); the 4-slot shape (22208/6912, `--max-num-seqs 4`) is the documented rollback. | 7770 |
|
||||
| Dankclaw | `frank-dragoon` | **halogen-flash-server 0.16.2** (`ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84…`, digest-pinned, plain `podman run` via `scripts/serve-frank-dragoon.sh`, root systemd unit) | Quat3rnion/halogen-swift1.5-qwen3.8-flash-next-v2-abliterated **`qwen38-flash-next-v2-swift15-abliterated.hgn`** (66,687,678,464 B = compacted r2 of the card's 89,775,868,032 B original, ~62 GiB resident; the v2 `.hgn` format loads in NO other engine), **+ REQUIRED `qwen38-flash-next-ngram.hgn`** (51,200,246,144 B, peonist @5cc17cea — read beside the checkpoint) **+ vision tower** (897,916,416 B, @e053f488 — **vision ON**, `image_url` parts only, ≥256 px probes), **MTP spec decode ALWAYS-ON (`HALOGEN_SPEC_ADAPT=0` — the adaptive default disables the drafter per request and HALVES decode)**, MTP head INSIDE the checkpoint (`HALOGEN_MTP_HEAD` unset; no sidecar fetched), **2 slots × 262,144 ctx** (KV pool 524,288 positions ≈ 14.0 GiB at ~28 KiB/position; the 27B slots>1-disables-drafting law does NOT carry over — batching and speculation coexist, the card's quickstart ships 2 with the head), harness window 217,088 = ceiling − 45k estimate headroom, **server-side sampler 1.0/0.95/20** (card recipe; models.yml extraBody matches), coupled cap/timeout 65536/7200, prompt cache ON by default (warm 100k turn ≈ 2 s vs ≈ 88 s cold)|7770|
|
||||
| Dankclaw | `frank-zealot` | llama.cpp master (0.5.0-dev, commit `5fc4f3c`), CUDA sm_86 build (root systemd unit, relinked) | HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive **Q6_K** (7,359,259,008 B; dense `qwen35` hybrid: 32 layers, 8 full-attention + 24 Gated DeltaNet, 4 KV heads × 256 dim, vocab 248,320, native 262,144, **no MTP tensors in the file**) + the publisher's own **BF16 mmproj** (921,704,448 B) for **vision**, on the RTX 3090, **TWO 200,704-token slots** (`--ctx-size 401408 --parallel 2`), **exact f16 K + f16 V** under flash attention (32 KiB/token over the 8 full layers = 12.25 GiB pool), batch 4096 / **ubatch 2048** (measured CUDA optimum), `--log-verbosity 4`, **MTP OFF** (forced: no `nextn` tensors exist). Priced from the MiMo Q6_K boots (same geometry, same card, same shape): 21,276/2,849 at ubatch 1024, 21,846/2,279 at 2048. Cutover 2026-10-07 (late) from Gemma4-26B-A4B Q4_K_M 1×200,704. All three endpoints route harness **compaction** here. | 7771 |
|
||||
|
||||
Retired endpoints (`frank-npu`, and every prior runtime) are in
|
||||
the [runbook index](#runbook-index). The full swap history lives in
|
||||
@@ -65,14 +66,14 @@ All edits flow through git. Never edit files on a remote machine directly.
|
||||
./scripts/deploy.sh dankclaw
|
||||
./scripts/deploy.sh dankserver
|
||||
./scripts/deploy.sh dankclaw --service searxng
|
||||
./scripts/deploy.sh dankserver --service frank-primary
|
||||
./scripts/deploy.sh dankserver --service frank-archon
|
||||
./scripts/deploy.sh dankclaw --dry-run
|
||||
```
|
||||
|
||||
The planner maps changed paths to services; the remote runner restarts only
|
||||
the selected service and records its revision outside the checkout.
|
||||
Documentation-only changes are a no-op. LLM services are selected only by
|
||||
their own files — every endpoint (`frank-large`/`frank-small`/`frank-primary`)
|
||||
their own files — every endpoint (`frank-dragoon`/`frank-zealot`/`frank-archon`)
|
||||
by its unit/env/serve trio in `scripts/` — so
|
||||
unrelated
|
||||
container changes cannot interrupt inference. Set `FRANK_SUDO_PW` for headless
|
||||
@@ -84,7 +85,7 @@ restarts each service whose marker is behind HEAD *and* whose paths changed
|
||||
since that marker — so a backlog accumulated by service-scoped deploys fires
|
||||
all at once, and one failing health gate aborts the run with every later
|
||||
service left pending. Name the service unless a drain is what you want
|
||||
(`./scripts/deploy.sh dankserver --service frank-primary`).
|
||||
(`./scripts/deploy.sh dankserver --service frank-archon`).
|
||||
|
||||
### Deployment completeness (MANDATORY)
|
||||
|
||||
@@ -225,17 +226,27 @@ archive.
|
||||
|
||||
## Benchmarking Policy (MANDATORY)
|
||||
|
||||
**Benchmarking is NOT part of the model-swap procedure** (2026-10-15). A
|
||||
model change is verified by driving the live service — boot ledger,
|
||||
per-request `timings`/`Server-Timing`, one real request per harness role
|
||||
(`serverAppConfigs-docs/LLM/Model Swap and Rollback.md` §6/§8). No bench
|
||||
window is scheduled for a swap, ever. The rest of this policy governs
|
||||
*deliberate* bench windows (flag A/Bs, shape sweeps), which are separate
|
||||
operations on a live host:
|
||||
**STOP the production service BEFORE benchmarking** — these servers serve
|
||||
real users, and a bench beside a live service starves the GPU and corrupts
|
||||
results. Use the runtime's own harness: `llama-bench` for the llama.cpp
|
||||
endpoints; a live llama.cpp server also reports honest per-request numbers in the `/completion` `timings` object (prefill/decode rates, `draft_n`, `draft_n_accepted`), which is how frank-large and frank-small were measured. Rolling your own bench
|
||||
endpoints; a live llama.cpp server also reports honest per-request numbers in the `/completion` `timings` object (prefill/decode rates, `draft_n`, `draft_n_accepted`), which is how frank-dragoon and frank-zealot were measured. Rolling your own bench
|
||||
requires asking first.
|
||||
Scratch ports are not used on LLM hosts — a
|
||||
test process cannot share the single GPU anyway; validate against the service
|
||||
itself.
|
||||
|
||||
(A gufo-era law; frank-large ran ROCmFPX llama.cpp 2026-10-02 → 10-05 and is the halogen container since, but the rollback tags keep it live.) **`gufo bench` cannot rank speculative-decode settings, and will invert the ranking.**
|
||||
Measured 2026-09-28 on frank-large at depth 32768: its corpus
|
||||
(A gufo-era law; frank-dragoon ran ROCmFPX llama.cpp 2026-10-02 → 10-05, the
|
||||
halogen 27B container 2026-10-05 → 10-06, and is flash-server since, but the
|
||||
rollback tags keep it live.) **`gufo bench` cannot rank speculative-decode
|
||||
settings, and will invert the ranking.**
|
||||
Measured 2026-09-28 on frank-dragoon at depth 32768: its corpus
|
||||
reports `acceptance=0.933 ± 0.000` with an **identical trace sha256** across
|
||||
draft widths, i.e. the same highly-predictable text every run, and 93 % of
|
||||
every proposal accepted. It reported 29–37 t/s decode where real traffic
|
||||
@@ -260,12 +271,12 @@ one: `ip neigh show <ip>` from a peer — FAILED means off/NIC dead, while a
|
||||
merely wedged kernel usually still answers ARP.
|
||||
|
||||
1. Stop and mask (self-healing would restart it):
|
||||
- Dankclaw `frank-large` (root unit, halogen podman on the iGPU):
|
||||
`sudo systemctl stop frank-large && sudo systemctl mask frank-large`
|
||||
- Dankclaw `frank-small` (root unit, llama.cpp on the 3090):
|
||||
`sudo systemctl stop frank-small && sudo systemctl mask frank-small`
|
||||
- Dankserver `frank-primary` (root unit, FreeToken on one R9700):
|
||||
`sudo systemctl stop frank-primary && sudo systemctl mask frank-primary`
|
||||
- Dankclaw `frank-dragoon` (root unit, halogen podman on the iGPU):
|
||||
`sudo systemctl stop frank-dragoon && sudo systemctl mask frank-dragoon`
|
||||
- Dankclaw `frank-zealot` (root unit, llama.cpp on the 3090):
|
||||
`sudo systemctl stop frank-zealot && sudo systemctl mask frank-zealot`
|
||||
- Dankserver `frank-archon` (root unit, radiance on both R9700s, TP=2):
|
||||
`sudo systemctl stop frank-archon && sudo systemctl mask frank-archon`
|
||||
2. Bench.
|
||||
NOTE: `systemctl mask` FAILS on a relinked unit (the unit file in
|
||||
`/etc/systemd/system/` is itself a symlink into the checkout — mask
|
||||
@@ -282,9 +293,18 @@ deletes the unit file from `/etc/systemd/system/`, not just the symlink.
|
||||
The harness reaches the fleet through `.omp/agent/models.yml` in the
|
||||
operator's home — `/home/blank/.omp/agent/models.yml` on the dev box, which is
|
||||
where the harness runs; there is no copy on the servers. Each provider is one
|
||||
HOST:PORT; each model `id` is the served alias: `frank-primary` →
|
||||
`192.168.0.235:7770`, `frank-large` → `192.168.0.143:7770`, `frank-small` →
|
||||
`modelRoles`.
|
||||
HOST:PORT; each model `id` is the served alias. **Official model ids are
|
||||
the StarCraft unit names (2026-10-08), on the wire since 2026-10-09, and
|
||||
since 2026-10-10 they are also the SERVICE names:** `frank-archon` →
|
||||
`192.168.0.235:7770` (`FRANK_ARCHON_SERVED_NAME`), `frank-dragoon` →
|
||||
`192.168.0.143:7770` (`HALOGEN_MODEL_ID`), `frank-zealot` →
|
||||
`192.168.0.143:7771` (`ALIAS`). The 2026-10-10 cutover renamed the units,
|
||||
scripts, env files, containers, deploy services and runbooks to match —
|
||||
the three former service names are fully retired and exist nowhere in
|
||||
the repo; historical text in runbooks uses the current names. Verify a
|
||||
rename on `/v1/models`, never
|
||||
on a chat response: radiance echoes the **requested** model name, so a 200
|
||||
proves nothing about the served id (llama.cpp and halogen ignore it).
|
||||
|
||||
Resolution rules: the default model resolves as a `defaultProvider` +
|
||||
`defaultModel` pair (the provider must serve the model); a subagent override
|
||||
@@ -296,17 +316,21 @@ When renaming a service, retiring an endpoint, or moving a model: update the
|
||||
provider block, model `id`, `contextWindow`, `maxTokens`, `input` in
|
||||
`models.yml`, then fix every `modelRoles` entry that named the old endpoint —
|
||||
a role pointing at a provider the file does not define fails at call time. As
|
||||
of 2026-10-05 **two of the three endpoints accept images** (`input: [text,
|
||||
image]`) — `frank-large` went TEXT-ONLY with the halogen swap (the engine
|
||||
ignores the vision encoder; the BF16-mmproj era ended 2026-10-05) — while
|
||||
`frank-primary` takes them natively (FreeToken builds the qwen3.8 vision
|
||||
tower and streams its block weights from host RAM two at a time by default),
|
||||
and `frank-small` through
|
||||
`mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf` passed as `--mmproj` — the MiMo
|
||||
GGUF itself carries no vision tensors. `frank-small`'s `contextWindow` is
|
||||
**200704** (resliced from 180224 on 2026-10-03), the per-slot window, not the
|
||||
model's native 262144: the harness must compact before llama.cpp answers a
|
||||
longer prompt with a 400. Note also
|
||||
of 2026-10-06 **all three endpoints accept images** (`input: [text,
|
||||
image]`) — `frank-dragoon` returned to vision with the flash-server v2 swap
|
||||
(the v2 vision tower loads beside the checkpoint; the text-only halogen
|
||||
27B window closed 2026-10-06) — while
|
||||
`frank-archon` takes them natively (radiance runs the checkpoint's
|
||||
27-layer Qwen-VL tower — patching, pad-token expansion and 3-part rotary
|
||||
are the engine core; `--mm-max-patches` available),
|
||||
and `frank-zealot` through
|
||||
the publisher's own BF16 mmproj passed as `--mmproj` — the model is natively
|
||||
multimodal but the Q6_K GGUF carries no vision tensors (same arrangement the
|
||||
Gemma4 era used). `frank-zealot`'s `contextWindow` is
|
||||
**200704** — the PER-SLOT window (`--ctx-size 401408 --parallel 2` since the
|
||||
2026-10-07 Qwen3.5-9B swap), not the pool and not the model's native 262144:
|
||||
the harness must compact before llama.cpp answers a longer prompt with a 400.
|
||||
Note also
|
||||
that
|
||||
the reasoning channel differs by runtime: llama.cpp AND SGLang (qwen3
|
||||
parser, live-verified) emit `reasoning_content`, vLLM emits `reasoning` —
|
||||
@@ -315,197 +339,309 @@ There is **no audio**
|
||||
input anywhere in the fleet: that capability left with the FLM NPU
|
||||
endpoint on 2026-09-11, returned 2026-09-17 with the Gemma4 projector's
|
||||
audio encoder, and left again with the Gemma4 retirement on 2026-10-01
|
||||
(the MiMo tower is vision-only).
|
||||
(neither the MiMo nor the Qwen3.5 tower carries audio).
|
||||
|
||||
### frank-large — halogen container, Qwen3.8-27B + DFlash2 (Dankclaw, ACTIVE)
|
||||
### frank-dragoon — halogen-flash-server 0.16.2, Qwen3.8-Flash-Next v2 Swift1.5 ABLITERATED (Dankclaw, ACTIVE)
|
||||
|
||||
The **halogen engine** (`ghcr.io/peonist-ai/halogen@sha256:dc0a39a0…`,
|
||||
digest-pinned, published tag 0.1.4) as a podman container driven by
|
||||
`scripts/serve-frank-large.sh` under a **root systemd unit**
|
||||
(`scripts/frank-large.service`, `User=dank`, `Restart=always`, `ExecStop=podman
|
||||
stop -t 120 frank-large-halogen`, the script `exec podman wait` so the restart
|
||||
policy is real) on port 7770 from the Strix Halo iGPU. Engine stdout goes to
|
||||
`/home/dank/logs/frank_large.log` — **the boot ledger (KV pool, drafter state,
|
||||
any clamp) lives there, not in the journal**. Serving
|
||||
`peonist-ai/halogen-qwen3.8-27b`: `qwen3.8-27b-p1w4d-d2.hgn` (35,865,565,184 B
|
||||
≈ 33.4 GiB, sha256 `274c3fc7…`, 1352 tensors; p1 weights / w4 activations /
|
||||
d2 KV) + the flat `tokenizer/` directory (5 files). Cutover 2026-10-05 from the ROCmFPX llama.cpp fork
|
||||
([[frank-large-rocmfpx-q6agent-uncensored-mtp]], measured 27.6–28.1 t/s prose),
|
||||
chosen for the publisher's 31.7 t/s DFlash2 mean.
|
||||
The **flash-server engine** (`ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84…`,
|
||||
digest-pinned, published tag 0.16.2, 3.6 GB image) as a podman container
|
||||
driven by `scripts/serve-frank-dragoon.sh` under a **root systemd unit**
|
||||
(`scripts/frank-dragoon.service`, `User=dank`, `Restart=always`,
|
||||
`ExecStop=podman stop -t 60 frank-dragoon-halogen`, the script runs `podman
|
||||
run` FOREGROUND — the container's exit code is the service's exit so
|
||||
`Restart=always` stays real, and the engine ledger lands on disk) on port
|
||||
7770 from the Strix Halo iGPU. Engine stdout goes to
|
||||
`/home/dank/logs/frank_dragoon.log` — **the boot ledger (weights resident, KV
|
||||
pool, MTP head, prompt cache, any clamp) lives there, not in the journal**.
|
||||
Cutover 2026-10-06 from the halogen 27B-dense era
|
||||
([[frank-dragoon-halogen27b-dflash2]], retired the same window).
|
||||
|
||||
**The format is the lock.** `.hgn` loads in nothing else — the runtime is
|
||||
chosen by the FILE, as ROCmFPX's tensor types were before it. This is NOT the
|
||||
2026-09 `halogen-flash-server` image (different repo, Flash-Next w4b era); the
|
||||
halogen-server repo restarted versioning at 0.1.x.
|
||||
Serving `Quat3rnion/halogen-swift1.5-qwen3.8-flash-next-v2-abliterated`:
|
||||
`qwen38-flash-next-v2-swift15-abliterated.hgn` (66,687,678,464 B ≈ 62.1
|
||||
GiB, sha256 `9c880b8b…` — a compacted r2 of the card's 89,775,868,032 B
|
||||
append-only original; the abliteration workflow appends and the repo's
|
||||
tooling drops dead padding, live tensors byte-identical) + the **REQUIRED**
|
||||
n-gram lookup table `qwen38-flash-next-ngram.hgn` (51,200,246,144 B,
|
||||
peonist @5cc17cea) + the vision tower `qwen38-flash-next-vision.hgn`
|
||||
(897,916,416 B, @e053f488) + the flat `tokenizer/` directory (6 files,
|
||||
fetched fresh from peonist). Cutover 2026-10-06 from the halogen
|
||||
27B-dense shape, chosen by Frank for the ABLITERATED base.
|
||||
|
||||
**Two product regressions, both accepted by Frank before deploy:** the
|
||||
endpoint is **text-only** (README honest limits: the vision encoder is
|
||||
ignored — frank-small owns images now) and the checkpoint is a finetune of the
|
||||
**censored** `Qwen/Qwen3.8-27B` base (refusals can return; the rollback is one
|
||||
commit-revert away with the AGENT payload still on disk).
|
||||
**The format is the lock.** The v2 `.hgn` loads ONLY in the
|
||||
flash-server line — the 27B-era halogen 0.1.4 container refuses it, so the
|
||||
engine moved with the model. The MTP head is INSIDE the checkpoint
|
||||
(`HALOGEN_MTP_HEAD` stays unset; the separate `qwen38-flash-next-mtp.hgn`
|
||||
sidecar is deliberately not fetched).
|
||||
|
||||
**Engine laws that shape the config** (all in `scripts/frank-large.env` with
|
||||
evidence): `HALOGEN_KV_SLOTS=1` — batch XOR speculation, slots>1 turns the
|
||||
drafter off silently. `HALOGEN_SLOT_CTX=200700` — the per-conversation HARD
|
||||
ceiling (400 naming the limit, never truncation); `models.yml`
|
||||
`contextWindow: 200700` must match or the harness discovers the wall as a 400.
|
||||
`HALOGEN_MAX_TOKENS_CAP=65536` + `HALOGEN_QUEUE_TIMEOUT=7200` are a coupled
|
||||
pair; the harness `maxTokens: 57344` stays under the cap (over-cap is a 400
|
||||
naming the limit — README, never a 413). Thinking tokens COUNT against the
|
||||
budget: exhausted mid-thought is `finish_reason: "length"` with empty
|
||||
`content` and the reasoning in `reasoning_content`.
|
||||
`HALOGEN_DRAFTER=2` = DFlash2. **No server-side sampler**: an absent
|
||||
temperature is greedy, so the thinking recipe exists only in `models.yml`
|
||||
`extraBody`. `response_format` is unsupported (no constrained decoding).
|
||||
Thinking lands in **`reasoning_content`** (harness field unchanged); the same
|
||||
Qwen3.8 template rejects `reasoning_effort` outside `xhigh|medium|low`, so the
|
||||
`models.yml` fold stays mandatory.
|
||||
**Product facts vs the 27B era, both regressions FIXED:** the base is
|
||||
**abliterated** (the 27B was a censored-base finetune; refusals were an
|
||||
accepted trade there) and **vision is ON** (the 27B engine silently
|
||||
ignored its encoder — frank-zealot owned images for exactly one day at this
|
||||
endpoint).
|
||||
|
||||
Fetch: `scripts/download-frank-large-halogen.sh` into
|
||||
`/home/dank/models/halogen-qwen3.8-27b/` (`-f` + byte count + LFS sha256 +
|
||||
`--verify-only`, no token on the command line; resume goes against the
|
||||
RESOLVED CDN URL because curl drops `Range` across HF's 302). The deployer
|
||||
and serve script both refuse a missing tokenizer file or **any symlink** —
|
||||
an HF cache snapshot symlinks each entry into `blobs/` and those links
|
||||
dangle inside the container mount.
|
||||
**Engine laws that shape the config** (all in `scripts/frank-dragoon.env`
|
||||
with evidence): `HALOGEN_SPEC_ADAPT=0` is MANDATORY — the default adaptive
|
||||
policy disables the MTP head for whole requests after low acceptance and
|
||||
HALVES decode; the head stays on and temp-1.0 traffic pays some acceptance
|
||||
for it (accepted). `HALOGEN_CTX=262144` is the PER-SLOT hard ceiling (400
|
||||
naming the limit, never truncation) — RAISED from 200192 on 2026-10-07
|
||||
under the 400-zone law: the harness clamps max_tokens to
|
||||
`min(45056, contextWindow − est − 4096)`, its chars/4 est UNDER-counts
|
||||
the rendered prompt by ~3 % (measured: est 152,938 → server 157,388,
|
||||
request 43,158 vs room 42,804), so a harness window == engine ceiling
|
||||
made every request over ~146k est a guaranteed 400. `models.yml`
|
||||
`contextWindow: 217088` leaves the 45k gap that absorbs the estimate
|
||||
error (worst-case request ≈ 219k ≤ 262144); compaction fires at 85 % of
|
||||
the harness window = 184,524 est ≈ 190k measured (Frank's 180k target).
|
||||
`HALOGEN_KV_SLOTS=2` is LEGAL here — the 27B "slots>1 disables the drafter"
|
||||
law does NOT carry over; batching and speculation coexist (the card's
|
||||
quickstart ships 2 slots with the head), so 2 is concurrent capacity.
|
||||
`HALOGEN_KV_POOL_POSITIONS=524288` (explicit 2 × ctx; ~28 KiB/position ⇒
|
||||
14.0 GiB). `HALOGEN_MAX_TOK=8192` is only the per-request DEFAULT (the
|
||||
harness sends at most its `maxTokens` 45056); `HALOGEN_MAX_TOKENS_CAP=65536` +
|
||||
`HALOGEN_QUEUE_TIMEOUT=7200` are a coupled pair (FLAGS.md table).
|
||||
**A server-side sampler EXISTS** (`HALOGEN_TEMPERATURE/TOP_P/TOP_K` =
|
||||
1.0/0.95/20, the card's thinking recipe) — an absent temperature is no
|
||||
longer greedy; `models.yml` extraBody carries the same values, consistent
|
||||
not load-bearing. Prompt cache is ON by default (`PROMPT_CACHE=2`): a warm
|
||||
100k follow-up costs ~2 s vs ~88 s cold — the 27B engine's manual
|
||||
`HALOGEN_CACHE_MB` pin era is over. `response_format` is unsupported.
|
||||
Thinking lands in **`reasoning_content`**; the v2 chat template rejects
|
||||
`reasoning_effort` outside `xhigh|medium|low`, so the `models.yml` fold
|
||||
stays mandatory. Thinking tokens COUNT against the budget (empty content +
|
||||
`finish_reason: "length"` when exhausted mid-thought). Vision: OpenAI
|
||||
`image_url` content parts only; ≥256 px probe images or the model answers
|
||||
"video".
|
||||
|
||||
Runbook: `serverAppConfigs-docs/LLM/frank-large-halogen27b-dflash2.md`.
|
||||
Rollback: revert the swap commit and redeploy — the ROCmFPX payload
|
||||
(`/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/`), the fork build
|
||||
and `scripts/build-rocmfpx.sh` (pin `36e9acd`) all remain on disk, seam-tested.
|
||||
The older gufo tag `frank-large-gufo-q4kxl-20261001` stays a **re-download**
|
||||
rollback (payloads removed 2026-10-03; `scripts/build-gufo.sh` stays).
|
||||
**Memory ledger** (121.2 GiB unified; the referee is the boot ledger):
|
||||
weights ~62 + KV 14.0 + arena ~8 (`3.8 + 8.4 × MAX_TOK/16384`) + vision
|
||||
~2 ≈ 86 GiB engine-side + `HALOGEN_HOST_RESERVE_GIB=20` → ~15 GiB
|
||||
margin over baseline co-tenancy. The 51.2 GB n-gram table rides as mmap'd
|
||||
page cache beside the checkpoint — the card's quickstart arithmetic does
|
||||
not bill it as resident, but READ the boot ledger and re-size if the
|
||||
engine prints a clamp.
|
||||
|
||||
### frank-small — llama.cpp on the RTX 3090 (Dankclaw, ACTIVE)
|
||||
Fetch: `scripts/download-frank-dragoon-flashnext.sh` into
|
||||
`/home/dank/models/halogen-flashnext-v2/` (`-f` + exact byte counts + LFS
|
||||
sha256 + `--verify-only`, no token on the command line; resume goes
|
||||
against the RESOLVED CDN URL because curl drops `Range` across HF's 302).
|
||||
The serve gates mirror the deployer EXACTLY (byte-exact checkpoint +
|
||||
n-gram + vision tower, flat tokenizer with all six files as regular
|
||||
files) — the 2026-10-01 crash-loop lesson.
|
||||
|
||||
**llama.cpp master** (CUDA sm_86 build
|
||||
`/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server`) as a
|
||||
**root systemd unit** (`scripts/frank-small.service`, `relink_unit`ed into
|
||||
Runbook:
|
||||
`serverAppConfigs-docs/LLM/frank-dragoon-flash-server-v2-swift15.md`.
|
||||
Rollback: revert the swap commit and redeploy — the 27B payload
|
||||
(`/home/dank/models/halogen-qwen3.8-27b/`), its fetch script and the
|
||||
halogen 0.1.4 image digest all remain on disk; the ROCmFPX door behind
|
||||
that (`scripts/build-rocmfpx.sh`, pin `36e9acd`) is untouched.
|
||||
|
||||
### frank-zealot — llama.cpp on the RTX 3090 (Dankclaw, ACTIVE)
|
||||
|
||||
**llama.cpp master** (0.5.0-dev, commit `5fc4f3c`; CUDA sm_86 build
|
||||
`/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server`,
|
||||
`GGML_CUDA_FA_ALL_QUANTS=ON`) as a **root systemd unit**
|
||||
(`scripts/frank-zealot.service`, `relink_unit`ed into
|
||||
`/etc/systemd/system/`), serving
|
||||
`bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF` at **Q6_K** (7,793,710,624 B,
|
||||
sha256 `ef96d05a…`) **plus `mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf`**
|
||||
(918,166,048 B, sha256 `ff348f…`), both `/home/dank/models/MiMo-Q6K/` (the
|
||||
projector is a hardlink of the Q5KM copy — the dir is self-contained), on
|
||||
port 7771. The rung went Q5_K_M → Q6_K on 2026-10-03 the same morning as the
|
||||
reslice, funded by the freed slots. The runtime arrived 2026-10-01 night for
|
||||
one reason:
|
||||
**asymmetric KV** — `--cache-type-k f16` + `--cache-type-v q8_0` is the fleet's
|
||||
only per-tensor KV lever, and SGLang's `--kv-cache-dtype` moves K and V
|
||||
together. On 2026-10-03 Frank spent the headroom that lever bought: the reslice
|
||||
from three slots to two made room for **fully unquantized V**
|
||||
(`CACHE_TYPE_V=f16`); both flags stay explicit in the serve script. Vision came
|
||||
2026-10-02 via `--mmproj`: the GGUF carries no `v.*` tensors, so the projector
|
||||
file is the entire vision path (and ~1.4 GiB of the card).
|
||||
`HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive` at **Q6_K**
|
||||
(7,359,259,008 B, sha256 `c0ba7beb…` computed locally — the Hub carries no
|
||||
LFS oid) **plus the publisher's own BF16 mmproj** (921,704,448 B), both
|
||||
`/home/dank/models/Qwen35-9B-HauhauCS-Q6K/`, on port 7771. Cutover
|
||||
2026-10-07 (late) from the Gemma4-26B-A4B Q4_K_M 1×200,704 shape (same day).
|
||||
|
||||
**Architecture: qwen3_5 hybrid** — 32 layers, 8 full-attention + 24 Gated
|
||||
DeltaNet, native 262,144 window. KV exists only in the 8 attention layers
|
||||
(4 KV heads × 256 dim × 2 B ≈ 2 KiB/token/layer, ~16 KiB/token for f16 K+V);
|
||||
the DeltaNet state is a fixed per-slot buffer, so slot count — not context —
|
||||
drives that term.
|
||||
**Architecture: qwen35 dense hybrid** — read from the GGUF's own header, not
|
||||
the model card: 32 layers, `full_attention_interval = 4` ⇒ **8 full-attention
|
||||
layers (3, 7, 11, …, 31) + 24 Gated DeltaNet linear layers**, 16 attention
|
||||
heads / **4 KV heads** × 256 dim, `ssm` conv_kernel 4 / state_size 128 /
|
||||
group_count 16 / time_step_rank 32 / inner_size 4096, native 262,144, vocab
|
||||
248,320. Tensor types are Q6_K × 250 + F32 × 177 — standard quants only, so
|
||||
the stock master build loads it (arch `qwen35` is registered in this build's
|
||||
`src/llama-arch.cpp`). **KV accounting:** the pool scales with the 8 full
|
||||
layers only — 8 × 4 × 256 = 8,192 elements/token per cache, so f16 K + f16 V
|
||||
bills **32 KiB/token** = 12.25 GiB at the 401,408 pool; the 24 DeltaNet
|
||||
layers bill a fixed recurrent state per SLOT, not per token, which is exactly
|
||||
why two 200k slots fit where the 26B MoE fit one.
|
||||
|
||||
**The shape (2026-10-03):** `--ctx-size 401408 --parallel 2` = **TWO
|
||||
conversations of 200,704 tokens each** (llama.cpp divides the pool:
|
||||
`n_ctx_slot = CTX_SIZE / PARALLEL`), `--flash-attn on` (the seam pins it),
|
||||
**f16 K + f16 V**, all layers on the GPU, `--mmproj` for vision, and since
|
||||
2026-10-05 **batch 4096 / ubatch 2048** (doubled; the boot-pair A/B is in the
|
||||
runbook). Measured trade at 8,928 prompt tokens: prefill 4,024 → **4,209 t/s,
|
||||
+4.6 %**, decode flat (91.9–93.0 vs 93.8–94.3 — noise), and a 279-token prompt,
|
||||
which sits below either ubatch, did not move at all. The old "ubatch 2048
|
||||
failed the scratch boot" note belongs to the Q5_K_M predecessor and does NOT
|
||||
generalize: this shape boots 2048 clean. Measured VRAM **21,846 used / 2,279
|
||||
free** of 24,576 MiB against **21,276 / 2,849** for the 1024 arm, reproduced
|
||||
byte-identically the same hour — the doubling costs **616 MiB** and lands at
|
||||
the BOTTOM of the 2–3 GiB floor Frank asked for (`4096/1536` is the middle rung
|
||||
if that margin is ever wanted; non-KV 9,302 MiB, up from 8,732 by exactly the
|
||||
compute buffers; f16 K+V is 32 KiB/token across the 8 attention layers;
|
||||
quote `nvidia-smi`'s `free` field, never `total − used` — the driver holds
|
||||
~450 MiB of reserve the subtraction hides). The 2026-10-02 3 × 180,224 +
|
||||
q8_0-V shape stays one env edit away (`CTX_SIZE=540672 PARALLEL=3
|
||||
CACHE_TYPE_V=q8_0`, and it must pair with Q5_K_M — Q6_K breaks the floor
|
||||
there). Three native windows need 18.4 GiB of KV alone and were never on the
|
||||
table.
|
||||
**The shape (2026-10-07):** `--ctx-size 401408 --parallel 2` = TWO
|
||||
conversations of 200,704 (Frank's cap; the harness `contextWindow` quotes
|
||||
the per-slot number), `--flash-attn on`, **f16 K + f16 V** (exact KV),
|
||||
batch 4096 / **ubatch 2048** (the measured CUDA optimum: +4.6 % prefill,
|
||||
decode flat, 616 MiB — MiMo-era A/B on this card), `--log-verbosity 4`,
|
||||
`--mmproj` for vision, **speculative decoding OFF** (`SPEC_TYPE=none`).
|
||||
|
||||
Verified at deploy (2026-10-03): boot logs `n_slots = 2, n_ctx_slot = 200704`
|
||||
and `loaded multimodal model`; decode **94.44 tok/s** at temp 0 on the probe
|
||||
that measured 104.84 on Q5_K_M — **the Q6_K rung costs a measured −10 %**:
|
||||
decode on this card is weight-bandwidth-bound and Q6_K streams ~10 % more
|
||||
bytes/token; health 200, e2e 33/33; native `tool_calls`; thinking in
|
||||
`reasoning_content`; a **512×512** solid-red PNG answers `Red` through an
|
||||
`image_url` content part (two traps: the legacy top-level `images` field is
|
||||
ignored, and a TINY probe image undersupplies image tokens — a 10×10 PNG made
|
||||
the model claim "it's a video"; probe with ≥256 px). A prompt over the slot
|
||||
is a **400 `exceed_context_size_error`**, which is why `models.yml` sets
|
||||
`contextWindow: 200704`.
|
||||
**Why MTP is off**: it is not a choice — **this Q6_K file carries no
|
||||
`nextn`/`mtp` tensors** (verified from the tensor-name list), and MTP is a
|
||||
property of the file, not of the flags. The publisher card advertises "MTP
|
||||
support" for the base architecture; the quantized export dropped the head.
|
||||
|
||||
Runbook: `serverAppConfigs-docs/LLM/frank-small-llama-mimo-q6k-2x200704.md`
|
||||
(the boot-pair table that prices pool changes lives in the retired
|
||||
[[frank-small-llama-mimo-q5km]]).
|
||||
Rollback: `MODEL_FILE=/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf`
|
||||
buys back 875 MiB and the −10 % decode rate (3,571 free, 104.84 tok/s, both
|
||||
measured); the older shape `CTX_SIZE=540672 PARALLEL=3 CACHE_TYPE_V=q8_0`
|
||||
pairs with Q5 only; SGLang rollback in
|
||||
[[frank-small-sglang-mimo-v26]] (venv on disk at
|
||||
`/home/dank/GitPackages/sglang-cu128/`, INT8 payload staged to
|
||||
`~dank/.AGENTTRASH/20261002-retired-llm-payloads/`).
|
||||
**Measured boots** (nvidia-smi, the referee): the deployed boot is
|
||||
**21,436 used / 2,689 free** (ubatch 2048) — the pricing boots were the
|
||||
MiMo-V2.6-Distill-Qwen-9B pair at this exact geometry (**21,276 / 2,849** at
|
||||
ubatch 1024, **21,846 / 2,279** at 2048), so the 415 MiB lighter weights
|
||||
bought back what the projection predicted. The engine ledger confirms the KV
|
||||
law to the MiB: `CUDA0 KV buffer size = 12544.00 MiB` (`K (f16): 6272.00`,
|
||||
`V (f16): 6272.00`), compute buffers 1,136 MiB on CUDA0 + 848 MiB host, the
|
||||
mmproj 879 MiB + 248 MiB. Decode is ubatch-invariant (Swift-era 3090 law). The
|
||||
Q6_K rung costs ~10 % decode vs Q5_K_M (weight bandwidth; MiMo-measured 94.4
|
||||
vs 104.8 t/s).
|
||||
|
||||
### frank-primary — FreeToken (PR #587 ROCm), Qwen3.8-Flash-Next FP8 (Dankserver, ACTIVE)
|
||||
**Vision**: the publisher's own BF16 mmproj via `--mmproj` (the model is
|
||||
natively multimodal; the Q6_K file carries no vision tensors). Probe traps
|
||||
carry over: ≥256 px probe images or the model answers "video", and only the
|
||||
OpenAI `image_url` content part reaches the model.
|
||||
|
||||
The **FreeToken engine at PR #587 head `77a7201`** (branch
|
||||
`rocm714_torch214`, built on the host into the local image
|
||||
`localhost/frank-primary-freetoken:pr587-77a7201` by
|
||||
`scripts/docker/frank-primary-freetoken.Dockerfile` — the PR's own
|
||||
docs/install_amd.md stack: digest-pinned
|
||||
`rocm/pytorch:rocm7.14_ubuntu24.04_py3.12_pytorch_release_2.11.0` base =
|
||||
torch 2.11.0+rocm7.14/triton 3.7, `--no-build-isolation --no-deps -e .`,
|
||||
curated portable deps, `flashlib==0.3.0 --no-deps`, and a `libhiprtc.so`
|
||||
dev-symlink the pip-ROCm wheel lacks) serving
|
||||
**`orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8`** (~172.8 GiB, 131 shards,
|
||||
standard modelopt block-FP8 — 75,264 `weight_scale_inv` tensors, no
|
||||
`input_scale`, loaded directly from HF safetensors, no FTW conversion) on
|
||||
**ONE R9700 (gfx1201)** — the second card sits idle. Plain `podman run`
|
||||
through `scripts/serve-frank-primary.sh` under the root systemd unit
|
||||
(`frank-primary.service`, `User=dank`, `LimitMEMLOCK=infinity`); the
|
||||
container name is the unit name (`frank-primary`; the vLLM-era
|
||||
`frank-primary-qwen` is in the legacy cleanup list). Cutover 2026-10-05;
|
||||
rollback tag `frank-primary-vllm-shapeG-20261003` (the tcclaviger vLLM trio
|
||||
and its 362-line env lore live in that tag; the tcclaviger image stays in
|
||||
dank's rootless storage).
|
||||
**Compaction hub**: `models.yml` routes `compactionModel` for
|
||||
`frank-archon`, `frank-dragoon` and `frank-zealot` (the 3090 endpoint) here — 200,704 is the
|
||||
only window in the fleet that fits a worst-case compaction input (~170k
|
||||
tokens). With `--parallel 2` a compaction request no longer serializes behind
|
||||
a live generation (the Gemma4 era's `--parallel 1` law).
|
||||
|
||||
**Shape**: all expert weights in **pinned host RAM** (~120 GiB) behind a
|
||||
GPU LRU (`--moe-cache-rate 0.10`, ~12 GiB — the single-card ceiling given
|
||||
the KV contract), the ~47.7 GiB PLE ngram table pinned in host RAM
|
||||
(→ root unit + memlock law), **KV pool `--num-tokens 299968`**
|
||||
(24 KiB/token f16 across the 12 full-attention layers ≈ 7.0 GiB; **must be
|
||||
a multiple of 64** — the auto attention backend is `qsa_sparse`, which
|
||||
overrides page size to 64; 300,000 was refused at first boot), 2 decode
|
||||
slots, `--max-output-tokens 36864` (the harness maxTokens; requests that
|
||||
omit one get this default). **Parsers are AUTO**: thinking lands in
|
||||
`reasoning_content` natively, tool calls in the qwen3.8 dialect — models.yml
|
||||
flips `reasoningContentField` to `reasoning_content` on the harness side.
|
||||
Vision works (models.yml keeps `input: [text, image]`); the vision tower's
|
||||
blocks stream from host RAM two at a time by default.
|
||||
Runbook:
|
||||
`serverAppConfigs-docs/LLM/frank-zealot-llama-qwen35-9b-q6k-2x200704.md`.
|
||||
Rollback: revert the swap commit — the Gemma4 payload
|
||||
(`/home/dank/models/Gemma4-26B-A4B/`), the MiMo Q6_K payload
|
||||
(`/home/dank/models/MiMo-Q6K/`), the Swift payload
|
||||
(`/home/dank/models/Swift15-UncIQ3XXS/`) and the Ornith Q8_0 payload
|
||||
(`/home/dank/models/Ornith-Q8/`) all remain on disk.
|
||||
|
||||
**Measured at cutover**: boot ~2.3 min cold-ish / 92 s warm → /health
|
||||
status ok; decode **17.1 tok/s** single-stream temp 0 (the vLLM incumbent
|
||||
measured ~30 tok/s on TWO cards; the KV contract caps the single-card LRU,
|
||||
and **TP=2 exists but does not boot on ROCm** — the TP allreduce JIT
|
||||
(`pynccl.cu`) can't resolve `cuda_runtime.h` for gfx1201; probed
|
||||
2026-10-06, signature in the runbook). `/metrics` is a 404 — the Dankclaw
|
||||
Prometheus llm-fleet scrape of :7770 reads down until repointed.
|
||||
`ft ctl cache --kv/--moe` resizes pools live (grow only; shrink needs a
|
||||
restart); `ft ctl stats` reports occupancy.
|
||||
|
||||
**Deploy gate law (changed for FreeToken)**: `/health` returns **200 with
|
||||
`{"status":"loading"}`** while the engine loads — the FastAPI front end
|
||||
binds before the scheduler is up, so a bare `curl` false-greens (measured:
|
||||
"healthy after 6 s" mid-page-in). The deployer loops until the body says
|
||||
`"status":"ok"`; the MainPID stray-port guard is unchanged. The image gate
|
||||
is `podman image exists` (a local build carries no RepoDigests — the old
|
||||
digest grep refused exactly this shape; pinning lives in the Dockerfile).
|
||||
### frank-archon — Radiance (StillDeadcode), Swift1.5-Qwen3.8-Flash-Next GPTQ w4, TP=2 (Dankserver, ACTIVE)
|
||||
|
||||
Runbook: `serverAppConfigs-docs/LLM/frank-primary-freetoken-pr587.md`.
|
||||
The **radiance engine** (StillDeadcode, Apache-2.0; image
|
||||
`stilldeadcode/radiance@sha256:4c3f484564…` digest-pinned, **pin 1.0.13**
|
||||
for serve/convert/KLD; `libr4d` gfx1201 kernels, plain `podman run`, root
|
||||
systemd unit, container name = unit name) serving
|
||||
**ukisai/Swift1.5-Qwen3.8-Flash-Next (BF16, 335.3 GiB, 131 shards,
|
||||
397/397 sha256-verified)** converted by `scripts/convert-swift15-w4dn8.sh`
|
||||
with recipe `scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe` to
|
||||
`qwen4exp-swift15-w4dn8-i8-hc8m.rad` (~114 GiB): **all 49,152 experts GPTQ
|
||||
w4** (act-order, damp 0.05, 10,000,000-token calibration captured on engine
|
||||
**1.2.0** via `RADIANCE_CALIB_DIR`/`RADIANCE_CALIB_TOKENS` — the only pin
|
||||
with the token knob; the RADGRAM2 grams are format-identical across pins and
|
||||
cross the boundary by design), **24 whole experts per layer cast bf16** (the
|
||||
PROTECTED set, derived from the calibration grams' energy share),
|
||||
**int8 trunk/lm_head**, **fp8 HC mixtures** (`--hc-mix-dtype`), PLE n-gram
|
||||
table at fp8 fixed scale (47.7 GiB in the container, from 95.4 GiB bf16).
|
||||
The plugin's **one-table-one-encoding law** (measured at rad-convert
|
||||
declare): one expert table takes ONE encoding; only whole bf16 experts are
|
||||
legal beside it — no w4/fp8 split of the down/gate_up pair (24-value
|
||||
`rad_arch_quant_desc` array, one per layer). KLD gate vs the BF16
|
||||
reference (recorded on the same 1.0.13 pin) is the precision referee;
|
||||
the throughput gate (decode ≥ 30.8 t/s) was passed at 4.5× the bar
|
||||
(122–173 t/s at 44.5k depth). Measured decode/prefill in the runbook.
|
||||
Rollback: revert the swap commit — the fp8-native `.rad`
|
||||
(`qwen4exp-fp8-native.rad`) and every prior payload remain on disk.
|
||||
|
||||
**Shape (both R9700s, TP=2)**: `--tp-wire wht6`, `--max-num-seqs 3`,
|
||||
`--max-model-len 200000` (the harness contextWindow must quote this),
|
||||
`--placement expert_tiered --host-pool-mib 44928` + **explicit VRAM
|
||||
partition `--vram-weights-mib 25600 --vram-kv-mib 5120`** (the vram-pair
|
||||
replaced `--expert-vs-cache-ratio`; the two are ONE budget —
|
||||
`weights + kv ≤ 30,969 MiB`, derived from the engine's own budget table: card
|
||||
32,624 MiB, the pair at 29,120 settling 2,058 MiB free against the engine's
|
||||
209.19 MiB floor (`--gpu-headroom-mib 96` + 113.19 static), so 1,849 MiB is
|
||||
claimable and the old 29,120 bound was my hand-picked 1 GiB reserve, not a
|
||||
hardware wall), `--kv-cache-dtype` **`fp8` — quantized KV, restored 2026-10-13
|
||||
because it is the only dtype that funds the slot count at this window**,
|
||||
`--num-speculative-tokens 2`, `--max-num-batched-tokens 2048`, prefix cache
|
||||
8 GiB pinned host + 64 GiB disk at `/home/dank/radiance-state`,
|
||||
**`--ngram-placement ram`**, Froggeric v22.5 chat template vendored
|
||||
(`scripts/qwen38-froggeric-v22.5.jinja`, `--override-chat-template`). Boot
|
||||
ledger (2026-10-13, pair 25600/5120): `kv cache 5.00 GiB … backs 3 sequences
|
||||
all at 200000 tokens at once, which is the worst case` — `kv_attn` page
|
||||
26.00 KiB (fp8) at 607,740 tokens (the sizing law predicted 608,027),
|
||||
`kv_qsa_bkey` 3.25 KiB at the same token count, `kv_gdn_state` 54 MiB × 6 slots
|
||||
(3 sequences × 2 copies) = 324 MiB; the sampler prints **9 rows**
|
||||
(3 × (1+2), inside the 25-row `wht6` exactness bound); the slab filled to
|
||||
563 KiB of slack with **17,548 slots** (16,235 at 24,000, 14,753 at 22,208 —
|
||||
every 1,792 MiB buys ~1,300 resident experts, and the pinned host pool hands
|
||||
the same experts back: 8,893 → 7,580 slots); `unclaimed 477.30 MiB`, cards
|
||||
settled ~336 MiB free against the engine's 209.19 MiB floor, KV lending
|
||||
4.22 GiB to the expert cache. Prefill tracks this slab only up to a
|
||||
**threshold**: 22,208 → 3,397 and 24,000 → 3,290 t/s at 43.5k are one band
|
||||
(±3 %), and the 25,600 arm measured **3,290** — the third step bought nothing
|
||||
(the 3,584 arm did not reproduce at 3 slots); the cliff is at ~26 GiB
|
||||
(26,071 → 4,379 on 2026-10-11's arm), so below that the slab is a threshold,
|
||||
not a slope.
|
||||
|
||||
**Decode ladder (2026-10-13, 512 tokens, greedy, this shape)**: 99.0 t/s
|
||||
single-stream at 43.5k, 93.3 at 197k — decode is nearly window-free (−6 % for
|
||||
4.5× depth); **3 concurrent streams give 113.9/43.3/10.9 t/s = 18.4 t/s
|
||||
aggregate**, so the slot count is paid in decode concurrency (a fused-step
|
||||
lottery), not in prefill; leg D of `probe-frank-archon-kv.py` measures it.
|
||||
|
||||
Shape notes carried from the fp8-native era and unchanged: wht6 gains,
|
||||
MNDT 2048 law (2560 collapses decode to 2.6–7.7 t/s), the 64 GiB disk
|
||||
prefix cache, and the rule that a prompt over `--max-model-len` is
|
||||
refused, never truncated. What changed: the fp8-native era's NVMe expert
|
||||
streaming (the 3.5 GB/s wall) is gone — the w4 resident set fits the
|
||||
host pool, so decode ran 122–173 t/s in the campaign arms and ~190 t/s
|
||||
on the fp8 shape (512-token streamed probe). The campaign also
|
||||
measured the int8-KV + spec-3 arm (136.0 t/s median at 44.5k depth,
|
||||
accept 0.7246).
|
||||
|
||||
**The KV dtype law (measured 2026-10-11, applied at 4 slots 2026-10-13)**:
|
||||
`--kv-cache-dtype` takes ONLY `bf16|fp8` — one dtype for the whole cache, so
|
||||
llama.cpp's exact-K/quantized-V pair (what frank-zealot runs) has NO radiance
|
||||
equivalent; the rung under bf16 is fp8 on both halves. Measured per rank from
|
||||
the two boot ledgers: `kv_attn` (13 layers, 4-token blocks) pages 26.00 → 52.00
|
||||
KiB, while `kv_qsa_bkey` (13 layers) stays 3.25 KiB at BOTH dtypes (the QSA
|
||||
block key is an index, not attention state) — so a token costs 7.3125 KiB at fp8
|
||||
and 13.8125 KiB at bf16: **1.89×, not 2×**. Pool sizing must also pay the
|
||||
dtype-independent terms that live in the same pool (~110 MiB a sequence of
|
||||
recurrent state + 448 MiB of prefix checkpoints):
|
||||
`(vram_kv − 110·seqs − 448) MiB / rate` — bf16 at 6,912 gives 462,946 where the
|
||||
engine reported 462,752 (0.04 % page rounding); fp8 at 6,912 with four slots
|
||||
gives 843,566, i.e. the deployed capacity contract 4 × 200,000. **That is why
|
||||
the dtype is fp8: the pool must back `seqs × max-model-len` as a hard limit, and
|
||||
bf16 at four slots costs 11,679 MiB — the slab would fall to 17,441 MiB (−33 %),
|
||||
where fp8 leaves it at 22,208.** The cost is paid twice on this engine, and
|
||||
half-refunded: idle KV VRAM is also the expert cache's spillover, and the bigger
|
||||
pool *lends* 6.08 GiB to the slab where the 3,543 MiB fp8 pool lent 2.69 GiB.
|
||||
`--max-num-seqs` rides 4 × (1 + spec) = 12 fused-all-reduce rows, under the
|
||||
25-row bound where wht6 is bit-exact (the "seqs 2 cannot boot" law belongs to
|
||||
the retired tcclaviger fork's torch.compile, not radiance). Disk-tier sizing
|
||||
law: the engine forces ~9.6 GiB of disk-tier weight buffers + arena onto the
|
||||
weights side, so `weights + kv` is capped near 29.6 GiB/card; boot with the
|
||||
pair, never `--kv-cache-size-mib`.
|
||||
|
||||
Shape notes carried from the fp8-native era and unchanged: wht6 gains,
|
||||
MNDT 2048 law (2560 collapses decode to 2.6–7.7 t/s), the 64 GiB disk
|
||||
prefix cache, and the rule that a prompt over `--max-model-len` is
|
||||
refused, never truncated. What changed: the fp8-native era's NVMe expert
|
||||
streaming (the 3.5 GB/s wall) is gone — the w4 resident set fits the
|
||||
host pool, so decode ran 122–173 t/s in the campaign arms and ~190 t/s
|
||||
on the fp8 shape (512-token streamed probe). The campaign also
|
||||
measured the int8-KV + spec-3 arm (136.0 t/s median at 44.5k depth,
|
||||
accept 0.7246).
|
||||
|
||||
**The pinned-RAM ledger is why this is a root unit**: 2 × 44 GiB expert
|
||||
pools + the 47.7 GiB n-gram copy (`--ngram-placement ram` is pinned RAM
|
||||
the card reads) + 8 GiB prefix host ≈ **147 GiB** charged against
|
||||
`RLIMIT_MEMLOCK` — the TTM floor is set just above it.
|
||||
`ttm.pages_limit`/`pages_max` cap GPU-mappable pinned pages, and **the knob is
|
||||
runtime state whose default is half of RAM**: `scripts/ttm-preflight.sh` (sourced
|
||||
by the serve script) therefore RAISES it to `FRANK_ARCHON_TTM_TARGET_MIB=204800`
|
||||
before refusing, because the 2026-10-12 power failure reset it to 125,988 MiB
|
||||
and the preflight re-refused 6,919 times. A manual sysfs write in the stop
|
||||
window is a convenience, not the repair; kernel cmdline persistence
|
||||
(`ttm.pages_limit=52428800`) is the belt to it.
|
||||
|
||||
**Contract facts** (source-verified): the HTTP listener binds AFTER the
|
||||
full load, so a bare `curl /health` gate is honest (`/health` == `/ping`,
|
||||
200 serving / 503 stopping); sampling params temperature…preserve_thinking
|
||||
accepted, **unknown params 400** (models.yml's extraBody set is fully
|
||||
inside it); `response_format` supported; `logprobs` REFUSED; reasoning in
|
||||
`reasoning_content` and tool calls in the qwen3.8 dialect via the auto
|
||||
chat format — no parser flags; vision native (the 27-layer Qwen-VL tower,
|
||||
`--mm-max-patches`); `/metrics` carries vLLM-compatible names +
|
||||
`radiance:*` families — the Dankclaw Prometheus scrape is re-added.
|
||||
|
||||
Runbook: `serverAppConfigs-docs/LLM/frank-archon-radiance-swift15-w4dn8.md`.
|
||||
Retired fp8-native era: `serverAppConfigs-docs/LLM/frank-archon-radiance-qwen38-native-fp8.md`.
|
||||
Retired FreeToken era: `serverAppConfigs-docs/LLM/frank-archon-freetoken-pr587.md`.
|
||||
Retired vLLM era: `serverAppConfigs-docs/LLM/dankserver-qwen38-fp8-expert-offload-tuning.md`.
|
||||
Retired GLM experiment: `serverAppConfigs-docs/LLM/frank-primary-glm53-flash-w4a16.md`.
|
||||
Retired GLM experiment: `serverAppConfigs-docs/LLM/frank-archon-glm53-flash-w4a16.md`.
|
||||
|
||||
## Key Lessons Learned
|
||||
Durable operational lessons from the fleet's history. Full detail in the
|
||||
@@ -667,11 +803,22 @@ linked runbooks.
|
||||
- Swap resize ordering: create and `swapon` the replacement BEFORE `swapoff`
|
||||
of the old one, or live pages must land in RAM that `MemAvailable`
|
||||
overstates. Dankclaw swap (64 GiB at `/swap.img`, fstab-driven) protects
|
||||
the co-tenants, not frank-large — pinned weights don't swap, and paging
|
||||
the co-tenants, not frank-dragoon — pinned weights don't swap, and paging
|
||||
the KV pool is worse than the OOM it prevents.
|
||||
- Host swap state is invisible to `deploy.sh` (no repo file) — check
|
||||
`swapon --show` before reasoning about headroom.
|
||||
- **Silent RAM corruption outranks every software theory.** frank-primary's
|
||||
- **A sysfs raise is a countdown to the next reboot.** `ttm.pages_limit` defaults
|
||||
to half of RAM, so the manual raise to 200 GiB (2026-10-05) vanished with the
|
||||
2026-10-12 23:17 POWER FAILURE on dankserver: the cap fell to 125,988 MiB,
|
||||
frank-archon's pinned-ledger preflight (floor 150,000 MiB) refused the start,
|
||||
and `Restart=always` re-ran that refusal **6,919 times in ~20 h** — a unit in
|
||||
`activating (auto-restart)` that exits 1 in 50 ms with no container and no
|
||||
engine log is a PREFLIGHT refusal, not an engine fault. Repo law now:
|
||||
`scripts/ttm-preflight.sh` (sourced by `serve-frank-archon.sh`) raises the knob
|
||||
to `FRANK_ARCHON_TTM_TARGET_MIB` itself, re-reads, and refuses by name only
|
||||
after a write the kernel rejected. Never prescribe a manual sysfs write as the
|
||||
recovery procedure for a service that can boot before a human reaches it.
|
||||
- **Silent RAM corruption outranks every software theory.** frank-archon's
|
||||
~40k-token output corruption (2026-10-04/05, repetition/garbage in long
|
||||
agent conversations) was marginal DDR5 at 4800 MT/s on four DIMMs, fixed
|
||||
by 4600. Consumer non-ECC memory raises **no MCE** for a silent flip, and
|
||||
@@ -694,7 +841,7 @@ linked runbooks.
|
||||
`LimitMEMLOCK=infinity` (systemd is privileged; the value flows through
|
||||
podman to runc without any raise). Upstream halogen's own run lines pass
|
||||
NO `--ulimit` at all — copy vendor run lines before adding flags
|
||||
(measured 2026-10-05 on the frank-large cutover).
|
||||
(measured 2026-10-05 on the frank-dragoon cutover).
|
||||
- **Kairic Edge** (if ever re-deployed): the pinned CK commit needs
|
||||
`third_party/composable_kernel/include/ck/config.h` generated manually
|
||||
(untracked; `git clean` removes it), and its greedy-argmax fastpath rejects
|
||||
@@ -720,7 +867,7 @@ linked runbooks.
|
||||
121 GiB box, pushed `available` to 3 GiB, and every later rate was suspect.
|
||||
Record the child pid and kill it in a `finally`, then prove `free` came back
|
||||
before believing a number. Kill by pid — never `pkill -f llama-server`,
|
||||
which also hits frank-small.
|
||||
which also hits frank-zealot.
|
||||
|
||||
**llama.cpp serving**
|
||||
|
||||
@@ -744,7 +891,7 @@ linked runbooks.
|
||||
builds its context against whatever model the draft context holds. Two
|
||||
context-backed types cannot share one process, and MTP's head only loads
|
||||
when MTP **is** the draft model. **A LEGAL chain can still lose on a
|
||||
recurrent model:** `draft-mtp,ngram-mod` on frank-large's qwen35 hybrid
|
||||
recurrent model:** `draft-mtp,ngram-mod` on frank-dragoon's qwen35 hybrid
|
||||
measured −15 % prose / −7 % block-copy (2026-10-03) — adding the ngram layer
|
||||
logs `recurrent state rollback is not compatible with 'ngram-mod' -
|
||||
disabling rollback support`, and MTP verification depends on that rollback,
|
||||
@@ -786,7 +933,7 @@ linked runbooks.
|
||||
free. That was a CONTIGUITY failure, not a total-bytes one, and no totals
|
||||
model — the sweep's or the seam's — can see it.)
|
||||
- **On the CUDA 3090 the same knob has NO interior optimum up to 2048, so
|
||||
frank-small ships 4096/2048.** Measured 2026-10-05 by boot-pair A/B on the
|
||||
frank-zealot ships 4096/2048.** Measured 2026-10-05 by boot-pair A/B on the
|
||||
live service: at 8,928 prompt tokens ubatch 1024 gives 3,972 / 4,077 t/s and
|
||||
ubatch 2048 gives 4,207 / 4,212 — **+4.6 %** — for **616 MiB** (free 2,849 →
|
||||
2,279). Decode did not move (91.9–93.0 vs 93.8–94.3, i.e. noise), and a
|
||||
@@ -795,9 +942,9 @@ linked runbooks.
|
||||
than the ubatch cannot see it. The logical `--batch-size` was again free. And
|
||||
the predecessor's "ubatch 2048 failed the scratch boot" was a shape
|
||||
observation, not a law: the same file boots 2048 clean at 2 × 200,704. Bench
|
||||
it by restarting the SERVICE through a transient `FRANK_SMALL_ENV_FILE`
|
||||
it by restarting the SERVICE through a transient `FRANK_ZEALOT_ENV_FILE`
|
||||
drop-in, never a scratch port — one card holds one server. See
|
||||
[[frank-small-llama-mimo-q6k-2x200704]].
|
||||
[[frank-zealot-llama-mimo-q6k-2x200704]].
|
||||
- **One `-ts` step is one LAYER, and a hybrid stack's layers are not the same
|
||||
size.** qwen35 is 16 x (3 Gated DeltaNet + 1 Gated Attention), so a step is
|
||||
worth ~0.43 GiB on a DeltaNet boundary and ~1.43 GiB on a full-attention one
|
||||
@@ -819,7 +966,7 @@ linked runbooks.
|
||||
CTX_SIZE / PARALLEL`, and llama.cpp refuses a prompt longer than the SLOT
|
||||
(`400 exceed_context_size_error: request (184001 tokens) exceeds the
|
||||
available context size (180224 tokens)`) even when the pool has room. So
|
||||
frank-small's 2 → 3 reslice moved each conversation from the model's native
|
||||
frank-zealot's 2 → 3 reslice moved each conversation from the model's native
|
||||
262,144 down to 180,224, and `models.yml` `contextWindow` had to follow or
|
||||
the harness would discover the ceiling as a 400 on a long conversation. Side
|
||||
effect worth knowing: the shorter slot made decode ~5 % FASTER (106.8 vs
|
||||
@@ -830,9 +977,9 @@ linked runbooks.
|
||||
day the freed space also bought the Q6_K rung (now 94.4 tok/s — the rung
|
||||
itself is a further measured −10 %, weight-bandwidth, not slot geometry).
|
||||
All moves measured; both slot directions cost or buy window, never pool.
|
||||
See [[frank-small-llama-mimo-q6k-2x200704]].
|
||||
See [[frank-zealot-llama-mimo-q6k-2x200704]].
|
||||
- **Price a VRAM budget from two boots, never from a formula — and an mmproj
|
||||
costs ~1.5x its file size.** The frank-small 3090 fit was solved from two
|
||||
costs ~1.5x its file size.** The frank-zealot 3090 fit was solved from two
|
||||
measurements: 2 × 262,144 with no projector = 20,190 MiB used (KV 12,800 ⇒
|
||||
**non-KV 7,390**), 3 × 196,608 with the f16 projector = 22,972 used (KV
|
||||
14,113 ⇒ **non-KV 8,859**). The difference is the vision tower: a 918 MB file
|
||||
@@ -840,24 +987,24 @@ linked runbooks.
|
||||
ride with it. Subtracting the two rows is exact; extrapolating weights+KV+
|
||||
"a bit for buffers" is what leaves 1.1 GiB free when you meant 2.7. Then
|
||||
solve `KV ≤ card − non-KV − headroom` and round to a multiple of
|
||||
`PARALLEL × 4096`. Recorded in `scripts/frank-small.env` so the next resize
|
||||
`PARALLEL × 4096`. Recorded in `scripts/frank-zealot.env` so the next resize
|
||||
starts from measurement.
|
||||
- **`images` and `image_url` are NOT interchangeable: only the content part
|
||||
reaches the model.** `--mmproj` loads, `usage.prompt_tokens` rises, and the
|
||||
model still invents an answer if the client used the legacy top-level
|
||||
`images` field on `/v1/chat/completions` — frank-small answered "Brown." for
|
||||
`images` field on `/v1/chat/completions` — frank-zealot answered "Brown." for
|
||||
a solid-red PNG that way and "Red." through the OpenAI `image_url` part. The
|
||||
failure is silent because a hallucinated color is a plausible answer. Probe
|
||||
vision with a solid color and expect an exact word.
|
||||
- **`--jinja` is what `tools` needs.** On 2026-09-16 a subagent died with
|
||||
`500 tools param requires --jinja flag`, from the ik_llama.cpp runtime that
|
||||
then served frank-large: every tool-using agent on that endpoint was broken
|
||||
then served frank-dragoon: every tool-using agent on that endpoint was broken
|
||||
and the failure read like a model fault. Mainline llama.cpp 0.4.1 defaults
|
||||
`--jinja` on and the serve script pins it; halogen also accepts `tools` and
|
||||
returns `tool_calls`. Check tool support before routing a role at an endpoint.
|
||||
- **Quantized KV on CUDA needs `GGML_CUDA_FA_ALL_QUANTS=ON`** — the same
|
||||
option and the same ~12x prefill collapse described in the ROCm note above.
|
||||
f16 KV needs nothing; frank-small ran f16 for that reason until
|
||||
f16 KV needs nothing; frank-zealot ran f16 for that reason until
|
||||
2026-09-17, when `build-cuda` was rebuilt with the option ON so the
|
||||
endpoint could take q8_0 KV.
|
||||
- **llama-bench separates a LIST with `/` and SWEEP values with `,`.** Both
|
||||
@@ -875,7 +1022,7 @@ linked runbooks.
|
||||
raises on anything else — HTTP 500, not a clamp — so the harness's levels
|
||||
must be folded onto that set in `models.yml` (`compat.reasoningEffortMap`).
|
||||
A role whose level is unmapped fails on its FIRST request, and the failure
|
||||
names Jinja, not the model: every subagent on `frank-small` died with
|
||||
names Jinja, not the model: every subagent on `frank-zealot` died with
|
||||
`Unexpected reasoning effort high` until `high: xhigh` was added on
|
||||
2026-09-16. Halogen accepts the wider set, so the two entries legitimately
|
||||
differ.
|
||||
@@ -895,7 +1042,7 @@ linked runbooks.
|
||||
times (`RuntimeError: No HIP GPUs are available`).
|
||||
- SearXNG outbound traffic must stay on the Gluetun tunnel; do not move its
|
||||
ports off the gluetun service.
|
||||
- `amd_iommu=off` on Dankclaw (bought 13–16 % prefill for frank-large,
|
||||
- `amd_iommu=off` on Dankclaw (bought 13–16 % prefill for frank-dragoon,
|
||||
2026-09-11) permanently retired the XDNA2 NPU: the driver's SVA/PASID bind
|
||||
needs an enabled IOMMU (`/dev/accel/accel0` fails `err=-19`). Revert path
|
||||
in `serverAppConfigs-docs/LLM/dankclaw-iommu-translation-off.md`.
|
||||
@@ -949,9 +1096,9 @@ Active:
|
||||
|
||||
| Service | Runbook |
|
||||
|---------|---------|
|
||||
| frank-large (halogen container, Qwen3.8-27B .hgn + DFlash2, 1×200,700, text-only, Dankclaw) | `serverAppConfigs-docs/LLM/frank-large-halogen27b-dflash2.md` |
|
||||
| frank-small (llama.cpp CUDA, MiMo-V2.6-Distill-Qwen-9B Q6_K, 2×200,704 f16 KV, Dankclaw) | `serverAppConfigs-docs/LLM/frank-small-llama-mimo-q6k-2x200704.md` |
|
||||
| frank-primary (FreeToken PR #587 ROCm, Qwen3.8-Flash-Next FP8, 1×R9700, Dankserver) | `serverAppConfigs-docs/LLM/frank-primary-freetoken-pr587.md` |
|
||||
| frank-dragoon (flash-server 0.16.2, abliterated Swift1.5 v2 .hgn + MTP always-on, 2×262,144, vision ON, Dankclaw) | `serverAppConfigs-docs/LLM/frank-dragoon-flash-server-v2-swift15.md` |
|
||||
| frank-zealot (llama.cpp CUDA, Qwen3.5-9B-Uncensored Q6_K, 2×200,704 exact f16 KV MTP-off, vision ON, Dankclaw) | `serverAppConfigs-docs/LLM/frank-zealot-llama-qwen35-9b-q6k-2x200704.md` |
|
||||
| frank-archon (radiance StillDeadcode, Qwen3.8-Flash-Next-FP8 native, 2×R9700 TP=2, Dankserver) | `serverAppConfigs-docs/LLM/frank-archon-radiance-qwen38-native-fp8.md` |
|
||||
| Dankclaw kernel posture (`amd_iommu=off`) | `serverAppConfigs-docs/LLM/dankclaw-iommu-translation-off.md` |
|
||||
| Deployment flow + incidents | `serverAppConfigs-docs/LLM/deployment.md` |
|
||||
|
||||
@@ -959,36 +1106,41 @@ Retired (rollback targets and history):
|
||||
|
||||
| Runtime | Host | Window | Runbook |
|
||||
|---------|------|--------|---------|
|
||||
| ROCmFPX llama.cpp fork Qwen3.8-27B Uncensored Q6_0_ROCMFPX_AGENT + MTP (`frank-large`) | Dankclaw | 2026-10-03 → 10-05 (rollback target, payload + fork build on disk) | `frank-large-rocmfpx-q6agent-uncensored-mtp.md` |
|
||||
| tcclaviger vLLM fork Qwen3.8-Flash-Next-Uncensored FP8, TP=2, shape G (`frank-primary`) | Dankserver | 2026-09-29 → 10-05 (rollback target — tag `frank-primary-vllm-shapeG-20261003`; trio + env lore in the tag, image in rootless storage) | `dankserver-qwen38-fp8-expert-offload-tuning.md` |
|
||||
| ROCmFPX llama.cpp fork Qwen3.8-27B Q8_0_ROCMFPX_AGENT + MTP (`frank-large`) | Dankclaw | 2026-10-02 → 10-03 (rollback target, payload on disk) | `frank-large-rocmfpx-q8agent-mtp.md` |
|
||||
| gufo v0.1.1 Qwen3.8-27B UD-Q8_K_XL/UD-Q4_K_XL + DFlash2 (`frank-large`) | Dankclaw | 2026-09-28 → 10-01, again 10-01 → 10-02 (payloads removed from disk 2026-10-03; rollback = re-download) | `frank-large-gufo-qwen38-27b.md` |
|
||||
| vLLM TheRock MiMo-V2.6-Distill-Qwen-9B BF16 + cross-model MTP (`frank-large`) | Dankclaw | 2026-10-01 (same-day rollback to gufo) | `frank-large-vllm-mimo-v26-bf16.md` |
|
||||
| llama.cpp master MiMo-V2.6-Distill-Qwen-9B Q6_K (`frank-small`) | Dankclaw | 2026-10-01 → 10-02 (the rung returned ACTIVE 2026-10-03 at the 2×200,704 shape — `frank-small-llama-mimo-q6k-2x200704.md`) | `frank-small-llama-mimo-q6k.md` |
|
||||
| llama.cpp master MiMo-V2.6-Distill-Qwen-9B Q5_K_M 2×200,704 f16 KV (`frank-small`) | Dankclaw | 2026-10-02 → 10-03 (rollback target — one `MODEL_FILE` line, buys back 875 MiB + the −10 % rung tax; owns the boot-pair table) | `frank-small-llama-mimo-q5km.md` |
|
||||
| SGLang MiMo-V2.6 INT8-W8A16-AutoRound (`frank-small`) | Dankclaw | 2026-10-01 (same-day swap to the GGUF path) | `frank-small-sglang-mimo-v26.md` |
|
||||
|tcclaviger vLLM GLM-5.3-Flash W4A16 int4, +/- DFlash2 drafter (`frank-primary`)|Dankserver|2026-09-30 → 10-01|`frank-primary-glm53-flash-w4a16.md`|
|
||||
| llama.cpp b11030 Qwen3.8-27B UD-Q8_K_XL + DFlash2 (`frank-large`) | Dankclaw | 2026-09-21 → 09-28 | `frank-large-unsloth-qwen38-27b-q8xl.md` |
|
||||
| R9V radiance-vLLM Qwen3.8-Flash-Next-Uncensored IQ4_XS (`frank-primary`) | Dankserver | 2026-09-27 (same-day swap to Q8_0) | `frank-primary-r9v-radiance-qwen38-uncensored.md` |
|
||||
| llama.cpp b11030 Qwen3.8-Flash-Next Q8_0 + shared MTP (`frank-primary`) | Dankserver | 2026-09-26 → 09-27 | `frank-primary-llamacpp-qwen38-flash-next-q8.md` |
|
||||
| ik_llama.cpp Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M (`frank-large`) | Dankclaw | 2026-09-15 → 09-16 | `frank-large-ik-uncensored.md` |
|
||||
| llama.cpp CUDA Gemma4-26B-A4B QAT Q4_K_M, 1×200,704 f16-K/q8_0-V MTP-off (`frank-zealot`) | Dankclaw | 2026-10-07 → 10-07 (same-day swap; rollback target — payload on disk; owns the `attention_k_eq_v` pair-billing KV law and the gemma4-assistant drafter-cost boots) | `frank-zealot-llama-gemma4-q4km-200704.md` |
|
||||
| ROCmFPX llama.cpp fork Qwen3.8-27B Uncensored Q6_0_ROCMFPX_AGENT + MTP (`frank-dragoon`) | Dankclaw | 2026-10-03 → 10-05 (rollback target, payload + fork build on disk) | `frank-dragoon-rocmfpx-q6agent-uncensored-mtp.md` |
|
||||
| tcclaviger vLLM fork Qwen3.8-Flash-Next-Uncensored FP8, TP=2, shape G (`frank-archon`) | Dankserver | 2026-09-29 → 10-05 (rollback target — tag `frank-archon-vllm-shapeG-20261003`; trio + env lore in the tag, image in rootless storage) | `dankserver-qwen38-fp8-expert-offload-tuning.md` |
|
||||
| FreeToken PR #587 ROCm, Qwen3.8-Flash-Next-Uncensored-FP8, ONE R9700 (`frank-archon`) | Dankserver | 2026-10-05 → 10-05 (rollback target — tag `frank-archon-freetoken-pr587-20261005`; trio + Dockerfile in the tag, image `localhost/frank-archon-freetoken:pr587-77a7201` and orcarouter payload on disk) | `frank-archon-freetoken-pr587.md` |
|
||||
| llama.cpp master MiMo-V2.6-Distill-Qwen-9B Q6_K 2×200,704 f16 KV (`frank-zealot`) | Dankclaw | 2026-10-03 → 10-05 (rollback target — payload on disk; owns the ubatch-2048 A/B; the Ornith swap of 10-05 was committed but never deployed, so this was the LIVE payload Swift replaced) | `frank-zealot-llama-mimo-q6k-2x200704.md` |
|
||||
| llama.cpp Ornith-1.5-9B Q8_0, ONE 262,144 f16 KV (`frank-zealot`) | Dankclaw | 2026-10-05 (committed 77decb5→380aafa, never deployed — superseded by Swift the same evening; payload on disk) | trail in `frank-zealot-llama-swift15-unc-iq3xxs-mtp-114688.md` |
|
||||
| llama.cpp CUDA Swift-1.5-27B Uncensored RCO IQ3_XXS-mtp, 2×114,688 draft-mtp q8_0 KV (`frank-zealot`) | Dankclaw | 2026-10-06 → 2026-10-07 (rollback target — payload on disk; owns the 3090 ubatch ladder A/B and the 34 KiB/token qwen35 KV law) | `frank-zealot-llama-swift15-unc-iq3xxs-mtp-114688.md` |
|
||||
| ROCmFPX llama.cpp fork Qwen3.8-27B Q8_0_ROCMFPX_AGENT + MTP (`frank-dragoon`) | Dankclaw | 2026-10-02 → 10-03 (rollback target, payload on disk) | `frank-dragoon-rocmfpx-q8agent-mtp.md` |
|
||||
| gufo v0.1.1 Qwen3.8-27B UD-Q8_K_XL/UD-Q4_K_XL + DFlash2 (`frank-dragoon`) | Dankclaw | 2026-09-28 → 10-01, again 10-01 → 10-02 (payloads removed from disk 2026-10-03; rollback = re-download) | `frank-dragoon-gufo-qwen38-27b.md` |
|
||||
| vLLM TheRock MiMo-V2.6-Distill-Qwen-9B BF16 + cross-model MTP (`frank-dragoon`) | Dankclaw | 2026-10-01 (same-day rollback to gufo) | `frank-dragoon-vllm-mimo-v26-bf16.md` |
|
||||
| llama.cpp master MiMo-V2.6-Distill-Qwen-9B Q6_K (`frank-zealot`) | Dankclaw | 2026-10-01 → 10-02 (the rung returned ACTIVE 2026-10-03 at the 2×200,704 shape — `frank-zealot-llama-mimo-q6k-2x200704.md`) | `frank-zealot-llama-mimo-q6k.md` |
|
||||
| llama.cpp master MiMo-V2.6-Distill-Qwen-9B Q5_K_M 2×200,704 f16 KV (`frank-zealot`) | Dankclaw | 2026-10-02 → 10-03 (rollback target — one `MODEL_FILE` line, buys back 875 MiB + the −10 % rung tax; owns the boot-pair table) | `frank-zealot-llama-mimo-q5km.md` |
|
||||
| SGLang MiMo-V2.6 INT8-W8A16-AutoRound (`frank-zealot`) | Dankclaw | 2026-10-01 (same-day swap to the GGUF path) | `frank-zealot-sglang-mimo-v26.md` |
|
||||
|tcclaviger vLLM GLM-5.3-Flash W4A16 int4, +/- DFlash2 drafter (`frank-archon`)|Dankserver|2026-09-30 → 10-01|`frank-archon-glm53-flash-w4a16.md`|
|
||||
| llama.cpp b11030 Qwen3.8-27B UD-Q8_K_XL + DFlash2 (`frank-dragoon`) | Dankclaw | 2026-09-21 → 09-28 | `frank-dragoon-unsloth-qwen38-27b-q8xl.md` |
|
||||
| R9V radiance-vLLM Qwen3.8-Flash-Next-Uncensored IQ4_XS (`frank-archon`) | Dankserver | 2026-09-27 (same-day swap to Q8_0) | `frank-archon-r9v-radiance-qwen38-uncensored.md` |
|
||||
| llama.cpp b11030 Qwen3.8-Flash-Next Q8_0 + shared MTP (`frank-archon`) | Dankserver | 2026-09-26 → 09-27 | `frank-archon-llamacpp-qwen38-flash-next-q8.md` |
|
||||
| ik_llama.cpp Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M (`frank-dragoon`) | Dankclaw | 2026-09-15 → 09-16 | `frank-dragoon-ik-uncensored.md` |
|
||||
| vllm-radiance TP=2 MXFP4 + DFlash2 (`frank-medium`) | Dankserver | 2026-09-20 → 09-26 | `frank-medium-radiance-mxfp4.md` |
|
||||
| Lucebox DS4.1 DeepSeek-V4.1-Flash ROCmFP23 (`frank-primary`) | Dankserver | 2026-09-26 (same-day swap) | `frank-primary-lucebox-ds4.md` |
|
||||
| Lucebox DS4.1 DeepSeek-V4.1-Flash ROCmFP23 (`frank-archon`) | Dankserver | 2026-09-26 (same-day swap) | `frank-archon-lucebox-ds4.md` |
|
||||
| kingjones STRIX Qwen3.8-Flash-Next ROCmFP4 | Dankclaw | 2026-08-30 → 09-11 | `kingjones-strix-rocmfp4.md` |
|
||||
| halogen-flash-server Qwen3.8-Flash-Next w4b (`frank-large`) | Dankclaw | 2026-09-16 → 09-19 | `halogen-flash-server.md` |
|
||||
| CIRU v4.4.1 Orca Qwen3.8-Flash (`frank-large`) | Dankclaw | 2026-09-19 → 09-21 | `frank-large-ciru-orca.md` |
|
||||
| unsloth llama.cpp b11030 Qwen3.8-Flash-Next UD-IQ4_XS (`frank-large`) | Dankclaw | 2026-09-21 (same-day swap) | `frank-large-unsloth-qwen38-flash-next.md` |
|
||||
| Kairic Edge Qwen3.8-27B IU4 W4A4 (`frank-large`) | Dankclaw | 2026-09-21 (same-day swap) | `frank-large-kairic-edge-qwen38-27b.md` |
|
||||
| ISTA-DASLab Qwen3.8-27B-GSQ-RCO IQ3_S (`frank-small`) | Dankclaw | 2026-09-16 → 09-17 | `frank-small-qwen38-gsq-rco.md` |
|
||||
| vLLM Gemma4-12B w4a16 (`frank-small`) | Dankclaw | 2026-09-19 (same-day revert) | `frank-small-vllm-gemma4.md` |
|
||||
| llama.cpp Gemma4-12B QAT Q4_K_M (`frank-small`) | Dankclaw | 2026-09-17 → 10-01 | `frank-small-gemma4-12b.md` |
|
||||
| halogen-flash-server Qwen3.8-Flash-Next w4b (`frank-dragoon`) | Dankclaw | 2026-09-16 → 09-19 | `halogen-flash-server.md` |
|
||||
| CIRU v4.4.1 Orca Qwen3.8-Flash (`frank-dragoon`) | Dankclaw | 2026-09-19 → 09-21 | `frank-dragoon-ciru-orca.md` |
|
||||
| unsloth llama.cpp b11030 Qwen3.8-Flash-Next UD-IQ4_XS (`frank-dragoon`) | Dankclaw | 2026-09-21 (same-day swap) | `frank-dragoon-unsloth-qwen38-flash-next.md` |
|
||||
| Kairic Edge Qwen3.8-27B IU4 W4A4 (`frank-dragoon`) | Dankclaw | 2026-09-21 (same-day swap) | `frank-dragoon-kairic-edge-qwen38-27b.md` |
|
||||
| ISTA-DASLab Qwen3.8-27B-GSQ-RCO IQ3_S (`frank-zealot`) | Dankclaw | 2026-09-16 → 09-17 | `frank-zealot-qwen38-gsq-rco.md` |
|
||||
| vLLM Gemma4-12B w4a16 (`frank-zealot`) | Dankclaw | 2026-09-19 (same-day revert) | `frank-zealot-vllm-gemma4.md` |
|
||||
| llama.cpp Gemma4-12B QAT Q4_K_M (`frank-zealot`) | Dankclaw | 2026-09-17 → 10-01 | `frank-zealot-gemma4-12b.md` |
|
||||
| agentionai ROCmFP4-FAST imatrix | Dankclaw | 2026-08-29 → 08-30 | `agentionai-fast-imatrix.md` |
|
||||
| eemin Qwen3.8-27B ROCmI4 W4A4 MTP | Dankclaw | 2026-08-18 → 08-25 | `eemin-rocmi4-mtp.md` |
|
||||
| Kairic Edge IU4 | Dankclaw | 2026-08-22 (same-day revert) | `kairic-edge.md` |
|
||||
| CIRU ActiveFPX v3 / PromptForge | Dankclaw | through 2026-08-22 | `ciru-activefpx-promptforge.md` |
|
||||
| DFlash2 Vulkan | Dankclaw | 2026-08-21 (same-day revert) | `dflash2-large-vulkan.md` |
|
||||
| FastFlowLM NPU (`frank-npu`) | Dankclaw | twice, retired 2026-09-11 | `frank-npu.md`, `FastFlowLM NPU Inference.md` |
|
||||
| FreeToken (official + Huihui FP8, `frank-small`) | Dankserver | 2026-09-02 → 09-15 | `freetoken-frank-small.md` |
|
||||
| FreeToken (official + Huihui FP8, `frank-zealot`) | Dankserver | 2026-09-02 → 09-15 | `freetoken-frank-small.md` |
|
||||
| FastMTP patched llama.cpp | Dankserver | retired 2026-08-18 | `qwen3.8-spec-matrix.md` |
|
||||
| ik_llama.cpp Ornith CRACK | Dankserver | retired 2026-09-02 | `ikllama-ornith-dankserver.md` |
|
||||
| llama.cpp layer split + DFlash2 (`frank-medium`) | Dankserver | 2026-09-16 → 09-20 | `frank-medium-llamacpp-dflash2.md` |
|
||||
@@ -1012,7 +1164,7 @@ history.
|
||||
| Unstaged local changes | **Commit or discard** — check `git status` before starting work; ask if unsure. |
|
||||
| Deploying a committed change | **`scripts/deploy.sh <target>`** — pushes, pulls ff-only, plans, dispatches, health-checks. |
|
||||
| Container without a Quadlet unit | **Give it one** — boot-persistent, self-healing `.container` via git. |
|
||||
| Benchmarking | **Stop and mask the service first**, bench, unmask, restart, verify health. |
|
||||
| Benchmarking | **Not part of a model swap** — swaps verify on the live service. A deliberate bench window: **stop and mask the service first**, bench, unmask, restart, verify health. |
|
||||
| Deleting a file | **Move it to trash** — `.AGENTTRASH/` (in-repo) or `~/.AGENTTRASH/YYYYMMDD-topic`. Never `rm`. |
|
||||
| Tests exist | **Run them** — verify tests pass after every change. Do not commit failing tests. |
|
||||
| Real secrets | **Never commit** — host `.env` files; `.env.example` holds placeholders only. |
|
||||
|
||||
@@ -13,7 +13,7 @@ one deploy script.
|
||||
| Host | User | IP | Podman stack | LLM service | LLM port |
|
||||
|------|------|----|--------------|-------------|----------|
|
||||
| dankserver | dank | 192.168.0.235 | media and library | frank-medium (ROCmFPX llama.cpp, Qwen3.8-27B AGENT) | 7770 |
|
||||
| dankclaw | dank | 192.168.0.143 | AI, search, development, observability | frank-large (gufo, Qwen3.8-27B UD-Q8_K_XL) | 7770 |
|
||||
| dankclaw | dank | 192.168.0.143 | AI, search, development, observability | frank-dragoon (gufo, Qwen3.8-27B UD-Q8_K_XL) | 7770 |
|
||||
|
||||
## Active Services
|
||||
|
||||
@@ -30,7 +30,7 @@ one deploy script.
|
||||
| Gitea | dankclaw | gitea.dankclaw.local | 3001 |
|
||||
| Grafana | dankclaw | grafana.dankclaw.local | 3002 |
|
||||
| Prometheus | dankclaw | prometheus.dankclaw.local | 9090 |
|
||||
| frank-large | dankclaw | — (bare endpoint) | 7770 |
|
||||
| frank-dragoon | dankclaw | — (bare endpoint) | 7770 |
|
||||
|
||||
## Structure
|
||||
|
||||
|
||||
+22
-113
@@ -1,117 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
# autoresearch.sh — canonical benchmark entrypoint: speed of frank-primary.
|
||||
# autoresearch.sh — canonical frank-archon benchmark entry point (dev box).
|
||||
#
|
||||
# Runs on the operator box and drives one bench window on DANKSERVER:
|
||||
# 1. fingerprint the candidate trio (env + serve script + unit)
|
||||
# 2. memo fast-path: an identical fingerprint that already produced a FULLY
|
||||
# successful window (bench + verified production restore) returns the
|
||||
# cached metrics without touching the fleet
|
||||
# 3. stage the candidate trio plus the measure client into
|
||||
# /home/dank/.autoresearch-frank-primary/ (ephemeral; the repo checkout
|
||||
# on the host stays the production source of truth)
|
||||
# 4. run the window (scripts/autoresearch/frank-primary-window.sh):
|
||||
# stop + mask the production unit, boot the staged config AS THE SERVICE
|
||||
# (same serve script, same container, same port 7770), wait for health,
|
||||
# run the in-container usage-counter measure, then restore the production
|
||||
# unit and verify its health — AGENTS.md Benchmarking Policy
|
||||
# 5. print METRIC lines (parse by run_experiment)
|
||||
#
|
||||
# Primary metric: decode_tps (HIGHER is better) — median across the three
|
||||
# prompt depths (115/1056/4096) of per-depth medians of
|
||||
# completion_tokens / (wall - ttft), 3 runs each after a warm pass,
|
||||
# temperature 0, the campaign's deterministic word-seed workload.
|
||||
#
|
||||
# Exit 0 with METRIC lines on success; non-zero on any failure (the window
|
||||
# script's EXIT trap restores production on every path).
|
||||
set -euo pipefail
|
||||
# The service runs on dankserver (192.168.0.235:7770), the two R9700s live
|
||||
# there, and the benchmark must own those cards, so the whole stop -> boot
|
||||
# scratch -> bench -> restore sequence runs ON dankserver (see
|
||||
# scripts/autoresearch-frank-archon.sh). This wrapper:
|
||||
# 1. copies the candidate config (the repo's scripts/frank-archon.env, the
|
||||
# file the autoresearch loop edits) and the bench script to dankserver,
|
||||
# 2. runs the bench over ssh, relaying its stdout (the METRIC lines) and
|
||||
# exit code back here verbatim.
|
||||
# It deliberately does NOT push to origin: the scp dirties dankserver's
|
||||
# working tree, and the loop's uncommitted candidate config must not be
|
||||
# pushed. Only this file and scripts/autoresearch-frank-archon.sh are
|
||||
# committed; the candidate env rides over ssh each run.
|
||||
set -uo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REMOTE=dankserver
|
||||
REMOTE_REPO=/home/dank/serverAppConfigs
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
HOST="${FRANK_PRIMARY_AR_HOST:-dankserver}"
|
||||
STAGE="/home/dank/.autoresearch-frank-primary"
|
||||
scp -q "$DIR/scripts/frank-archon.env" \
|
||||
"$REMOTE:$REMOTE_REPO/scripts/frank-archon.env" || { echo "scp env failed" >&2; exit 1; }
|
||||
scp -q "$DIR/scripts/autoresearch-frank-archon.sh" \
|
||||
"$REMOTE:$REMOTE_REPO/scripts/autoresearch-frank-archon.sh" || { echo "scp script failed" >&2; exit 1; }
|
||||
|
||||
# FRANK_PRIMARY_AR_ENV_FILE stages a DIFFERENT env than the repo's — that is how
|
||||
# the arms driver A/Bs engine knobs without mutating the checkout. The
|
||||
# fingerprint covers whatever file is staged, so the memo cannot cross arms.
|
||||
ENV_FILE="${FRANK_PRIMARY_AR_ENV_FILE:-$SCRIPT_DIR/scripts/frank-primary.env}"
|
||||
SERVE="$SCRIPT_DIR/scripts/serve-frank-primary.sh"
|
||||
UNIT="$SCRIPT_DIR/scripts/frank-primary.service"
|
||||
MEASURE="$SCRIPT_DIR/scripts/autoresearch/frank-primary-measure.py"
|
||||
WINDOW="$SCRIPT_DIR/scripts/autoresearch/frank-primary-window.sh"
|
||||
|
||||
for f in "$ENV_FILE" "$SERVE" "$UNIT" "$MEASURE" "$WINDOW"; do
|
||||
[ -f "$f" ] || { echo "[autoresearch] missing harness file: $f" >&2; exit 1; }
|
||||
done
|
||||
command -v python3 >/dev/null 2>&1 || { echo "[autoresearch] python3 required on operator box" >&2; exit 1; }
|
||||
|
||||
# Fingerprint of everything that can change the engine's argv or environment.
|
||||
HASH="$(cat "$ENV_FILE" "$SERVE" "$UNIT" | sha256sum | cut -d' ' -f1)"
|
||||
echo "[autoresearch] candidate fingerprint $HASH" >&2
|
||||
|
||||
ssh_run() { ssh -o BatchMode=yes -o ConnectTimeout=15 "$HOST" "$@"; }
|
||||
|
||||
emit_metrics() { # $1 = run.json
|
||||
python3 -c '
|
||||
import json, sys
|
||||
run = json.loads(sys.argv[1])
|
||||
m = run["measure"]
|
||||
print("METRIC decode_tps=%s" % m["decode_tps"])
|
||||
for k in ("decode_tps_115", "decode_tps_1056", "decode_tps_4096"):
|
||||
print("METRIC %s=%s" % (k, m[k]))
|
||||
print("METRIC ttft_ms_4096=%s" % m["ttft_ms_4096"])
|
||||
print("METRIC kv_tokens=%s" % (run.get("kv_tokens") or 0))
|
||||
print("METRIC boot_seconds=%s" % (run.get("boot_seconds") or -1))
|
||||
print("METRIC completion_tokens_4096=%s" % m.get("completion_tokens_4096", 0))
|
||||
# Any extra sampling temperature the measure swept (AR_TEMPS), e.g. the 0.7
|
||||
# arm — that is where speculative-decode acceptance shows up, since a greedy
|
||||
# draft matches a greedy target exactly and is rejected more under real
|
||||
# sampling. Suffixed keys only (_t0.7), so the historical lines are untouched.
|
||||
for k in sorted(m):
|
||||
if "_t" in k and (k.startswith("decode_tps") or k.startswith("ttft_ms")):
|
||||
print("METRIC %s=%s" % (k, m[k]))
|
||||
' "$1"
|
||||
}
|
||||
|
||||
# --- memo fast-path -------------------------------------------------------
|
||||
cached_hash="$(ssh_run "cat '$STAGE/last/hash' 2>/dev/null" 2>/dev/null || true)"
|
||||
if [ -n "$cached_hash" ] && [ "$cached_hash" = "$HASH" ]; then
|
||||
cached_run="$(ssh_run "cat '$STAGE/last/run.json' 2>/dev/null" || true)"
|
||||
if [ -n "$cached_run" ]; then
|
||||
echo "[autoresearch] memo hit — cached metrics for $HASH, fleet untouched" >&2
|
||||
emit_metrics "$cached_run"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- stage the candidate ----------------------------------------------------
|
||||
echo "[autoresearch] staging candidate on $HOST:$STAGE" >&2
|
||||
ssh_run "mkdir -p '$STAGE'"
|
||||
scp -q "$ENV_FILE" "$HOST:$STAGE/frank-primary.env"
|
||||
scp -q "$SERVE" "$HOST:$STAGE/serve.sh"
|
||||
scp -q "$MEASURE" "$HOST:$STAGE/measure.py"
|
||||
scp -q "$WINDOW" "$HOST:$STAGE/window.sh"
|
||||
ssh_run "printf '%s\n' '$HASH' > '$STAGE/hash'"
|
||||
|
||||
# --- the window -------------------------------------------------------------
|
||||
# Budgets: bench boot up to 20 min (warm boots measured ~7 min), measure up
|
||||
# to 30 min, restore boot up to 20 min — call run_experiment with
|
||||
# timeout_seconds >= 3600.
|
||||
echo "[autoresearch] opening bench window (stop+mask -> boot -> measure -> restore)" >&2
|
||||
set +e
|
||||
ssh_run "bash '$STAGE/window.sh'"
|
||||
rc=$?
|
||||
set -e
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
echo "[autoresearch] window failed (rc=$rc); host-side evidence:" >&2
|
||||
ssh_run "tail -60 '$STAGE/fail.log' 2>/dev/null || tail -40 '$STAGE/serve.out' 2>/dev/null || tail -40 '$STAGE/measure.out' 2>/dev/null || true" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
run_json="$(ssh_run "cat '$STAGE/run.json'")" || {
|
||||
echo "[autoresearch] could not read $STAGE/run.json back" >&2; exit 1; }
|
||||
echo "[autoresearch] window result:" >&2
|
||||
printf '%s\n' "$run_json" >&2
|
||||
emit_metrics "$run_json"
|
||||
|
||||
echo "[autoresearch] engine evidence (plan / compile path / spec counters):" >&2
|
||||
ssh_run "cat '$STAGE/evidence.log' 2>/dev/null" >&2 || true
|
||||
exit 0
|
||||
ssh "$REMOTE" "bash $REMOTE_REPO/scripts/autoresearch-frank-archon.sh $REMOTE_REPO/scripts/frank-archon.env"
|
||||
|
||||
+10
-1
@@ -337,7 +337,16 @@ http://:8017 {
|
||||
try_files /docs.html
|
||||
file_server
|
||||
}
|
||||
@data path /services.json /status.json /app.js /style.css /deployed.js
|
||||
# /whoami echoes the outpost-injected identity headers so the page
|
||||
# can gate the admin button client-side (the /admin* 403 above stays
|
||||
# the real gate; this only decides what is shown). The outpost strips
|
||||
# client-supplied X-Authentik-* before injecting, so this is spoof-proof.
|
||||
handle /whoami {
|
||||
templates
|
||||
header Content-Type application/json
|
||||
respond "{\"user\":\"{header.X-Authentik-Username}\",\"groups\":\"{header.X-Authentik-Groups}\"}"
|
||||
}
|
||||
@data path /services.json /status.json /app.js /style.css /deployed.js /img/* /sfx/*
|
||||
handle @data {
|
||||
file_server
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"3": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"_meta": { "title": "Positive prompt" },
|
||||
"inputs": {
|
||||
"text": "A detailed illustration of a 1930s Soviet industrial design bureau at night: a well-measured engineer at a drafting table, a warm desk lamp, rolled blueprints, gears and calipers on the desk, deep red and cream palette, gouache painterly style, intricate mechanical detail, soft shadows",
|
||||
"clip": [ "6", 0 ]
|
||||
}
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"_meta": { "title": "Negative prompt" },
|
||||
"inputs": {
|
||||
"text": "lowres, blurry, deformed, extra limbs, watermark, signature, jpeg artifacts",
|
||||
"clip": [ "6", 0 ]
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"class_type": "UNETLoader",
|
||||
"_meta": { "title": "Qwen Image 2.1 (int8)" },
|
||||
"inputs": { "unet_name": "qwen_image_2.1_int8_convrot.safetensors", "weight_dtype": "default" }
|
||||
},
|
||||
"6": {
|
||||
"class_type": "CLIPLoader",
|
||||
"_meta": { "title": "Qwen3-VL 8B text encoder (w4a8)" },
|
||||
"inputs": { "clip_name": "qwen3vl_8b_w4a8.safetensors", "type": "qwen_image" }
|
||||
},
|
||||
"7": {
|
||||
"class_type": "VAELoader",
|
||||
"_meta": { "title": "VAE" },
|
||||
"inputs": { "vae_name": "qwen_image_2.1_vae_bf16.safetensors" }
|
||||
},
|
||||
"8": {
|
||||
"class_type": "EmptySD3LatentImage",
|
||||
"_meta": { "title": "Empty Latent 1024x1024" },
|
||||
"inputs": { "width": 1024, "height": 1024, "batch_size": 1 }
|
||||
},
|
||||
"9": {
|
||||
"class_type": "KSampler",
|
||||
"_meta": { "title": "KSampler" },
|
||||
"inputs": {
|
||||
"model": [ "5", 0 ],
|
||||
"seed": 1930,
|
||||
"steps": 20,
|
||||
"cfg": 1.0,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"positive": [ "3", 0 ],
|
||||
"negative": [ "4", 0 ],
|
||||
"latent_image": [ "8", 0 ],
|
||||
"denoise": 1.0
|
||||
}
|
||||
},
|
||||
"10": {
|
||||
"class_type": "VAEDecode",
|
||||
"_meta": { "title": "VAE Decode" },
|
||||
"inputs": { "samples": [ "9", 0 ], "vae": [ "7", 0 ] }
|
||||
},
|
||||
"11": {
|
||||
"class_type": "SaveImage",
|
||||
"_meta": { "title": "Save Image (art)" },
|
||||
"inputs": { "filename_prefix": "qwen21-art", "images": [ "10", 0 ] }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"3": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"_meta": { "title": "Positive prompt (legible text)" },
|
||||
"inputs": {
|
||||
"text": "Clean minimalist movie-poster design, dark charcoal background with a subtle vintage grain. Centered large bold white sans-serif text reading \"SOLIDARITY OF THE SOVIET COMPUTER SYSTEMS\", perfectly legible, crisp letterforms, even letter spacing. Below it, a smaller line of yellow text reading \"DANKCLAW DANKSERVER EST 2026\", equally legible. Flat graphic design, high contrast, generous margins, no other words, no watermark",
|
||||
"clip": [ "6", 0 ]
|
||||
}
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"_meta": { "title": "Negative prompt" },
|
||||
"inputs": {
|
||||
"text": "blurry, illegible, misspelled, warped letters, extra text, watermark, signature, lowres, jpeg artifacts",
|
||||
"clip": [ "6", 0 ]
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"class_type": "UNETLoader",
|
||||
"_meta": { "title": "Qwen Image 2.1 (int8)" },
|
||||
"inputs": { "unet_name": "qwen_image_2.1_int8_convrot.safetensors", "weight_dtype": "default" }
|
||||
},
|
||||
"6": {
|
||||
"class_type": "CLIPLoader",
|
||||
"_meta": { "title": "Qwen3-VL 8B text encoder (w4a8)" },
|
||||
"inputs": { "clip_name": "qwen3vl_8b_w4a8.safetensors", "type": "qwen_image" }
|
||||
},
|
||||
"7": {
|
||||
"class_type": "VAELoader",
|
||||
"_meta": { "title": "VAE" },
|
||||
"inputs": { "vae_name": "qwen_image_2.1_vae_bf16.safetensors" }
|
||||
},
|
||||
"8": {
|
||||
"class_type": "EmptySD3LatentImage",
|
||||
"_meta": { "title": "Empty Latent 832x1216 poster" },
|
||||
"inputs": { "width": 832, "height": 1216, "batch_size": 1 }
|
||||
},
|
||||
"9": {
|
||||
"class_type": "KSampler",
|
||||
"_meta": { "title": "KSampler" },
|
||||
"inputs": {
|
||||
"model": [ "5", 0 ],
|
||||
"seed": 2026,
|
||||
"steps": 25,
|
||||
"cfg": 1.0,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"positive": [ "3", 0 ],
|
||||
"negative": [ "4", 0 ],
|
||||
"latent_image": [ "8", 0 ],
|
||||
"denoise": 1.0
|
||||
}
|
||||
},
|
||||
"10": {
|
||||
"class_type": "VAEDecode",
|
||||
"_meta": { "title": "VAE Decode" },
|
||||
"inputs": { "samples": [ "9", 0 ], "vae": [ "7", 0 ] }
|
||||
},
|
||||
"11": {
|
||||
"class_type": "SaveImage",
|
||||
"_meta": { "title": "Save Image (text poster)" },
|
||||
"inputs": { "filename_prefix": "qwen21-text", "images": [ "10", 0 ] }
|
||||
}
|
||||
}
|
||||
@@ -10,15 +10,15 @@ model:
|
||||
provider: dankclaw-large
|
||||
base_url: http://192.168.0.143:7770/v1
|
||||
api_key: not-needed
|
||||
default: frank-large
|
||||
default: frank-dragoon
|
||||
|
||||
providers:
|
||||
dankclaw-large:
|
||||
api: http://192.168.0.143:7770/v1
|
||||
discover_models: false
|
||||
models:
|
||||
frank-large:
|
||||
context_length: 114688
|
||||
frank-dragoon:
|
||||
context_length: 262144
|
||||
dankserver-small:
|
||||
api: http://192.168.0.235:7770/v1
|
||||
discover_models: false
|
||||
@@ -29,7 +29,7 @@ providers:
|
||||
# Auxiliary routing from the live configuration. Named providers route via
|
||||
# their `api:` URL; `custom` entries route by their explicit `base_url`.
|
||||
auxiliary:
|
||||
vision: {provider: dankclaw-large, model: frank-large}
|
||||
vision: {provider: dankclaw-large, model: frank-dragoon}
|
||||
web_extract: {provider: dankserver-small, model: frank-medium}
|
||||
compression: {provider: dankserver-small, model: frank-medium}
|
||||
skills_hub: {provider: dankserver-small, model: frank-medium}
|
||||
@@ -50,8 +50,8 @@ auxiliary:
|
||||
effort: none
|
||||
memory_query_rewrite: {provider: custom, model: frank-medium, base_url: http://192.168.0.235:7770/v1}
|
||||
tts_audio_tags: {provider: custom, model: frank-medium, base_url: http://192.168.0.235:7770/v1}
|
||||
triage_specifier: {provider: dankclaw-large, model: frank-large}
|
||||
kanban_decomposer: {provider: dankclaw-large, model: frank-large}
|
||||
triage_specifier: {provider: dankclaw-large, model: frank-dragoon}
|
||||
kanban_decomposer: {provider: dankclaw-large, model: frank-dragoon}
|
||||
profile_describer: {provider: dankserver-small, model: frank-medium}
|
||||
curator: {provider: dankserver-small, model: frank-medium}
|
||||
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
@@ -1,9 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# download-frank-primary-fp8.sh — fetch/verify the Qwen3.8-Flash-Next-FP8
|
||||
# download-frank-archon-fp8.sh — fetch/verify the Qwen3.8-Flash-Next-FP8
|
||||
# payload (131 safetensors shards, 172.78 GiB total, plus aux files) for the
|
||||
# vLLM frank-primary deployment on dankserver.
|
||||
# vLLM frank-archon deployment on dankserver.
|
||||
#
|
||||
# bash scripts/download-frank-primary-fp8.sh [dest_dir]
|
||||
# bash scripts/download-frank-archon-fp8.sh [dest_dir]
|
||||
#
|
||||
# The manifest (scripts/manifests/qwen3.8-flash-next-fp8.manifest) carries
|
||||
# per-file sizes + LFS sha256s from the Hub; hf-fetch.sh is idempotent — an
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the z-lab Qwen3.5-9B-DFlash drafter for frank-large's DFlash
|
||||
# Download the z-lab Qwen3.5-9B-DFlash drafter for frank-dragoon's DFlash
|
||||
# speculative-decode arm (vLLM --speculative-config method=dflash). Public
|
||||
# repo — no auth header on the command line.
|
||||
#
|
||||
@@ -10,7 +10,7 @@
|
||||
# the draft for that target.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-large-dflash.sh [DEST_DIR] [--verify-only]
|
||||
# scripts/download-frank-dragoon-dflash.sh [DEST_DIR] [--verify-only]
|
||||
set -euo pipefail
|
||||
|
||||
REPO="z-lab/Qwen3.5-9B-DFlash"
|
||||
@@ -1,14 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the BF16 MiMo-V2.6-Distill-Qwen-9B payload for frank-large
|
||||
# Download the BF16 MiMo-V2.6-Distill-Qwen-9B payload for frank-dragoon
|
||||
# (vLLM/kyuz0 gfx1151, Strix Halo iGPU). Public repo — no auth header on the
|
||||
# command line (a Bearer token in argv is world-readable via /proc).
|
||||
#
|
||||
# Per-file FRESH downloads, no resume: the house rule (see
|
||||
# download-frank-small.sh); -f/--fail-with-body plus size+sha256 against the
|
||||
# download-frank-zealot.sh); -f/--fail-with-body plus size+sha256 against the
|
||||
# Hub LFS oid is what makes an access error fail instead of "succeeding".
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-large-mimo.sh [DEST_DIR] [--verify-only]
|
||||
# scripts/download-frank-dragoon-mimo.sh [DEST_DIR] [--verify-only]
|
||||
#
|
||||
# The verified set below is exactly what the 2026-10-01 bench window served:
|
||||
# 17.53 GiB of BF16 weights (4 shards, 18,819,720,848 B total with the rest).
|
||||
@@ -1,14 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the source Qwen3.5-9B checkpoint — the ONLY place this model
|
||||
# family's MTP head (15 mtp.* tensors) exists. frank-large runs it as the
|
||||
# family's MTP head (15 mtp.* tensors) exists. frank-dragoon runs it as the
|
||||
# vLLM cross-model MTP draft (MiMo-V2.6-Distill-Qwen-9B itself ships zero
|
||||
# mtp tensors; that is why SGLang's NEXTN on frank-small draws 0.00
|
||||
# mtp tensors; that is why SGLang's NEXTN on frank-zealot draws 0.00
|
||||
# acceptance). Public repo — no auth header on the command line.
|
||||
#
|
||||
# NOTE the odd upstream shard names (model.safetensors-0000N-of-00004...).
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-large-qwen35.sh [DEST_DIR] [--verify-only]
|
||||
# scripts/download-frank-dragoon-qwen35.sh [DEST_DIR] [--verify-only]
|
||||
set -euo pipefail
|
||||
|
||||
REPO="Qwen/Qwen3.5-9B"
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound for frank-small
|
||||
# Download MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound for frank-zealot
|
||||
# (SGLang, RTX 3090). Public repo — no auth header on the command line.
|
||||
#
|
||||
# Per-file FRESH downloads, no resume: `curl -C -` once produced a 213 MB
|
||||
@@ -7,7 +7,7 @@
|
||||
# the house rule.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-small.sh [DEST_DIR] [--verify-only]
|
||||
# scripts/download-frank-zealot.sh [DEST_DIR] [--verify-only]
|
||||
#
|
||||
# LFS shards are verified against the Hub LFS sha256; non-LFS text files
|
||||
# against their declared byte size. Idempotent: complete files are skipped.
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# bench-frank-large.sh — 64k-context benchmark for the frank-default-large stack.
|
||||
# bench-frank-dragoon.sh — 64k-context benchmark for the frank-default-large stack.
|
||||
#
|
||||
# Measures generation throughput of Qwen3.6-27B Q8_K_P + DFlash + ngram-mod on
|
||||
# a single backend at 64k context. It starts llama-server on a scratch port,
|
||||
@@ -7,15 +7,15 @@
|
||||
# prompt, and prints tokens/sec.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/bench-frank-large.sh <ROCm0|Vulkan0> [--generations N]
|
||||
# ./scripts/bench-frank-dragoon.sh <ROCm0|Vulkan0> [--generations N]
|
||||
#
|
||||
# Example:
|
||||
# ./scripts/bench-frank-large.sh ROCm0
|
||||
# ./scripts/bench-frank-large.sh Vulkan0 --generations 3
|
||||
# ./scripts/bench-frank-dragoon.sh ROCm0
|
||||
# ./scripts/bench-frank-dragoon.sh Vulkan0 --generations 3
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DEVICE="${1:?usage: bench-frank-large.sh <ROCm0|Vulkan0> [--generations N]}"
|
||||
DEVICE="${1:?usage: bench-frank-dragoon.sh <ROCm0|Vulkan0> [--generations N]}"
|
||||
GENERATIONS=1
|
||||
if [ "${2:-}" = "--generations" ]; then
|
||||
GENERATIONS="${3:?--generations needs a count}"
|
||||
@@ -41,7 +41,7 @@ SERVICE_HEALTH[dankclaw]="http://127.0.0.1:7770/health"
|
||||
# Drop page caches before restart: with the model's pages fully cached, the
|
||||
# KV/graph allocation stalls at load (VRAM stuck at 28 GiB). See AGENTS.md.
|
||||
DROP_CACHES[dankclaw]=1
|
||||
# Passwordless restart helper (root-owned, installed via /etc/sudoers.d/frank-large).
|
||||
# Passwordless restart helper (root-owned, installed via /etc/sudoers.d/frank-dragoon).
|
||||
# When present, deploy.sh uses it instead of inline sudo commands.
|
||||
HELPER[dankclaw]="/usr/local/sbin/frank-restart-large.sh"
|
||||
# Health wait in iterations of 2s. Cold loads after the cache drop measured
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Tunables for the Ling-3.0-flash TRAIL on dankclaw (frank-large-ling).
|
||||
# Tunables for the Ling-3.0-flash TRAIL on dankclaw (frank-dragoon-ling).
|
||||
# This REPLACES frank-default-large (Qwen3.6-27B) temporarily. To switch
|
||||
# back: systemctl stop frank-large-ling && systemctl start frank-default-large.
|
||||
# back: systemctl stop frank-dragoon-ling && systemctl start frank-default-large.
|
||||
# Runtime: llama.cpp FORK bailingmoe3-support; Vulkan + AMD GTT.
|
||||
#
|
||||
# Memory model (amdgpu Strix Halo gfx1151, 8x16GB LPDDR5 = 128 GiB physical):
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[Unit]
|
||||
Description=llama.cpp Ling-3.0-flash Q6_K Vulkan+MTP (frank-large-ling TRIAL)
|
||||
Description=llama.cpp Ling-3.0-flash Q6_K Vulkan+MTP (frank-dragoon-ling TRIAL)
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
@@ -8,7 +8,7 @@
|
||||
# Vulkan exposes as an ~111 GiB heap). MTP drafter is bundled in the same
|
||||
# GGUF (block 42) - no separate --spec-draft-model needed.
|
||||
#
|
||||
# Tie-back: to restore the Qwen3.6-27B service, stop frank-large-ling and
|
||||
# Tie-back: to restore the Qwen3.6-27B service, stop frank-dragoon-ling and
|
||||
# start frank-default-large (its env/unit are committed and unchanged).
|
||||
#
|
||||
# Tunables come from frank-default-large.ling.env in this directory unless
|
||||
|
||||
@@ -4,11 +4,11 @@
|
||||
# Runtime: llama.cpp llama-server (>= b10411).
|
||||
#
|
||||
# Tunables come from frank-default-large.env in this directory unless the
|
||||
# FRANK_LARGE_ENV_FILE environment variable overrides that path.
|
||||
# FRANK_DRAGOON_ENV_FILE environment variable overrides that path.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}"
|
||||
ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
+1
-1
File diff suppressed because one or more lines are too long
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Decode throughput benchmark for frank-large.
|
||||
Decode throughput benchmark for frank-dragoon.
|
||||
Starts the server with a given --threads-batch value, sends an 8K+ token prompt,
|
||||
generates 256 tokens, measures generation tok/s.
|
||||
|
||||
@@ -13,12 +13,12 @@ SERVE = "/home/dank/GitPackages/ROCmFPX/build-qwen38/bin/llama-server"
|
||||
PORT = 7770
|
||||
N_GEN = 256
|
||||
PROMPT_FILE = "/tmp/bench-prompts/prompt_code.txt"
|
||||
ENV_FILE = os.path.join(os.path.dirname(__file__), "frank-large.env")
|
||||
ENV_FILE = os.path.join(os.path.dirname(__file__), "frank-dragoon.env")
|
||||
SERVER_DIR = os.path.dirname(SERVE)
|
||||
ROCM_PATH = "/opt/rocm"
|
||||
|
||||
def make_env():
|
||||
"""Build the same env as serve-frank-large.sh."""
|
||||
"""Build the same env as serve-frank-dragoon.sh."""
|
||||
env = {**os.environ, "HSA_OVERRIDE_GFX_VERSION": "11.5.1", "HIP_VISIBLE_DEVICES": "0"}
|
||||
|
||||
# Source env file for tunables
|
||||
@@ -59,7 +59,7 @@ def wait_ready():
|
||||
def send_request(prompt, max_tokens=N_GEN):
|
||||
"""Send chat completion, measure generation throughput."""
|
||||
body = json.dumps({
|
||||
"model": "frank-large",
|
||||
"model": "frank-dragoon",
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens,
|
||||
"stream": False,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Benchmark n-gram speculative decoding vs no-spec for frank-large.
|
||||
Benchmark n-gram speculative decoding vs no-spec for frank-dragoon.
|
||||
|
||||
Tests prompt-processing throughput with 16K token prompts (manageable request size).
|
||||
Code-heavy prompt (high n-gram redundancy) vs prose prompt (low redundancy).
|
||||
@@ -103,7 +103,7 @@ def start_server(port, spec_type):
|
||||
def run_test(prompt_text, spec_type, port):
|
||||
"""Send a request and extract prompt processing metrics."""
|
||||
data = json.dumps({
|
||||
"model": "frank-large",
|
||||
"model": "frank-dragoon",
|
||||
"messages": [{"role": "user", "content": prompt_text}],
|
||||
"max_tokens": TEST_TOKENS,
|
||||
"stream": False,
|
||||
@@ -189,7 +189,7 @@ def main():
|
||||
print(f" Prose: {prose_prompt_file.stat().st_size:,} bytes")
|
||||
|
||||
print("=" * 60)
|
||||
print(" N-gram Mod Benchmark — frank-large")
|
||||
print(" N-gram Mod Benchmark — frank-dragoon")
|
||||
print(f" STRIX ROCmFP4 · 64K context · 16K prompts")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
# Benchmark n-gram speculative decoding vs no-spec for frank-large.
|
||||
# Benchmark n-gram speculative decoding vs no-spec for frank-dragoon.
|
||||
#
|
||||
# This tests prompt-processing throughput at 64k context with two distinct
|
||||
# prompts: one code-heavy and one prose-heavy. Uses llama-server on scratch
|
||||
# ports with the OpenAI-compatible API to measure prompt_eval_duration_ms.
|
||||
#
|
||||
# Run on Dankclaw after stopping the production service:
|
||||
# sudo systemctl stop frank-large
|
||||
# sudo systemctl stop frank-dragoon
|
||||
# ./scripts/bench-ngram-mod.sh
|
||||
# sudo systemctl start frank-large
|
||||
# sudo systemctl start frank-dragoon
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a; source "$ENV_FILE"; set +a
|
||||
fi
|
||||
@@ -184,7 +184,7 @@ run_single_test() {
|
||||
response=$(curl -s -X POST "http://127.0.0.1:$port/v1/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"model\": \"frank-large\",
|
||||
\"model\": \"frank-dragoon\",
|
||||
\"prompt\": \"$prompt_text\",
|
||||
\"max_tokens\": 128,
|
||||
\"temperature\": 0,
|
||||
@@ -220,7 +220,7 @@ run_single_test() {
|
||||
}
|
||||
|
||||
echo "╔══════════════════════════════════════════════════════════╗"
|
||||
echo "║ N-gram Mod Benchmark — frank-large ║"
|
||||
echo "║ N-gram Mod Benchmark — frank-dragoon ║"
|
||||
echo "║ STRIX ROCmFP4 · 64k context · two prompts ║"
|
||||
echo "╚══════════════════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
|
||||
@@ -15,9 +15,9 @@ import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
SERVICE = "frank-large"
|
||||
ENV_FILE = "/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ALIAS = "frank-large"
|
||||
SERVICE = "frank-dragoon"
|
||||
ENV_FILE = "/home/dank/serverAppConfigs/scripts/frank-dragoon.env"
|
||||
ALIAS = "frank-dragoon"
|
||||
DEFAULT_HOST = "localhost"
|
||||
DEFAULT_PORT = 7770
|
||||
DEFAULT_MAX_TOKENS = 16384
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
# Tunables for frank-large (mradermacher i1-Q4_K_M of
|
||||
# Tunables for frank-dragoon (mradermacher i1-Q4_K_M of
|
||||
# orcarouter/Qwen3.8-Flash-Next-Uncensored on ik_llama.cpp, Dankclaw,
|
||||
# RTX 3090 CUDA + Strix Halo CPU path, since 2026-09-15).
|
||||
#
|
||||
@@ -65,7 +65,7 @@ MMPROJ_PATH=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K
|
||||
CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
|
||||
|
||||
# Alias + network.
|
||||
ALIAS=frank-large
|
||||
ALIAS=frank-dragoon
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7770
|
||||
|
||||
+3
-3
@@ -1,5 +1,5 @@
|
||||
[Unit]
|
||||
Description=ik_llama.cpp Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M 177B MoE (dankclaw, frank-large)
|
||||
Description=ik_llama.cpp Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M 177B MoE (dankclaw, frank-dragoon)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
@@ -12,8 +12,8 @@ Group=dank
|
||||
# budget still needs room. Parity with the halogen quadlet's
|
||||
# Ulimit=memlock=-1:-1.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
|
||||
Environment="FRANK_DRAGOON_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-dragoon.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-dragoon.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
+3
-3
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve mradermacher/Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M as frank-large
|
||||
# Serve mradermacher/Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M as frank-dragoon
|
||||
# via ik_llama.cpp on Dankclaw.
|
||||
#
|
||||
# Topology (ikawrakow's hybrid recipe; upstream issue #2402 for the PLE):
|
||||
@@ -26,7 +26,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
@@ -66,7 +66,7 @@ fi
|
||||
# CPU-overridden weights mmapped (src/llama-load-tensors.cpp); the env file
|
||||
# sets it. Refuse to run without it.
|
||||
if [[ "${GGML_CUDA_NO_PINNED_WEIGHTS:-}" != "1" ]]; then
|
||||
echo "frank-large: refusing to start — GGML_CUDA_NO_PINNED_WEIGHTS=1 is mandatory on this host" >&2
|
||||
echo "frank-dragoon: refusing to start — GGML_CUDA_NO_PINNED_WEIGHTS=1 is mandatory on this host" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -112,7 +112,7 @@ fi
|
||||
: "${SPEC_DRAFT_POLL_BATCH:=1}"
|
||||
|
||||
# Text only: the server must not try to autoload the vision tower. Vision
|
||||
# stays on frank-large (Dankclaw), which is where the fleet routes images.
|
||||
# stays on frank-dragoon (Dankclaw), which is where the fleet routes images.
|
||||
NO_MMPROJ="${NO_MMPROJ:-1}"
|
||||
|
||||
export HIP_VISIBLE_DEVICES
|
||||
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
# apply-protected-cast.sh — RUNS ON DANKSERVER. Rewrite the recipe's
|
||||
# PROTECTED-BEGIN/END section from Swift's OWN calibration grams: run
|
||||
# derive-protected-experts.py over the calib dir, splice its cast lines into
|
||||
# the marked section of the recipe in place, keep the pre-edit recipe in the
|
||||
# campaign trash. Refuses to run twice (marker text says derived).
|
||||
set -euo pipefail
|
||||
CALIB=${1:-/home/dank/corpora/calib}
|
||||
RECIPE=${2:-/home/dank/serverAppConfigs-swift15/scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe}
|
||||
PY=/home/dank/venvs/radcorpus/bin/python
|
||||
DERIVE=/home/dank/serverAppConfigs-swift15/scripts/derive-protected-experts.py
|
||||
TS=$(date +%Y%m%d-%H%M%S)
|
||||
|
||||
grep -q "derive-protected-experts.py rewrote" "$RECIPE" && {
|
||||
echo "recipe already carries derived cast lines; refusing to re-derive" >&2; exit 1; }
|
||||
grep -q "PROTECTED-BEGIN" "$RECIPE" || { echo "no PROTECTED markers in $RECIPE" >&2; exit 1; }
|
||||
|
||||
"$PY" "$DERIVE" "$CALIB" > "/tmp/protected-cast-$TS.txt"
|
||||
N=$(grep -c 'cast dtype=bf16' "/tmp/protected-cast-$TS.txt")
|
||||
[ "$N" -ge 2 ] || { echo "derive produced $N cast lines; refusing" >&2; exit 1; }
|
||||
echo "derived $N protected experts -> /tmp/protected-cast-$TS.txt"
|
||||
|
||||
cp -f "$RECIPE" "/home/dank/.AGENTTRASH/$TS-qwen4exp-swift15-w4dn8-i8-hc8m.recipe"
|
||||
"$PY" - "$RECIPE" "/tmp/protected-cast-$TS.txt" <<'PYEOF'
|
||||
import sys
|
||||
recipe, castfile = sys.argv[1], sys.argv[2]
|
||||
lines = open(recipe).read().splitlines()
|
||||
cast = [l for l in open(castfile).read().splitlines() if 'cast dtype=bf16' in l]
|
||||
out, insec = [], False
|
||||
for l in lines:
|
||||
if 'PROTECTED-BEGIN' in l:
|
||||
insec = True
|
||||
out.append(l)
|
||||
out.extend(cast)
|
||||
elif 'PROTECTED-END' in l:
|
||||
insec = False
|
||||
out.append(l)
|
||||
out.append('# (derive-protected-experts.py rewrote this section from Swift\'s own grams)')
|
||||
elif not insec:
|
||||
out.append(l)
|
||||
open(recipe, 'w').write('\n'.join(out) + '\n')
|
||||
print(f'spliced {len(cast)} cast lines into {recipe}')
|
||||
PYEOF
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""autoresearch-bench.py — deterministic greedy battery for frank-large.
|
||||
"""autoresearch-bench.py — deterministic greedy battery for frank-dragoon.
|
||||
|
||||
Runs ON Dankclaw (no LAN jitter) against the candidate container on :7770.
|
||||
Fixed embedded corpus, temperature 0, streaming so decode speed is measured
|
||||
@@ -104,7 +104,7 @@ def stream_completion(url: str, model: str, prompt: str):
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--url", default="http://127.0.0.1:7770")
|
||||
ap.add_argument("--model", default="frank-large")
|
||||
ap.add_argument("--model", default="frank-dragoon")
|
||||
args = ap.parse_args()
|
||||
|
||||
with urllib.request.urlopen(args.url.rstrip("/") + "/v1/models",
|
||||
|
||||
Executable
+187
@@ -0,0 +1,187 @@
|
||||
#!/usr/bin/env bash
|
||||
# autoresearch-frank-archon.sh — RUNS ON DANKSERVER, driven over ssh by the
|
||||
# dev-box autoresearch.sh. It is the canonical frank-archon benchmark entry
|
||||
# point for the autoresearch loop.
|
||||
#
|
||||
# Method (mandated by the Benchmarking Policy): the production service serves
|
||||
# real users and shares the two R9700s, so a bench beside it starves the GPUs
|
||||
# and corrupts results. This script therefore:
|
||||
# 1. stops the production frank-archon unit (root, relink symlink: stop,
|
||||
# NOT mask — mask refuses to replace a symlinked unit file),
|
||||
# 2. boots a scratch radiance container named `arbench` on the SAME two
|
||||
# cards with the candidate engine argv (the env file the loop edits),
|
||||
# 3. polls /health until the listener is live (it binds AFTER the full
|
||||
# load, so a bare curl is an honest gate),
|
||||
# 4. runs ONE self-contained completion request locally (127.0.0.1, no LAN
|
||||
# jitter): a fixed ~140k-char real-text prompt (~32k tokens) decoded for
|
||||
# 256 tokens, greedy. The engine's `timings` object reports the honest
|
||||
# prefill and decode rates for that single request.
|
||||
# 5. prints METRIC lines and tears the scratch down,
|
||||
# 6. restores production (systemctl start), via a trap so a failed bench
|
||||
# never leaves frank-archon down (the stale-probe-tail lesson).
|
||||
#
|
||||
# Determinism: the prompt is built from a fixed, sorted corpus of repo files
|
||||
# with a fixed random seed, so prompt_n is stable across runs. Sampling is
|
||||
# greedy (temperature 0), so the decode length is fixed; the only residual
|
||||
# nondeterminism is the wht6 lossy TP all-reduce, which is a constant across
|
||||
# runs and cancels in A/B comparisons.
|
||||
#
|
||||
# The scratch mirrors the production container contract exactly (serve-frank-
|
||||
# primary.sh): --network host, --init, seccomp=unconfined (io_uring for the
|
||||
# n-gram table), memlock unbounded (the ~132 GiB pinned ledger), /dev/kfd +
|
||||
# /dev/dri, the .rad ro, the state dir rw, the tune cache mount. The ONLY
|
||||
# differences: the container name (arbench, so prod's identity is untouched)
|
||||
# and a FRESH prefix-cache disk tier (arbench/, wiped per run) so the cold
|
||||
# prefill is cold — the prod tier at kvcache/frank-archon is left intact.
|
||||
set -uo pipefail
|
||||
|
||||
ENV_FILE="${1:-/home/dank/serverAppConfigs/scripts/frank-archon.env}"
|
||||
SCRATCH=arbench
|
||||
PORT=7770
|
||||
HEALTH="http://127.0.0.1:$PORT/health"
|
||||
COMPLETE="http://127.0.0.1:$PORT/v1/completions"
|
||||
BODY=/tmp/arbench-body.json
|
||||
LOG=/home/dank/logs/arbench.log
|
||||
|
||||
restore() {
|
||||
podman rm -f "$SCRATCH" >/dev/null 2>&1 || true
|
||||
sudo systemctl start frank-archon >/dev/null 2>&1 || true
|
||||
}
|
||||
trap restore EXIT
|
||||
|
||||
# --- source the candidate config (same file the production unit reads) ------
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
echo "no env file at $ENV_FILE" >&2; exit 1
|
||||
fi
|
||||
# shellcheck disable=SC1090
|
||||
set -a; . "$ENV_FILE"; set +a
|
||||
|
||||
# --- stop production; the scratch needs both cards to itself ----------------
|
||||
sudo systemctl stop frank-archon >/dev/null 2>&1 || true
|
||||
podman rm -f "$SCRATCH" >/dev/null 2>&1 || true
|
||||
|
||||
# --- fresh prefix-cache tier for a cold prefill (prod's tier untouched) ------
|
||||
FRESH_KV="$FRANK_ARCHON_STATE_DIR/kvcache/$SCRATCH"
|
||||
rm -rf "$FRESH_KV"; mkdir -p "$FRESH_KV"
|
||||
mkdir -p "$FRANK_ARCHON_STATE_DIR/tune"
|
||||
|
||||
# --- boot the scratch: the production run line, scratch name + fresh KV ------
|
||||
podman run -d --name "$SCRATCH" \
|
||||
--network host \
|
||||
--init \
|
||||
--security-opt seccomp=unconfined \
|
||||
--ulimit memlock=-1:-1 \
|
||||
--device /dev/kfd --device /dev/dri \
|
||||
-v "$FRANK_ARCHON_MODEL_DIR":/models:ro \
|
||||
-v "$FRANK_ARCHON_STATE_DIR":/data \
|
||||
-v "$FRANK_ARCHON_STATE_DIR/tune":/opt/radiance/share/radiance/tune \
|
||||
-v "$FRANK_ARCHON_TEMPLATE":/templates/template.jinja:ro \
|
||||
"$FRANK_ARCHON_IMAGE" \
|
||||
--model "/models/$FRANK_ARCHON_MODEL_FILE" \
|
||||
--host :: --port "$PORT" \
|
||||
--served-model-name "$FRANK_ARCHON_SERVED_NAME" \
|
||||
--override-chat-template /templates/template.jinja \
|
||||
--tp "$FRANK_ARCHON_TP" --tp-wire "$FRANK_ARCHON_TP_WIRE" \
|
||||
--max-num-seqs "$FRANK_ARCHON_MAX_SEQS" \
|
||||
--max-model-len "$FRANK_ARCHON_MAX_MODEL_LEN" \
|
||||
--placement "$FRANK_ARCHON_PLACEMENT" \
|
||||
--host-pool-mib "$FRANK_ARCHON_HOST_POOL_MIB" \
|
||||
--gpu-headroom-mib "$FRANK_ARCHON_GPU_HEADROOM_MIB" \
|
||||
--vram-weights-mib "$FRANK_ARCHON_VRAM_WEIGHTS_MIB" \
|
||||
--vram-kv-mib "$FRANK_ARCHON_VRAM_KV_MIB" \
|
||||
--kv-cache-dtype "$FRANK_ARCHON_KV_DTYPE" \
|
||||
--prefix-cache-host-mib "$FRANK_ARCHON_PREFIX_HOST_MIB" \
|
||||
--prefix-cache-dir "/data/kvcache/$SCRATCH" \
|
||||
--prefix-cache-disk-mib "$FRANK_ARCHON_PREFIX_DISK_MIB" \
|
||||
--ngram-placement "$FRANK_ARCHON_NGRAM_PLACEMENT" \
|
||||
--num-speculative-tokens "$FRANK_ARCHON_SPEC_TOKENS" \
|
||||
--max-num-batched-tokens "$FRANK_ARCHON_MAX_BATCHED_TOKENS" \
|
||||
${FRANK_ARCHON_EXTRA_ARGS:+$FRANK_ARCHON_EXTRA_ARGS} \
|
||||
> "$LOG" 2>&1
|
||||
if [ $? -ne 0 ]; then echo "scratch boot failed; see $LOG" >&2; exit 1; fi
|
||||
|
||||
# --- poll health (listener binds after the full load; cold boot ~110 s) ------
|
||||
ok=0
|
||||
for _ in $(seq 1 360); do
|
||||
code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 3 "$HEALTH" 2>/dev/null || echo 000)
|
||||
if [ "$code" = "200" ]; then ok=1; break; fi
|
||||
sleep 2
|
||||
done
|
||||
if [ "$ok" -ne 1 ]; then
|
||||
echo "scratch never reached health 200; tail of $LOG:" >&2
|
||||
tail -30 "$LOG" >&2; exit 1
|
||||
fi
|
||||
|
||||
# --- build the workload: a fixed ~140k-char real-text prompt (repo corpus) ---
|
||||
python3 - "$BODY" <<'PY'
|
||||
import json, sys, random, glob
|
||||
random.seed(1234)
|
||||
root = "/home/dank/serverAppConfigs"
|
||||
files = sorted(
|
||||
glob.glob(f"{root}/serverAppConfigs-docs/**/*.md", recursive=True)
|
||||
+ glob.glob(f"{root}/scripts/*.sh")
|
||||
+ glob.glob(f"{root}/scripts/*.py")
|
||||
+ glob.glob(f"{root}/AGENTS.md")
|
||||
)
|
||||
corpus = []
|
||||
total = 0
|
||||
for f in files:
|
||||
try:
|
||||
t = open(f, encoding="utf-8", errors="ignore").read()
|
||||
except Exception:
|
||||
continue
|
||||
if not t.strip():
|
||||
continue
|
||||
corpus.append(t)
|
||||
total += len(t)
|
||||
if total > 240000:
|
||||
break
|
||||
corpus = "\n".join(corpus)
|
||||
TGT = 140000
|
||||
parts, n = [], 0
|
||||
while n < TGT:
|
||||
start = random.randint(0, max(0, len(corpus) - 500))
|
||||
seg = corpus[start:start + random.randint(200, 500)]
|
||||
parts.append(seg)
|
||||
n += len(seg) + 1
|
||||
prompt = " ".join(parts)
|
||||
json.dump(
|
||||
{"model": "frank-archon", "prompt": prompt,
|
||||
"max_tokens": 256, "temperature": 0.0, "stream": False,
|
||||
"stop": ["<|im_end|>"]},
|
||||
open(sys.argv[1], "w"),
|
||||
)
|
||||
PY
|
||||
if [ ! -f "$BODY" ]; then echo "prompt build failed" >&2; exit 1; fi
|
||||
|
||||
# --- the bench request (prefill ~32k + decode 256, greedy) -------------------
|
||||
RESP=$(curl -s --max-time 600 -X POST "$COMPLETE" \
|
||||
-H 'Content-Type: application/json' -d "@$BODY")
|
||||
echo "$RESP" > /tmp/arbench-resp.json
|
||||
|
||||
# --- parse the timings into METRIC lines -------------------------------------
|
||||
python3 - <<'PY'
|
||||
import json, sys
|
||||
try:
|
||||
r = json.load(open("/tmp/arbench-resp.json"))
|
||||
except Exception as e:
|
||||
sys.stderr.write(f"unparseable response: {e}\n"); sys.exit(1)
|
||||
if "timings" not in r:
|
||||
sys.stderr.write(f"no timings in response: {json.dumps(r)[:500]}\n"); sys.exit(1)
|
||||
t = r["timings"]
|
||||
dtps = t.get("predicted_per_second")
|
||||
ptps = t.get("prompt_per_second")
|
||||
pms = t.get("prompt_ms")
|
||||
dn = t.get("draft_n", 0)
|
||||
da = t.get("draft_n_accepted", 0)
|
||||
if dtps is None:
|
||||
sys.stderr.write("predicted_per_second null (no decode produced)\n"); sys.exit(1)
|
||||
print(f"METRIC decode_tps={dtps:.4f}")
|
||||
if ptps is not None:
|
||||
print(f"METRIC prefill_tps={ptps:.4f}")
|
||||
if pms is not None:
|
||||
print(f"METRIC ttft_ms={pms:.4f}")
|
||||
if dn:
|
||||
print(f"METRIC draft_accept={da/dn:.4f}")
|
||||
print(f"INFO prompt_n={t.get('prompt_n')} cache_n={t.get('cache_n')} predicted_n={t.get('predicted_n')}")
|
||||
PY
|
||||
+32
-32
@@ -1,19 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
# frank-primary-arms.sh — run a SEQUENCE of frank-primary bench arms and keep
|
||||
# frank-archon-arms.sh — run a SEQUENCE of frank-archon bench arms and keep
|
||||
# every arm's evidence. Runs on the operator box; each arm is one full window
|
||||
# of ./autoresearch.sh (stop+mask -> boot the arm as the service -> measure ->
|
||||
# restore production and verify), so the fleet is only ever down for one arm at
|
||||
# a time and the production unit is back up between arms.
|
||||
#
|
||||
# An arm is the repo's own frank-primary.env plus an OVERLAY of engine knobs
|
||||
# An arm is the repo's own frank-archon.env plus an OVERLAY of engine knobs
|
||||
# appended after it (later assignment wins when the serve script sources it),
|
||||
# staged through FRANK_PRIMARY_AR_ENV_FILE so the checkout is never mutated and
|
||||
# staged through FRANK_ARCHON_AR_ENV_FILE so the checkout is never mutated and
|
||||
# each arm gets its own fingerprint (the memo can never answer one arm with
|
||||
# another arm's numbers).
|
||||
#
|
||||
# Usage: scripts/autoresearch/frank-primary-arms.sh [arm ...]
|
||||
# scripts/autoresearch/frank-primary-arms.sh --list
|
||||
# Output, under /tmp/frank-primary-arms-<UTC stamp>/ (override with ARMS_DIR):
|
||||
# Usage: scripts/autoresearch/frank-archon-arms.sh [arm ...]
|
||||
# scripts/autoresearch/frank-archon-arms.sh --list
|
||||
# Output, under /tmp/frank-archon-arms-<UTC stamp>/ (override with ARMS_DIR):
|
||||
#
|
||||
# <arm>.out full autoresearch output (metrics + engine evidence)
|
||||
# <arm>.env the exact staged env
|
||||
@@ -26,8 +26,8 @@ set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO="$(cd -- "$SCRIPT_DIR/../.." && pwd)"
|
||||
BASE_ENV="$REPO/scripts/frank-primary.env"
|
||||
ARMS_DIR="${ARMS_DIR:-/tmp/frank-primary-arms-$(date -u +%Y%m%dT%H%M%SZ)}"
|
||||
BASE_ENV="$REPO/scripts/frank-archon.env"
|
||||
ARMS_DIR="${ARMS_DIR:-/tmp/frank-archon-arms-$(date -u +%Y%m%dT%H%M%SZ)}"
|
||||
|
||||
# --- arms ------------------------------------------------------------------
|
||||
# baseline: the shipped shape E, measured with THIS harness so the comparison is
|
||||
@@ -45,18 +45,18 @@ LADDER3='{"cudagraph_capture_sizes":[4,8,12],"max_cudagraph_capture_size":12,"in
|
||||
# the engine as {temperature:0.7}.
|
||||
GEN07='{"temperature":0.7}'
|
||||
OVER_B=(
|
||||
"FRANK_PRIMARY_SPEC='$SPEC3'"
|
||||
"FRANK_PRIMARY_COMPCONFIG='$LADDER3'"
|
||||
'FRANK_PRIMARY_GEN_CONFIG='"'$GEN07'"
|
||||
"FRANK_ARCHON_SPEC='$SPEC3'"
|
||||
"FRANK_ARCHON_COMPCONFIG='$LADDER3'"
|
||||
'FRANK_ARCHON_GEN_CONFIG='"'$GEN07'"
|
||||
)
|
||||
# B2: identical to B with the default (7.60 GiB) overhead, i.e. precise memory
|
||||
# OFF. It is the fallback if B dies at graph capture, and it prices the arena
|
||||
# that precise memory buys (7,522 slots -> ~6,496).
|
||||
OVER_B2=(
|
||||
"FRANK_PRIMARY_SPEC='$SPEC3'"
|
||||
"FRANK_PRIMARY_COMPCONFIG='$LADDER3'"
|
||||
'FRANK_PRIMARY_GEN_CONFIG='"'$GEN07'"
|
||||
'FRANK_PRIMARY_PRECISE_MEMORY='
|
||||
"FRANK_ARCHON_SPEC='$SPEC3'"
|
||||
"FRANK_ARCHON_COMPCONFIG='$LADDER3'"
|
||||
'FRANK_ARCHON_GEN_CONFIG='"'$GEN07'"
|
||||
'FRANK_ARCHON_PRECISE_MEMORY='
|
||||
)
|
||||
# C: the vendor's concurrency shape — SEQS 16 with the [4..32] ladder and util
|
||||
# 0.95. This is also the direct test of the theory that pinning the capture
|
||||
@@ -64,12 +64,12 @@ OVER_B2=(
|
||||
# the constant 3, which is NOT in the default ladder; 4 is).
|
||||
LADDER16='{"cudagraph_capture_sizes":[4,8,12,16,20,24,28,32],"max_cudagraph_capture_size":32,"inductor_compile_config":{"combo_kernels":false,"benchmark_combo_kernel":false}}'
|
||||
OVER_C=(
|
||||
"FRANK_PRIMARY_SPEC='$SPEC3'"
|
||||
"FRANK_PRIMARY_COMPCONFIG='$LADDER16'"
|
||||
'FRANK_PRIMARY_GEN_CONFIG='"'$GEN07'"
|
||||
'FRANK_PRIMARY_SEQS=16'
|
||||
'FRANK_PRIMARY_MNBT=4096'
|
||||
'FRANK_PRIMARY_UTIL=0.95'
|
||||
"FRANK_ARCHON_SPEC='$SPEC3'"
|
||||
"FRANK_ARCHON_COMPCONFIG='$LADDER16'"
|
||||
'FRANK_ARCHON_GEN_CONFIG='"'$GEN07'"
|
||||
'FRANK_ARCHON_SEQS=16'
|
||||
'FRANK_ARCHON_MNBT=4096'
|
||||
'FRANK_ARCHON_UTIL=0.95'
|
||||
)
|
||||
|
||||
# D/E: the CONTEXT arms. Measured 2026-10-02 (arms run 20261002T233855Z): the
|
||||
@@ -83,11 +83,11 @@ OVER_C=(
|
||||
# D is arm B with EXACTLY ONE variable changed (the KV bytes), so its result
|
||||
# says whether the 4k-depth collapse (B 23.09 vs baseline 33.99) was KV or the
|
||||
# batch shape.
|
||||
KV401K='FRANK_PRIMARY_KV_BYTES=4050000000'
|
||||
KV401K='FRANK_ARCHON_KV_BYTES=4050000000'
|
||||
OVER_D=(
|
||||
"FRANK_PRIMARY_SPEC='$SPEC3'"
|
||||
"FRANK_PRIMARY_COMPCONFIG='$LADDER3'"
|
||||
'FRANK_PRIMARY_GEN_CONFIG='"'$GEN07'"
|
||||
"FRANK_ARCHON_SPEC='$SPEC3'"
|
||||
"FRANK_ARCHON_COMPCONFIG='$LADDER3'"
|
||||
'FRANK_ARCHON_GEN_CONFIG='"'$GEN07'"
|
||||
"$KV401K"
|
||||
)
|
||||
# E: D with C's max_num_batched_tokens (4096 — C is the only arm that did NOT
|
||||
@@ -97,12 +97,12 @@ OVER_D=(
|
||||
# only 0.32 GiB above the allowance while the CUDA-graph cost is an ANALYTIC
|
||||
# estimate (C's was 1.47 GiB over 24 graphs), so it is measured, not assumed.
|
||||
OVER_E=(
|
||||
"FRANK_PRIMARY_SPEC='$SPEC3'"
|
||||
"FRANK_PRIMARY_COMPCONFIG='$LADDER3'"
|
||||
'FRANK_PRIMARY_GEN_CONFIG='"'$GEN07'"
|
||||
"FRANK_ARCHON_SPEC='$SPEC3'"
|
||||
"FRANK_ARCHON_COMPCONFIG='$LADDER3'"
|
||||
'FRANK_ARCHON_GEN_CONFIG='"'$GEN07'"
|
||||
"$KV401K"
|
||||
'FRANK_PRIMARY_MNBT=4096'
|
||||
'FRANK_PRIMARY_UTIL=0.99'
|
||||
'FRANK_ARCHON_MNBT=4096'
|
||||
'FRANK_ARCHON_UTIL=0.99'
|
||||
)
|
||||
|
||||
arm_overlay() {
|
||||
@@ -142,7 +142,7 @@ for arm in "${ARMS[@]}"; do
|
||||
# holds JSON must still be one valid JSON object after the file is SOURCED
|
||||
# (that is the step that silently eats unquoted braces).
|
||||
if ! bash -c 'set -a; . "$1" >/dev/null; set +a
|
||||
for k in FRANK_PRIMARY_SPEC FRANK_PRIMARY_COMPCONFIG FRANK_PRIMARY_GEN_CONFIG; do
|
||||
for k in FRANK_ARCHON_SPEC FRANK_ARCHON_COMPCONFIG FRANK_ARCHON_GEN_CONFIG; do
|
||||
v="${!k:-}"
|
||||
[ -n "$v" ] || continue
|
||||
case "$v" in
|
||||
@@ -158,7 +158,7 @@ for arm in "${ARMS[@]}"; do
|
||||
fi
|
||||
|
||||
echo "[arms] === $arm: opening window $(date -u +%H:%M:%S)Z ===" >&2
|
||||
FRANK_PRIMARY_AR_ENV_FILE="$ARMS_DIR/$arm.env" \
|
||||
FRANK_ARCHON_AR_ENV_FILE="$ARMS_DIR/$arm.env" \
|
||||
"$REPO/autoresearch.sh" >"$ARMS_DIR/$arm.out" 2>&1
|
||||
rc=$?
|
||||
echo "$rc" >"$ARMS_DIR/$arm.rc"
|
||||
+3
-3
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""frank-primary decode measurement — runs INSIDE the engine container.
|
||||
"""frank-archon decode measurement — runs INSIDE the engine container.
|
||||
|
||||
The method is the fleet's established one (the 2026-09-29 / 2026-10-01
|
||||
tuning campaigns, /home/dank/mtp_measure.py): SSE chunk counting is invalid
|
||||
@@ -28,7 +28,7 @@ Workload (fixed, deterministic, offline):
|
||||
|
||||
Env:
|
||||
AR_BASE endpoint (default http://127.0.0.1:7770)
|
||||
AR_MODEL served name (default frank-primary)
|
||||
AR_MODEL served name (default frank-archon)
|
||||
AR_PROMPT_CEILING hard prompt ceiling in tokens (default 100000)
|
||||
AR_TEMPS comma-separated sampling temperatures (default "0,0.7")
|
||||
|
||||
@@ -45,7 +45,7 @@ import urllib.error
|
||||
import urllib.request
|
||||
|
||||
BASE = os.environ.get("AR_BASE", "http://127.0.0.1:7770")
|
||||
MODEL = os.environ.get("AR_MODEL", "frank-primary")
|
||||
MODEL = os.environ.get("AR_MODEL", "frank-archon")
|
||||
PROMPT_CEILING = int(os.environ.get("AR_PROMPT_CEILING", "100000"))
|
||||
DEPTHS = (115, 1056, 4096)
|
||||
WARM_TOKENS = 64
|
||||
+14
-14
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# frank-primary-window.sh — one full bench window for frank-primary, run ON
|
||||
# frank-archon-window.sh — one full bench window for frank-archon, run ON
|
||||
# DANKSERVER as the deploy user (invoked over ssh by ./autoresearch.sh).
|
||||
#
|
||||
# Discipline (AGENTS.md Benchmarking Policy): the production unit is
|
||||
@@ -12,9 +12,9 @@
|
||||
#
|
||||
# Staging layout (created by the orchestrator, ephemeral — never a config
|
||||
# source; the repo checkout is):
|
||||
# /home/dank/.autoresearch-frank-primary/
|
||||
# /home/dank/.autoresearch-frank-archon/
|
||||
# serve.sh staged candidate serve script
|
||||
# frank-primary.env staged candidate env (the candidate knobs)
|
||||
# frank-archon.env staged candidate env (the candidate knobs)
|
||||
# measure.py the in-container measure client
|
||||
# hash candidate fingerprint
|
||||
# run.json result (metrics + boot + kv + fingerprint)
|
||||
@@ -27,16 +27,16 @@
|
||||
# outcome; a failed restore forces exit 1).
|
||||
set -uo pipefail
|
||||
|
||||
STAGE="${FRANK_PRIMARY_AR_STAGE:-/home/dank/.autoresearch-frank-primary}"
|
||||
UNIT="${FRANK_PRIMARY_AR_UNIT:-frank-primary}"
|
||||
CTR="${FRANK_PRIMARY_AR_CTR:-frank-primary-qwen}"
|
||||
STAGE="${FRANK_ARCHON_AR_STAGE:-/home/dank/.autoresearch-frank-archon}"
|
||||
UNIT="${FRANK_ARCHON_AR_UNIT:-frank-archon}"
|
||||
CTR="${FRANK_ARCHON_AR_CTR:-frank-archon-qwen}"
|
||||
HEALTH_URL="http://127.0.0.1:7770/health"
|
||||
MODELS_URL="http://127.0.0.1:7770/v1/models"
|
||||
BOOT_TIMEOUT="${FRANK_PRIMARY_AR_BOOT_TIMEOUT:-1200}"
|
||||
RESTORE_TIMEOUT="${FRANK_PRIMARY_AR_RESTORE_TIMEOUT:-1200}"
|
||||
MEASURE_TIMEOUT="${FRANK_PRIMARY_AR_MEASURE_TIMEOUT:-1800}"
|
||||
SERVED="$(sed -n 's/^FRANK_PRIMARY_SERVED_NAME=//p' "$STAGE/frank-primary.env" 2>/dev/null | head -1)"
|
||||
SERVED="${SERVED:-frank-primary}"
|
||||
BOOT_TIMEOUT="${FRANK_ARCHON_AR_BOOT_TIMEOUT:-1200}"
|
||||
RESTORE_TIMEOUT="${FRANK_ARCHON_AR_RESTORE_TIMEOUT:-1200}"
|
||||
MEASURE_TIMEOUT="${FRANK_ARCHON_AR_MEASURE_TIMEOUT:-1800}"
|
||||
SERVED="$(sed -n 's/^FRANK_ARCHON_SERVED_NAME=//p' "$STAGE/frank-archon.env" 2>/dev/null | head -1)"
|
||||
SERVED="${SERVED:-frank-archon}"
|
||||
|
||||
RESTORED=0
|
||||
|
||||
@@ -72,7 +72,7 @@ capture_bench_failure() {
|
||||
echo "=== staged serve.out (tail 60) ==="
|
||||
tail -60 "$STAGE/serve.out" 2>/dev/null || true
|
||||
echo "=== engine log (tail 60) ==="
|
||||
tail -60 /home/dank/logs/frank_primary.log 2>/dev/null || true
|
||||
tail -60 /home/dank/logs/frank_archon.log 2>/dev/null || true
|
||||
} >"$STAGE/fail.log" 2>&1 || true
|
||||
grep -E 'OutOfMemory|Tried to allocate|Engine core initialization failed|Error|error' \
|
||||
"$STAGE/fail.log" 2>/dev/null | tail -25 >&2 || true
|
||||
@@ -114,7 +114,7 @@ trap 'restore; rc=$?; [ "$rc" -ne 0 ] && exit 1' EXIT
|
||||
|
||||
main() {
|
||||
local f
|
||||
for f in serve.sh frank-primary.env measure.py hash; do
|
||||
for f in serve.sh frank-archon.env measure.py hash; do
|
||||
[ -f "$STAGE/$f" ] || { log "staged file missing: $STAGE/$f"; exit 1; }
|
||||
done
|
||||
mkdir -p "$STAGE/last"
|
||||
@@ -133,7 +133,7 @@ main() {
|
||||
|
||||
t0="$(date +%s)"
|
||||
log "starting bench instance from staged config (fingerprint $(tr -d '[:space:]' <"$STAGE/hash"))"
|
||||
FRANK_PRIMARY_ENV_FILE="$STAGE/frank-primary.env" \
|
||||
FRANK_ARCHON_ENV_FILE="$STAGE/frank-archon.env" \
|
||||
setsid nohup bash "$STAGE/serve.sh" >"$STAGE/serve.out" 2>&1 </dev/null &
|
||||
echo "$!" >"$STAGE/serve.pid"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Decode throughput benchmark for frank-large.
|
||||
# Decode throughput benchmark for frank-dragoon.
|
||||
# Measures generation tok/s with 8K+ token prompts and 256 generated tokens.
|
||||
set -euo pipefail
|
||||
|
||||
@@ -57,7 +57,7 @@ run_bench() {
|
||||
# Warmup (small request)
|
||||
curl -s -m 60 http://127.0.0.1:$PORT/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model":"frank-large","messages":[{"role":"user","content":"Hi"}],"max_tokens":16,"stream":false}' > /dev/null 2>&1 || true
|
||||
-d '{"model":"frank-dragoon","messages":[{"role":"user","content":"Hi"}],"max_tokens":16,"stream":false}' > /dev/null 2>&1 || true
|
||||
|
||||
# Build 8K prompt
|
||||
local prompt
|
||||
@@ -75,7 +75,7 @@ print(json.dumps(' '.join(words)))
|
||||
t0=$(date +%s%N)
|
||||
resp=$(curl -s -m 600 http://127.0.0.1:$PORT/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"model\":\"frank-large\",\"messages\":[{\"role\":\"user\",\"content\":$prompt}],\"max_tokens\":$N_GEN,\"stream\":false}" 2>&1)
|
||||
-d "{\"model\":\"frank-dragoon\",\"messages\":[{\"role\":\"user\",\"content\":$prompt}],\"max_tokens\":$N_GEN,\"stream\":false}" 2>&1)
|
||||
t1=$(date +%s%N)
|
||||
|
||||
if ! echo "$resp" | python3 -c "import sys,json; json.load(sys.stdin)" 2>/dev/null; then
|
||||
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""bench-probe.py — RUNS ON DANKSERVER. The campaign's decode probe: one
|
||||
/completion request, a prompt assembled from calib.jsonl to ~TARGET tokens,
|
||||
temperature 0, max_tokens 512. Reports the engine's own timings: decode rate,
|
||||
prefill rate, draft accept. Run 3x; the median is the arm's number."""
|
||||
import json, sys, urllib.request
|
||||
|
||||
TARGET = 43600 # tokens
|
||||
BYTES_PER_TOK = 3.8
|
||||
URL = "http://localhost:7770/v1/completions"
|
||||
|
||||
def build_prompt():
|
||||
want = int(TARGET * BYTES_PER_TOK)
|
||||
parts = []
|
||||
n = 0
|
||||
with open("/home/dank/corpora/out/calib.jsonl") as f:
|
||||
for line in f:
|
||||
d = json.loads(line)
|
||||
p = d.get("prompt") or d.get("messages", [{}])[0].get("content", "")
|
||||
parts.append(p)
|
||||
n += len(p.encode())
|
||||
if n >= want:
|
||||
break
|
||||
return "".join(parts)
|
||||
|
||||
def probe(salt):
|
||||
prompt = f"[probe {salt}]\n" + build_prompt()
|
||||
body = json.dumps({
|
||||
"model": "frank-archon",
|
||||
"prompt": prompt, "temperature": 0.0, "max_tokens": 512,
|
||||
}).encode()
|
||||
req = urllib.request.Request(URL, data=body,
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=1800) as r:
|
||||
d = json.load(r)
|
||||
t = d.get("timings", {})
|
||||
return {
|
||||
"prompt_n": t.get("prompt_n"),
|
||||
"prefill_tps": round(t.get("prompt_per_second", 0), 1),
|
||||
"pred_n": t.get("predicted_n"),
|
||||
"decode_tps": round(t.get("predicted_per_second", 0), 2),
|
||||
"draft_n": t.get("draft_n"),
|
||||
"accept": (round(t["draft_n_accepted"] / t["draft_n"], 4)
|
||||
if t.get("draft_n") else None),
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(json.dumps(probe(sys.argv[1] if len(sys.argv) > 1 else "a")))
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
HTTP-based prefill-throughput sweep for frank-large.
|
||||
HTTP-based prefill-throughput sweep for frank-dragoon.
|
||||
Starts the server with a given --threads-batch value, sends a 64K-prompt via
|
||||
the chat API (max_tokens=0), measures prompt_tokens/wall_time.
|
||||
|
||||
@@ -39,7 +39,7 @@ def send_prefill_request():
|
||||
with open(PROMPT_FILE, "r") as f:
|
||||
prompt = f.read()
|
||||
body = json.dumps({
|
||||
"model": "frank-large",
|
||||
"model": "frank-dragoon",
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": 1,
|
||||
"stream": False
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Threads-batch sweep for frank-large on dankclaw
|
||||
# Threads-batch sweep for frank-dragoon on dankclaw
|
||||
# Benches prefill throughput at 65536 prompt tokens across threads-batch values.
|
||||
set -uo pipefail
|
||||
|
||||
@@ -10,7 +10,7 @@ TB_LIST=(16 32 48 64 80 96 128)
|
||||
ENV_FILE="/home/dank/GitPackages/ROCmFPX/build-qwen38/bin"
|
||||
|
||||
echo "========================================"
|
||||
echo "THREADS-BATCH SWEEP — frank-large"
|
||||
echo "THREADS-BATCH SWEEP — frank-dragoon"
|
||||
echo "Model: $(basename "$MODEL") ($(du -h "$MODEL" | cut -f1))"
|
||||
echo "Threads-batch values: ${TB_LIST[*]}"
|
||||
echo "Prompt: 65536 tokens, n-gen: 0 (prefill-only)"
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# Benchmark ubatch with llama-bench for frank-large.
|
||||
# Benchmark ubatch with llama-bench for frank-dragoon.
|
||||
#
|
||||
# This is intentionally a ubatch-only benchmark. It uses one llama-bench
|
||||
# process, an 8K prompt workload, and equal batch/ubatch values. No server
|
||||
# process, HTTP requests, or log scraping are used.
|
||||
#
|
||||
# Run on Dankclaw after stopping frank-large:
|
||||
# sudo systemctl stop frank-large
|
||||
# Run on Dankclaw after stopping frank-dragoon:
|
||||
# sudo systemctl stop frank-dragoon
|
||||
# ./scripts/bench-ubatch.sh
|
||||
# sudo systemctl start frank-large
|
||||
# sudo systemctl start frank-dragoon
|
||||
set -euo pipefail
|
||||
|
||||
: "${LLAMA_BENCH:=/home/dank/GitPackages/ROCmFPX/build-strix-rocmfp4/bin/llama-bench}"
|
||||
|
||||
Executable
+166
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env bash
|
||||
# build-frank-archon-rad.sh — build frank-archon's .rad container from the
|
||||
# OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 checkpoint, at NATIVE precision.
|
||||
#
|
||||
# bash scripts/build-frank-archon-rad.sh # convert
|
||||
# bash scripts/build-frank-archon-rad.sh --plan-only # declare, no I/O
|
||||
# bash scripts/build-frank-archon-rad.sh --restage # redo staging
|
||||
# bash scripts/build-frank-archon-rad.sh --force # rebuild existing
|
||||
#
|
||||
# WHY A STAGING DIR (the one non-obvious step): rad-convert selects its
|
||||
# architecture plugin by the checkpoint's quantisation DESCRIPTOR —
|
||||
# derived from config.json's quantization_config (core/format/checkpoint.cpp:
|
||||
# "fp8" here). No shipped plugin claims (qwen4exp, "fp8"): the shipped
|
||||
# qwen4exp plugin claims (qwen4exp, ""), the descriptor recipe-built
|
||||
# containers carry (they are built from the bf16 upstream, which has no
|
||||
# quantization_config at all). The match is exact with no fallback, so an
|
||||
# unmodified convert refuses before declare. The descriptor is only a
|
||||
# selection key; per-weight encodings are data — the checkpoint reader
|
||||
# derives them from the tensors themselves (X.weight_scale_inv is the
|
||||
# scale plane; checkpoint.h). Staging drops the quantization_config key
|
||||
# (everything else is a relative symlink into the pristine download) so
|
||||
# the convert selects the shipped plugin. EVERY weight is then copied through
|
||||
# in its own encoding — block-FP8 codes + bf16 scale planes stay block-FP8 —
|
||||
# except TWO value-preserving reshapes that stack-qwen38-official-fp8.py
|
||||
# performs on the staging dir first, and ONE recipe rule that finishes the
|
||||
# PLE table:
|
||||
# * experts: the plugin maps the STACKED layout (mlp.experts.gate_up_proj
|
||||
# [E,2F,H] with the _scale_inv plane at the same index) while the official
|
||||
# repo ships per-expert unfused tensors — staging byte-stacks them
|
||||
# (gate rows first, no requantisation).
|
||||
# * PLE: the plugin's fp8 branch wants the table codes beside a ONE-ELEMENT
|
||||
# bf16 scale plane; the official repo ships 128 SCALELESS fp8 shards plus
|
||||
# one bf16 scalar. Staging decodes to bf16 with the checkpoint's own
|
||||
# scale, and the convert's rtn rule re-encodes at that same scale —
|
||||
# every original code is restored exactly and the scale plane lands in
|
||||
# the form the plugin declares.
|
||||
#
|
||||
# WHY A CONTAINER AT ALL: a checkpoint dir has no mapped/file tiers, and a
|
||||
# weight only a host kernel reads (the n-gram / PLE table, 51B params)
|
||||
# cannot be served from one (GUIDE §3.4). rad-convert also bakes the
|
||||
# tokenizer and the checkpoint's chat template; the runtime replaces the
|
||||
# template with froggeric's (--override-chat-template).
|
||||
#
|
||||
# The convert runs rad-convert inside the digest-pinned radiance image
|
||||
# (the host has no ROCm and needs none): /home/dank/models is mounted at
|
||||
# /models so the staging dir's RELATIVE symlinks resolve inside the
|
||||
# container. Output lands beside the raw download, owned by dank
|
||||
# (rootless podman maps container-root to the invoking user).
|
||||
set -euo pipefail
|
||||
|
||||
SRC="/home/dank/models/Qwen/Qwen3.8-Flash-Next-FP8"
|
||||
STAGE="/home/dank/models/stage-qwen4exp-fp8"
|
||||
OUT_DIR="/home/dank/models/rad"
|
||||
OUT="$OUT_DIR/qwen4exp-fp8-native.rad"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/frank-archon.env"
|
||||
IMAGE="$FRANK_ARCHON_IMAGE"
|
||||
|
||||
plan_only=0 restage=0 force=0
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
--plan-only) plan_only=1 ;;
|
||||
--restage) restage=1 ;;
|
||||
--force) force=1 ;;
|
||||
*) echo "unknown flag: $a" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ ! -f "$SRC/config.json" ]; then
|
||||
echo "ERROR: raw checkpoint missing: $SRC (run scripts/download-frank-archon.sh --fetch)" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "verifying raw checkpoint first..."
|
||||
bash "$SCRIPT_DIR/download-frank-archon.sh"
|
||||
|
||||
if [ -f "$OUT" ] && [ "$plan_only" -eq 0 ] && [ "$force" -eq 0 ]; then
|
||||
echo "container already exists: $OUT"
|
||||
echo "rebuild it with --force (the old file moves to ~/.AGENTTRASH)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ---- staging ----------------------------------------------------------------
|
||||
if [ "$restage" -eq 1 ] && [ -e "$STAGE" ]; then
|
||||
TRASH="$HOME/.AGENTTRASH/$(date +%Y%m%d)-frank-archon-stage"
|
||||
mkdir -p "$TRASH"
|
||||
mv "$STAGE" "$TRASH/stage-$(date +%H%M%S)"
|
||||
echo "old staging dir moved to $TRASH"
|
||||
fi
|
||||
if [ ! -f "$STAGE/config.json" ]; then
|
||||
mkdir -p "$STAGE"
|
||||
python3 - "$SRC/config.json" "$STAGE/config.json" <<'PY'
|
||||
import json, sys
|
||||
src, dst = sys.argv[1], sys.argv[2]
|
||||
cfg = json.load(open(src))
|
||||
dropped = cfg.pop("quantization_config", None)
|
||||
json.dump(cfg, open(dst, "w"), indent=2)
|
||||
print(f"staging config.json written (quantization_config dropped: {dropped is not None})")
|
||||
PY
|
||||
# The index is COPIED like config.json, never symlinked: the reshape step
|
||||
# rewrites the stage's index, and a symlink would write through to the
|
||||
# raw checkpoint (measured 2026-10-05: the raw index was restamped with
|
||||
# the stacked layout and the raw verify failed 143/144). Shards stay
|
||||
# relative symlinks — relative so the targets resolve inside the convert
|
||||
# container (the stage's parent is mounted at /models).
|
||||
for f in "$SRC"/*; do
|
||||
name="$(basename "$f")"
|
||||
case "$name" in config.json|model.safetensors.index.json) continue ;; esac
|
||||
ln -sfn "../Qwen/Qwen3.8-Flash-Next-FP8/$name" "$STAGE/$name"
|
||||
done
|
||||
cp "$SRC/model.safetensors.index.json" "$STAGE/"
|
||||
echo "staging dir ready: $STAGE ($(ls "$STAGE" | wc -l) entries)"
|
||||
fi
|
||||
|
||||
# ---- checkpoint reshape (stacked experts + bf16 PLE table) -------------------
|
||||
# stack-qwen38-official-fp8.py reshapes the staged checkpoint in place
|
||||
# (idempotent) and writes the PLE scale to $STAGE/ple_scale.txt — see the
|
||||
# script's docstring for why each transform is value-preserving.
|
||||
echo "stacking experts + merging the PLE table (value-preserving)..."
|
||||
python3 "$SCRIPT_DIR/stack-qwen38-official-fp8.py" --stage "$STAGE"
|
||||
PLE_SCALE="$(tr -d '\n' < "$STAGE/ple_scale.txt")"
|
||||
echo "PLE fixed scale: $PLE_SCALE"
|
||||
|
||||
# ---- convert ----------------------------------------------------------------
|
||||
# rad-convert needs no GPU: declare resolves kernels and asks each about the
|
||||
# encoding it will read — CPU-side layout questions only. (If a plugin load
|
||||
# ever demands a card, add --device /dev/kfd --device /dev/dri to this run.)
|
||||
# --plan-only runs the same declare phase and exits before a byte is written.
|
||||
echo "rad-convert: $SRC (staged) -> $OUT"
|
||||
mkdir -p "$OUT_DIR"
|
||||
CONVERT_ARGS=(/models/stage-qwen4exp-fp8 -o /models/rad/qwen4exp-fp8-native.rad
|
||||
--home /opt/radiance/share/radiance)
|
||||
[ "$plan_only" -eq 1 ] && CONVERT_ARGS+=(--plan-only -v)
|
||||
# THE ONE RECIPE RULE (see header): the PLE table arrives from staging as a
|
||||
# bf16 merge of the checkpoint's own fp8 codes decoded with the checkpoint's
|
||||
# own scale; rtn re-encodes it at that same scale — the codes come back
|
||||
# exactly and the plugin's one-element scale plane is produced.
|
||||
CONVERT_ARGS+=(--quant "blk.*.ple_ngram.weight=rtn:codes=fp8_e4m3,block=*x*,scale=bf16,rule=fixed,scale_value=$PLE_SCALE")
|
||||
[ "$force" -eq 1 ] && [ -f "$OUT" ] && {
|
||||
TRASH="$HOME/.AGENTTRASH/$(date +%Y%m%d)-frank-archon-rad"
|
||||
mkdir -p "$TRASH"
|
||||
mv "$OUT" "$TRASH/qwen4exp-fp8-native.rad.$(date +%H%M%S)"
|
||||
echo "previous container moved to $TRASH"
|
||||
}
|
||||
podman run --rm \
|
||||
--entrypoint rad-convert \
|
||||
-v /home/dank/models:/models \
|
||||
"$IMAGE" "${CONVERT_ARGS[@]}"
|
||||
|
||||
if [ "$plan_only" -eq 1 ]; then
|
||||
echo "plan-only run complete (no container written)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ---- record -----------------------------------------------------------------
|
||||
echo "rad-info summary:"
|
||||
podman run --rm --entrypoint rad-info -v /home/dank/models:/models:ro \
|
||||
"$IMAGE" "/models/rad/qwen4exp-fp8-native.rad" | head -20
|
||||
MIB=$(( $(stat -c %s "$OUT") / 1048576 ))
|
||||
echo "container: $OUT (${MIB} MiB; env floor $FRANK_ARCHON_RAD_MIN_MIB MiB)"
|
||||
if [ "$MIB" -lt "$FRANK_ARCHON_RAD_MIN_MIB" ]; then
|
||||
echo "ERROR: container below the env floor — the convert was truncated" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "build complete"
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the gufo runtime that serves frank-large on Dankclaw.
|
||||
# Build the gufo runtime that serves frank-dragoon on Dankclaw.
|
||||
#
|
||||
# bash scripts/build-gufo.sh # build + install the pinned tag
|
||||
# bash scripts/build-gufo.sh --verify # only check the installed tree
|
||||
@@ -93,4 +93,4 @@ cmake --install build/release
|
||||
|
||||
echo "== verify =="
|
||||
"$PREFIX/bin/gufo" diagnose
|
||||
echo "frank-large runtime ready: $PREFIX/bin/gufo ($GUFO_TAG @ $commit)"
|
||||
echo "frank-dragoon runtime ready: $PREFIX/bin/gufo ($GUFO_TAG @ $commit)"
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the ROCmFPX runtime that serves frank-large on Dankclaw.
|
||||
# Build the ROCmFPX runtime that serves frank-dragoon on Dankclaw.
|
||||
#
|
||||
# bash scripts/build-rocmfpx.sh # clone/fetch + build the pin
|
||||
# bash scripts/build-rocmfpx.sh --verify # only check the installed tree
|
||||
#
|
||||
# Runtime: kingjones30/ROCmFPX — charlie12345/ROCmFPX plus the model-card
|
||||
# patches, i.e. the fork that owns the ROCmFPx tensor formats. frank-large's
|
||||
# patches, i.e. the fork that owns the ROCmFPx tensor formats. frank-dragoon's
|
||||
# payload is ftype 115 (`Q8_0_ROCMFPX_AGENT`): 311 tensors of
|
||||
# GGML_TYPE_Q8_0_ROCMFPX (103), a type that exists ONLY in this fork, so
|
||||
# stock llama.cpp, gufo and every other loader on the host reject the file.
|
||||
@@ -30,7 +30,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
ROCMFPX_REPO="${ROCMFPX_REPO:-https://github.com/kingjones30/ROCmFPX.git}"
|
||||
# The commit frank-large was qualified with (measured 2026-10-02).
|
||||
# The commit frank-dragoon was qualified with (measured 2026-10-02).
|
||||
ROCMFPX_COMMIT="${ROCMFPX_COMMIT:-36e9acd40e10a87cd3c3ef8ec734668757dc8520}"
|
||||
SRC_DIR="${ROCMFPX_SRC_DIR:-/home/dank/GitPackages/kingjones-ROCmFPX}"
|
||||
BUILD_DIR="${ROCMFPX_BUILD_DIR:-$SRC_DIR/build}"
|
||||
@@ -86,4 +86,4 @@ cmake --build "$BUILD_DIR" --target llama-server llama-quantize -j "$JOBS"
|
||||
|
||||
echo "== verify =="
|
||||
"$BUILD_DIR/bin/llama-server" --version | head -1
|
||||
echo "frank-large runtime ready: $BUILD_DIR/bin/llama-server ($ROCMFPX_COMMIT)"
|
||||
echo "frank-dragoon runtime ready: $BUILD_DIR/bin/llama-server ($ROCMFPX_COMMIT)"
|
||||
|
||||
Executable
+110
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env python3
|
||||
"""calib-push.py — drive a calibration corpus through a radiance scratch
|
||||
server's prefill so its tap writes layer Hessians (docs/MOE-W4.md).
|
||||
|
||||
Runs ON DANKSERVER next to the scratch server (the one autoresearch- /
|
||||
serve-style scratch container booted with RADIANCE_CALIB_DIR set).
|
||||
Reads the corpus JSONL produced by make-calib-corpus.py
|
||||
({"prompt": "..."} a line), POSTs each prompt to /v1/completions with
|
||||
max_tokens=1 (prefill-only cost, one token out), at N-way concurrency
|
||||
matching the scratch server's --max-num-seqs budget.
|
||||
|
||||
Corpus rounds repeat until the engine prints the tap's completion line
|
||||
("calibration wrote layer Hessians") in `podman logs <container>` — that
|
||||
is the ground truth that RADIANCE_CALIB_TOKENS was reached. After that
|
||||
line, further prompts are pure waste, so the driver exits.
|
||||
|
||||
Mechanics notes:
|
||||
- requests carry no sampling params that matter (max_tokens=1, greedy).
|
||||
- a per-request timeout of 30 min: at ~1k tok/s prefill a 12k-token
|
||||
prompt is a few seconds; the timeout only catches a wedged engine.
|
||||
- failures print and continue: the tap pools over tokens, a few lost
|
||||
prompts do not bias the Hessians (the corpus is drawn round-robin over
|
||||
source classes, so loss is class-neutral as long as failures are rare;
|
||||
a systematic 4xx is loud in the log and must be fixed, not ignored).
|
||||
|
||||
Usage: calib-push.py [--port 7770] [--parallel 64] [--container arbench]
|
||||
[--corpus /home/dank/corpora/out/calib.jsonl] [--max-rounds 4]
|
||||
"""
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
DONE_MARK = "calibration wrote"
|
||||
|
||||
|
||||
def tap_done(container):
|
||||
"""True if the engine logged the tap's completion line."""
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["podman", "logs", "--tail", "50", container],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
return DONE_MARK in (out.stdout + out.stderr)
|
||||
|
||||
|
||||
def push_one(url, prompt):
|
||||
body = json.dumps({
|
||||
"model": "frank-archon",
|
||||
"prompt": prompt,
|
||||
"max_tokens": 1,
|
||||
"temperature": 0,
|
||||
"stream": False,
|
||||
}).encode()
|
||||
req = urllib.request.Request(
|
||||
url, data=body, headers={"Content-Type": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=1800) as r:
|
||||
r.read()
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"FAIL {str(e)[:120]}", flush=True)
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--port", type=int, default=7770)
|
||||
ap.add_argument("--parallel", type=int, default=64)
|
||||
ap.add_argument("--container", default="arbench")
|
||||
ap.add_argument("--corpus", default="/home/dank/corpora/out/calib.jsonl")
|
||||
ap.add_argument("--max-rounds", type=int, default=4)
|
||||
a = ap.parse_args()
|
||||
url = f"http://127.0.0.1:{a.port}/v1/completions"
|
||||
|
||||
prompts = []
|
||||
with open(a.corpus) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
prompts.append(json.loads(line)["prompt"])
|
||||
print(f"corpus: {len(prompts)} prompts", flush=True)
|
||||
|
||||
for rnd in range(1, a.max_rounds + 1):
|
||||
if tap_done(a.container):
|
||||
break
|
||||
t0 = time.time()
|
||||
ok = 0
|
||||
with concurrent.futures.ThreadPoolExecutor(a.parallel) as ex:
|
||||
for good in ex.map(lambda p: push_one(url, p), prompts):
|
||||
ok += 1 if good else 0
|
||||
dt = time.time() - t0
|
||||
print(f"round {rnd}: {ok}/{len(prompts)} ok in {dt:.0f}s "
|
||||
f"({sum(len(p) for p in prompts)/4/dt:.0f} chars/s est)",
|
||||
flush=True)
|
||||
if tap_done(a.container):
|
||||
break
|
||||
else:
|
||||
print("max rounds exhausted, tap not done", flush=True)
|
||||
sys.exit(1)
|
||||
print("TAP_DONE: calibration wrote layer Hessians", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+73
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env bash
|
||||
# convert-swift15-w4dn8.sh — RUNS ON DANKSERVER. The final recipe convert of
|
||||
# the Swift 1.5 checkpoint: gate_up experts through GPTQ against the 10M-token
|
||||
# calibration Grams, down projections block-fp8, protected experts bf16,
|
||||
# trunk int8, PLE fixed-scale fp8 (scripts/recipes/
|
||||
# qwen4exp-swift15-w4dn8-i8-hc8m.recipe).
|
||||
#
|
||||
# The convert engine is 1.0.13 — the SAME pin that will serve the result, so
|
||||
# the .rad crosses no version boundary. Only the Gram files cross versions
|
||||
# (1.2.0 wrote them; the RADGRAM2 magic, gram.r%d.%s.bin naming and the
|
||||
# calib_gram/calib_resolve/calib_stem symbols are byte-identical between the
|
||||
# two libquant.so). rad-convert runs on the host backend (no GPU visible in
|
||||
# the container) — production can stay up through this script.
|
||||
#
|
||||
# Prereqs (campaign order): calibration complete (calib-push printed TAP_DONE),
|
||||
# the protected-expert cast lines rewritten into the recipe by
|
||||
# derive-protected-experts.py, and the 335 GiB bf16 container deleted —
|
||||
# the peak-disk arithmetic needs that space (checkpoint 335.3 + output ~132
|
||||
# GiB on the volume; --reuse is NOT used, see the recipe header).
|
||||
set -euo pipefail
|
||||
|
||||
PIN_1013=sha256:4c3f484564bb2872784e46b0a59cbd71becc9c1e746e301a3596ffae253c322e
|
||||
IMAGE="stilldeadcode/radiance@$PIN_1013"
|
||||
CKPT=/home/dank/models/ukisai/Swift1.5-Qwen3.8-Flash-Next
|
||||
RECIPE=/home/dank/serverAppConfigs-swift15/scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe
|
||||
CALIB=/home/dank/corpora/calib
|
||||
OUT=/home/dank/models/rad/qwen4exp-swift15-w4dn8-i8-hc8m.rad
|
||||
LOG=/home/dank/logs/swift15_convert.log
|
||||
# gate_up ~37.6 + down ~18.8 + PLE 47.7 + trunk/embeddings/head ~10 GiB (all-w4 expert table;
|
||||
# the plugin's one-encoding law dropped the block-fp8 down tier -- see the recipe header).
|
||||
MIN_MIB=${FRANK_SWIFT15_RAD_MIN_MIB:-108000}
|
||||
|
||||
# Gate 1: inputs. A missing Gram directory converts UNCALIBRATED with a
|
||||
# per-tensor stderr note, which is silently worse than failing here.
|
||||
[ -d "$CKPT" ] || { echo "no checkpoint at $CKPT" >&2; exit 1; }
|
||||
[ -f "$RECIPE" ] || { echo "no recipe at $RECIPE" >&2; exit 1; }
|
||||
GRAMS=$(find "$CALIB" -maxdepth 1 -name 'gram.r*.bin' 2>/dev/null | wc -l)
|
||||
[ "$GRAMS" -ge 48 ] || { echo "calib dir $CALIB holds $GRAMS gram files (<48)" >&2; exit 1; }
|
||||
if [ -e "$OUT" ]; then echo "$OUT already exists; move it to trash first" >&2; exit 1; fi
|
||||
if [ -e /home/dank/models/rad/qwen4exp-swift15-bf16.rad ]; then
|
||||
echo "the bf16 container is still on disk; delete it first (peak disk)" >&2; exit 1
|
||||
fi
|
||||
|
||||
# Gate 2: disk. Output ~132 GiB plus convert staging, 20 GiB of slack.
|
||||
FREE_MIB=$(df --output=avail -m /home/dank/models | tail -1 | tr -d ' ')
|
||||
[ "$FREE_MIB" -ge $((MIN_MIB + 20480)) ] || {
|
||||
echo "only ${FREE_MIB} MiB free on /home/dank/models" >&2; exit 1; }
|
||||
|
||||
# The recipe names $CALIB; rad-convert expands it from the environment
|
||||
# (MOE-W4.md:275). The recipe's cast section must name THIS checkpoint's
|
||||
# protected experts — derive-protected-experts.py rewrote it.
|
||||
echo "convert: $CKPT -> $OUT (gptq calib=$CALIB, $(date -Is))" | tee "$LOG"
|
||||
podman run --rm --name swift15-convert \
|
||||
-v /home/dank/models:/models \
|
||||
-v /home/dank/serverAppConfigs-swift15:/campaign:ro \
|
||||
-v "$CALIB:/data/calib:ro" \
|
||||
-e CALIB=/data/calib \
|
||||
--entrypoint /opt/radiance/bin/rad-convert \
|
||||
"$IMAGE" "/models/${CKPT#/home/dank/models/}" \
|
||||
--recipe /campaign/"${RECIPE#/home/dank/serverAppConfigs-swift15/}" \
|
||||
-o /models/rad/"$(basename "$OUT")" 2>&1 | tee -a "$LOG"
|
||||
rc=${PIPESTATUS[0]}
|
||||
[ "$rc" -eq 0 ] || { echo "rad-convert rc=$rc; log: $LOG" >&2; exit "$rc"; }
|
||||
|
||||
# Gate 3: the artifact. rad-info's recipe echo is the receipt that GPTQ ran.
|
||||
SIZE_MIB=$(( $(stat -c %s "$OUT") / 1048576 ))
|
||||
[ "$SIZE_MIB" -ge "$MIN_MIB" ] || {
|
||||
echo "$OUT is ${SIZE_MIB} MiB < floor ${MIN_MIB}" >&2; exit 1; }
|
||||
podman run --rm --entrypoint /opt/radiance/bin/rad-info \
|
||||
-v /home/dank/models:/models:ro "$IMAGE" /models/rad/"$(basename "$OUT")" \
|
||||
2>&1 | tee -a "$LOG" | grep -c 'gptq' >/dev/null \
|
||||
|| echo "WARNING: rad-info shows no gptq rule in the recorded recipe" >&2
|
||||
echo "CONVERT_DONE ${SIZE_MIB} MiB -> $OUT"
|
||||
@@ -57,9 +57,9 @@ else
|
||||
echo " none"
|
||||
fi
|
||||
|
||||
section "frank-primary unit"
|
||||
printf ' active: %s\n' "$(systemctl is-active frank-primary 2>/dev/null | head -1)"
|
||||
printf ' enabled: %s\n' "$(systemctl is-enabled frank-primary 2>/dev/null | head -1)"
|
||||
section "frank-archon unit"
|
||||
printf ' active: %s\n' "$(systemctl is-active frank-archon 2>/dev/null | head -1)"
|
||||
printf ' enabled: %s\n' "$(systemctl is-enabled frank-archon 2>/dev/null | head -1)"
|
||||
|
||||
section "glm serve tail"
|
||||
if podman ps -a --format '{{.Names}}' 2>/dev/null | grep -qx "$GLM_CTR"; then
|
||||
|
||||
Executable
+55
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python3
|
||||
"""deep-probe.py — one chat completion at ~43.6k prompt tokens, temp 0, 512
|
||||
generated, streamed; decode = completion_tokens / (total - ttft)."""
|
||||
import json, time, urllib.request
|
||||
|
||||
TARGET_BYTES = int(43600 * 3.8)
|
||||
|
||||
def build():
|
||||
parts, n = [], 0
|
||||
with open("/home/dank/corpora/out/calib.jsonl") as f:
|
||||
for line in f:
|
||||
p = json.loads(line)["prompt"]
|
||||
parts.append(p)
|
||||
n += len(p.encode())
|
||||
if n >= TARGET_BYTES:
|
||||
break
|
||||
return "".join(parts)
|
||||
|
||||
def main(salt):
|
||||
body = json.dumps({
|
||||
"model": "frank-archon",
|
||||
"messages": [{"role": "user", "content": f"[probe {salt}] Continue and summarize this material in English, 400 words.\n\n" + build()}],
|
||||
"temperature": 0.0, "max_tokens": 512,
|
||||
"stream": True, "stream_options": {"include_usage": True},
|
||||
}).encode()
|
||||
req = urllib.request.Request("http://localhost:7770/v1/chat/completions",
|
||||
data=body, headers={"Content-Type": "application/json"})
|
||||
t0 = time.time(); ttft = None; ctok = 0; usage = {}
|
||||
with urllib.request.urlopen(req, timeout=2400) as r:
|
||||
for raw in r:
|
||||
line = raw.decode().strip()
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
d = line[5:].strip()
|
||||
if d == "[DONE]":
|
||||
break
|
||||
j = json.loads(d)
|
||||
if j.get("usage"):
|
||||
usage = j["usage"]
|
||||
for ch in j.get("choices", []):
|
||||
delta = (ch.get("delta") or {})
|
||||
piece = delta.get("content") or delta.get("reasoning_content") or ""
|
||||
if piece and ttft is None:
|
||||
ttft = time.time() - t0
|
||||
ctok += 1
|
||||
total = time.time() - t0
|
||||
gen = usage.get("completion_tokens") or ctok
|
||||
dec = gen / (total - (ttft or 0)) if total > (ttft or 0) else 0
|
||||
print(json.dumps({"salt": salt, "prompt_tokens": usage.get("prompt_tokens"),
|
||||
"completion_tokens": gen, "ttft_s": round(ttft or 0, 2),
|
||||
"total_s": round(total, 1), "decode_tps": round(dec, 2)}))
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
main(sys.argv[1] if len(sys.argv) > 1 else "x")
|
||||
+23
-23
@@ -61,9 +61,9 @@ SERVICE_ORDER = (
|
||||
"sonarr",
|
||||
"lidarr",
|
||||
"readarr",
|
||||
"frank-large",
|
||||
"frank-small",
|
||||
"frank-primary",
|
||||
"frank-dragoon",
|
||||
"frank-zealot",
|
||||
"frank-archon",
|
||||
)
|
||||
|
||||
DANKCLAW_SERVICES = {
|
||||
@@ -93,8 +93,8 @@ DANKCLAW_SERVICES = {
|
||||
"authentik-outpost-ldap",
|
||||
"caddy",
|
||||
"homepage",
|
||||
"frank-large",
|
||||
"frank-small",
|
||||
"frank-dragoon",
|
||||
"frank-zealot",
|
||||
}
|
||||
|
||||
DANKSERVER_SERVICES = {
|
||||
@@ -113,7 +113,7 @@ DANKSERVER_SERVICES = {
|
||||
"sonarr",
|
||||
"lidarr",
|
||||
"readarr",
|
||||
"frank-primary",
|
||||
"frank-archon",
|
||||
"node-exporter",
|
||||
}
|
||||
|
||||
@@ -316,7 +316,7 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
if len(parts) >= 2 and parts[1] == "open-webui":
|
||||
_add(services, "open-webui")
|
||||
return services
|
||||
# frank-large left the quadlet shape on 2026-09-19 (CIRU v4.4.1 host
|
||||
# frank-dragoon left the quadlet shape on 2026-09-19 (CIRU v4.4.1 host
|
||||
# build, root unit): its files live in scripts/ and map there.
|
||||
if len(parts) >= 2 and parts[1] == "observability":
|
||||
# The five observability containers became Quadlet units on
|
||||
@@ -371,10 +371,10 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
if name == "node-exporter":
|
||||
_add(services, "node-exporter")
|
||||
return services
|
||||
# frank-primary's quadlet-era directory left the tree on 2026-09-26
|
||||
# frank-archon's quadlet-era directory left the tree on 2026-09-26
|
||||
# (the Lucebox DS4.1 unit was retired for the llama.cpp root unit —
|
||||
# the trio in scripts/ now maps it). Nothing lives under
|
||||
# dankserver/frank-primary/ except dated trash, and trash plans
|
||||
# dankserver/frank-archon/ except dated trash, and trash plans
|
||||
# nothing (the retire rule above the service blocks).
|
||||
return services
|
||||
|
||||
@@ -400,32 +400,32 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
# change to it redeploying alertmanager is the only path that
|
||||
# reaches it on the host.
|
||||
_add(services, "alertmanager")
|
||||
if name in {"frank-small.env", "frank-small.service", "serve-frank-small.sh"}:
|
||||
# llama.cpp frank-small (2026-09-16) keeps its unit, env and serve
|
||||
# files here; each one selects frank-small only. The ik_llama.cpp
|
||||
if name in {"frank-zealot.env", "frank-zealot.service", "serve-frank-zealot.sh"}:
|
||||
# llama.cpp frank-zealot (2026-09-16) keeps its unit, env and serve
|
||||
# files here; each one selects frank-zealot only. The ik_llama.cpp
|
||||
# trio that lived here for two days is in scripts/.AGENTTRASH/,
|
||||
# which is archive material and plans nothing.
|
||||
_add(services, "frank-small")
|
||||
if name in {"frank-large.env", "frank-large.service", "serve-frank-large.sh"}:
|
||||
# CIRU v4.4.1 frank-large (2026-09-19) keeps its unit, env and
|
||||
_add(services, "frank-zealot")
|
||||
if name in {"frank-dragoon.env", "frank-dragoon.service", "serve-frank-dragoon.sh"}:
|
||||
# CIRU v4.4.1 frank-dragoon (2026-09-19) keeps its unit, env and
|
||||
# serve files here the same way: a root unit on a host-built
|
||||
# runtime. The download script fetches the payload, which is
|
||||
# host-side state, so it selects nothing.
|
||||
_add(services, "frank-large")
|
||||
_add(services, "frank-dragoon")
|
||||
if name == "qwen-sharp-v22.4.0.jinja":
|
||||
# The Qwen-Sharp chat template frank-large loads via
|
||||
# The Qwen-Sharp chat template frank-dragoon loads via
|
||||
# --chat-template-file. It lived under dankserver/frank-medium/ until
|
||||
# the 2026-09-26 cutover retired that directory; editing it changes
|
||||
# what the live frank-large endpoint renders, so it plans
|
||||
# frank-large.
|
||||
_add(services, "frank-large")
|
||||
if name in {"frank-primary.env", "frank-primary.service", "serve-frank-primary.sh"}:
|
||||
# llama.cpp frank-primary (2026-09-26, replacing the Lucebox
|
||||
# what the live frank-dragoon endpoint renders, so it plans
|
||||
# frank-dragoon.
|
||||
_add(services, "frank-dragoon")
|
||||
if name in {"frank-archon.env", "frank-archon.service", "serve-frank-archon.sh"}:
|
||||
# llama.cpp frank-archon (2026-09-26, replacing the Lucebox
|
||||
# DS4.1 quadlet) keeps its unit, env and serve files here the
|
||||
# same way: a root unit on the unsloth b11030-mix prebuilt
|
||||
# serving Qwen3.8-Flash-Next Q8_0. The download script fetches
|
||||
# the payload, which is host-side state, so it selects nothing.
|
||||
_add(services, "frank-primary")
|
||||
_add(services, "frank-archon")
|
||||
# Deployment/test/benchmark scripts and other repo tooling do not
|
||||
# restart a runtime service.
|
||||
return services
|
||||
|
||||
+178
-161
@@ -90,12 +90,12 @@ service_is_allowed() {
|
||||
dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
||||
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
|
||||
dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
|
||||
dankclaw:frank-large|dankclaw:frank-small|\
|
||||
dankclaw:frank-dragoon|dankclaw:frank-zealot|\
|
||||
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
|
||||
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
|
||||
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
|
||||
dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
|
||||
dankserver:frank-primary|dankserver:node-exporter|\
|
||||
dankserver:frank-archon|dankserver:node-exporter|\
|
||||
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
|
||||
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
|
||||
frankbook-pro:readarr)
|
||||
@@ -192,7 +192,7 @@ add_pending_services() {
|
||||
continue
|
||||
fi
|
||||
[[ -n "$recorded" && "$recorded" != "$REVISION" ]] || continue
|
||||
if [[ "$TARGET" == dankserver && "$service" == frank-large ]]; then continue; fi
|
||||
if [[ "$TARGET" == dankserver && "$service" == frank-dragoon ]]; then continue; fi
|
||||
if ! changed="$(python3 scripts/deploy-plan.py --target "$TARGET" \
|
||||
--repo "$REMOTE_DIR" --base "$recorded" --head "$REVISION")"; then
|
||||
echo "WARN: unable to compare state for $service; retaining it as pending" >&2
|
||||
@@ -322,6 +322,12 @@ deploy_homepage() {
|
||||
if [[ -d shared/homepage/fonts ]]; then
|
||||
run_privileged cp -r shared/homepage/fonts /srv/homepage/
|
||||
fi
|
||||
if [[ -d shared/homepage/img ]]; then
|
||||
run_privileged cp -r shared/homepage/img /srv/homepage/
|
||||
fi
|
||||
if [[ -d shared/homepage/sfx ]]; then
|
||||
run_privileged cp -r shared/homepage/sfx /srv/homepage/
|
||||
fi
|
||||
local deployed_at
|
||||
deployed_at="$(git log -1 --format=%cI)"
|
||||
printf 'window.DEPLOYED_AT = "%s";\n' "$deployed_at" > /tmp/deployed.js
|
||||
@@ -1086,41 +1092,45 @@ deploy_anubis() {
|
||||
wait_container_healthy anubis 60 || return 1
|
||||
}
|
||||
|
||||
# frank-large is the halogen CONTAINER (ghcr.io/peonist-ai/halogen,
|
||||
# digest-pinned through scripts/frank-large.env) serving
|
||||
# peonist-ai/halogen-qwen3.8-27b — qwen3.8-27b-p1w4d-d2.hgn, 33.4 GiB,
|
||||
# DFlash2 spec decode, 1 slot x 200700, text-only — on the Strix Halo
|
||||
# iGPU. It stays a ROOT unit because the serve script drives `podman run`
|
||||
# directly (the frank-primary pattern): relink_unit points
|
||||
# /etc/systemd/system/frank-large.service at the repo file, boot
|
||||
# frank-dragoon is the halogen-flash-server CONTAINER (0.16.2, digest-pinned
|
||||
# through scripts/frank-dragoon.env) serving the Quat3rnion ABLITERATED
|
||||
# Swift1.5 finetune of peonist's Qwen3.8-Flash-Next v2 checkpoint —
|
||||
# qwen38-flash-next-v2-swift15-abliterated.hgn (66,687,678,464 B, ~62 GiB
|
||||
# resident) + the REQUIRED n-gram lookup table + the vision tower —
|
||||
# MTP spec decode (HALOGEN_SPEC_ADAPT=0), 2 slots x 200192, image input —
|
||||
# on the Strix Halo iGPU. It stays a ROOT unit because the serve script
|
||||
# drives `podman run` directly (the frank-archon pattern): relink_unit
|
||||
# points /etc/systemd/system/frank-dragoon.service at the repo file, boot
|
||||
# persistence is that unit's [Install], and LimitMEMLOCK=infinity is
|
||||
# fleet parity. 2026-10-05 cutover from the ROCmFPX llama.cpp shape —
|
||||
# rollback = revert the swap commit's trio; that payload, its draft head
|
||||
# and the fork build all stay on disk
|
||||
# (~/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP,
|
||||
# ~/GitPackages/kingjones-ROCmFPX).
|
||||
# fleet parity. 2026-10-06 cutover from the halogen 27B-dense shape —
|
||||
# rollback = revert the swap commit's trio; the 27B payload stays on
|
||||
# disk (~/models/halogen-qwen3.8-27b).
|
||||
#
|
||||
# The gates mirror serve-frank-large.sh's predicates EXACTLY (the
|
||||
# The gates mirror serve-frank-dragoon.sh's predicates EXACTLY (the
|
||||
# 2026-10-01 crash-loop lesson): image digest in dank's podman storage,
|
||||
# the .hgn checkpoint at the exact byte count, the tokenizer directory
|
||||
# holding every named file as a regular (not symlinked) file — a green
|
||||
# deploy against a missing artifact would
|
||||
# leave a restart-looping unit and a dark endpoint. The port gate is the
|
||||
# same as every cutover since 2026-09-16: nothing starts until 7770 is
|
||||
# released.
|
||||
LLM_LARGE_IMAGE="${LLM_LARGE_IMAGE:-ghcr.io/peonist-ai/halogen@sha256:dc0a39a0016d6cfc58a197978febaafdf8d28403f724ded6111d98b5fb7ac0ea}"
|
||||
LLM_LARGE_CKPT="${LLM_LARGE_CKPT:-/home/dank/models/halogen-qwen3.8-27b/qwen3.8-27b-p1w4d-d2.hgn}"
|
||||
LLM_LARGE_CKPT_BYTES="${LLM_LARGE_CKPT_BYTES:-35865565184}"
|
||||
LLM_LARGE_TOK_DIR="${LLM_LARGE_TOK_DIR:-/home/dank/models/halogen-qwen3.8-27b/tokenizer}"
|
||||
LLM_LARGE_TOK_FILES="${LLM_LARGE_TOK_FILES:-tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json}"
|
||||
# each .hgn payload (checkpoint, n-gram table, vision tower) at the exact
|
||||
# byte count, the tokenizer directory holding every named file as a
|
||||
# regular (not symlinked) file — a green deploy against a missing
|
||||
# artifact would leave a restart-looping unit and a dark endpoint. The
|
||||
# port gate is the same as every cutover since 2026-09-16: nothing
|
||||
# starts until 7770 is released.
|
||||
LLM_DRAGOON_IMAGE="${LLM_DRAGOON_IMAGE:-ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84ac22308a53f5d1ca6b86806702d7039e5ebc51cae4c66621b92fe04a}"
|
||||
LLM_DRAGOON_CKPT="${LLM_DRAGOON_CKPT:-/home/dank/models/halogen-flashnext-v2/qwen38-flash-next-v2-swift15-abliterated.hgn}"
|
||||
LLM_DRAGOON_CKPT_BYTES="${LLM_DRAGOON_CKPT_BYTES:-66687678464}"
|
||||
LLM_DRAGOON_NGRAM="${LLM_DRAGOON_NGRAM:-/home/dank/models/halogen-flashnext-v2/qwen38-flash-next-ngram.hgn}"
|
||||
LLM_DRAGOON_NGRAM_BYTES="${LLM_DRAGOON_NGRAM_BYTES:-51200246144}"
|
||||
LLM_DRAGOON_VISION="${LLM_DRAGOON_VISION:-/home/dank/models/halogen-flashnext-v2/qwen38-flash-next-vision.hgn}"
|
||||
LLM_DRAGOON_VISION_BYTES="${LLM_DRAGOON_VISION_BYTES:-897916416}"
|
||||
LLM_DRAGOON_TOK_DIR="${LLM_DRAGOON_TOK_DIR:-/home/dank/models/halogen-flashnext-v2/tokenizer}"
|
||||
LLM_DRAGOON_TOK_FILES="${LLM_DRAGOON_TOK_FILES:-tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json}"
|
||||
|
||||
retire_halogen_large_quadlet() {
|
||||
retire_halogen_dragoon_quadlet() {
|
||||
local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
|
||||
systemctl --user stop frank-large.service 2>/dev/null || true
|
||||
if [[ -e "$quadlet_unit_dir/frank-large.container" ]]; then
|
||||
systemctl --user stop frank-dragoon.service 2>/dev/null || true
|
||||
if [[ -e "$quadlet_unit_dir/frank-dragoon.container" ]]; then
|
||||
mkdir -p "$trash"
|
||||
mv -- "$quadlet_unit_dir/frank-large.container" "$trash/frank-large.container"
|
||||
echo "retired quadlet frank-large.container -> $trash"
|
||||
mv -- "$quadlet_unit_dir/frank-dragoon.container" "$trash/frank-dragoon.container"
|
||||
echo "retired quadlet frank-dragoon.container -> $trash"
|
||||
fi
|
||||
systemctl --user daemon-reload
|
||||
}
|
||||
@@ -1144,175 +1154,184 @@ wait_port_free() {
|
||||
return 1
|
||||
}
|
||||
|
||||
deploy_llm_large() {
|
||||
deploy_llm_dragoon() {
|
||||
# Refuse rather than restart-loop; every gate below has a twin in
|
||||
# serve-frank-large.sh and they must agree (2026-10-01 lesson).
|
||||
# serve-frank-dragoon.sh and they must agree (2026-10-01 lesson).
|
||||
# Capture podman output before grepping: `podman images | grep -q`
|
||||
# races under pipefail (SIGPIPE 141 becomes "no match", 2026-09-30).
|
||||
if ! grep -qF "${LLM_LARGE_IMAGE#*@}" <<<"$(podman images --format '{{.RepoDigests}}' 2>/dev/null)"; then
|
||||
echo "ERROR: halogen image absent: $LLM_LARGE_IMAGE" >&2
|
||||
echo "frank-large: pull it: podman pull ${LLM_LARGE_IMAGE%@*}" >&2
|
||||
if ! grep -qF "${LLM_DRAGOON_IMAGE#*@}" <<<"$(podman images --format '{{.RepoDigests}}' 2>/dev/null)"; then
|
||||
echo "ERROR: halogen image absent: $LLM_DRAGOON_IMAGE" >&2
|
||||
echo "frank-dragoon: pull it: podman pull ${LLM_DRAGOON_IMAGE%@*}" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$LLM_LARGE_CKPT" ]]; then
|
||||
echo "ERROR: halogen checkpoint absent at $LLM_LARGE_CKPT" >&2
|
||||
echo "frank-large: fetch it: bash scripts/download-frank-large-halogen.sh" >&2
|
||||
if [[ ! -f "$LLM_DRAGOON_CKPT" ]]; then
|
||||
echo "ERROR: halogen checkpoint absent at $LLM_DRAGOON_CKPT" >&2
|
||||
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "$(stat -c %s "$LLM_LARGE_CKPT")" != "$LLM_LARGE_CKPT_BYTES" ]]; then
|
||||
echo "ERROR: halogen checkpoint is not $LLM_LARGE_CKPT_BYTES bytes (partial download?)" >&2
|
||||
echo "frank-large: re-check: bash scripts/download-frank-large-halogen.sh --verify-only" >&2
|
||||
if [[ "$(stat -c %s "$LLM_DRAGOON_CKPT")" != "$LLM_DRAGOON_CKPT_BYTES" ]]; then
|
||||
echo "ERROR: halogen checkpoint is not $LLM_DRAGOON_CKPT_BYTES bytes (partial download?)" >&2
|
||||
echo "frank-dragoon: re-check: bash scripts/download-frank-dragoon-flashnext.sh --verify-only" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ -L "$LLM_LARGE_TOK_DIR" || ! -d "$LLM_LARGE_TOK_DIR" ]]; then
|
||||
echo "ERROR: tokenizer dir missing or a symlink at $LLM_LARGE_TOK_DIR" >&2
|
||||
echo "frank-large: fetch it: bash scripts/download-frank-large-halogen.sh" >&2
|
||||
for gate in "$LLM_DRAGOON_NGRAM:$LLM_DRAGOON_NGRAM_BYTES:n-gram table" \
|
||||
"$LLM_DRAGOON_VISION:$LLM_DRAGOON_VISION_BYTES:vision tower"; do
|
||||
IFS=: read -r gpath gbytes gname <<<"$gate"
|
||||
if [[ ! -f "$gpath" ]]; then
|
||||
echo "ERROR: halogen $gname absent at $gpath" >&2
|
||||
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "$(stat -c %s "$gpath")" != "$gbytes" ]]; then
|
||||
echo "ERROR: halogen $gname is not $gbytes bytes (partial download?)" >&2
|
||||
echo "frank-dragoon: re-check: bash scripts/download-frank-dragoon-flashnext.sh --verify-only" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
if [[ -L "$LLM_DRAGOON_TOK_DIR" || ! -d "$LLM_DRAGOON_TOK_DIR" ]]; then
|
||||
echo "ERROR: tokenizer dir missing or a symlink at $LLM_DRAGOON_TOK_DIR" >&2
|
||||
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
for tf in $LLM_LARGE_TOK_FILES; do
|
||||
if [[ ! -f "$LLM_LARGE_TOK_DIR/$tf" || -L "$LLM_LARGE_TOK_DIR/$tf" ]]; then
|
||||
echo "ERROR: tokenizer file missing or a symlink: $LLM_LARGE_TOK_DIR/$tf" >&2
|
||||
echo "frank-large: fetch it: bash scripts/download-frank-large-halogen.sh" >&2
|
||||
for tf in $LLM_DRAGOON_TOK_FILES; do
|
||||
if [[ ! -f "$LLM_DRAGOON_TOK_DIR/$tf" || -L "$LLM_DRAGOON_TOK_DIR/$tf" ]]; then
|
||||
echo "ERROR: tokenizer file missing or a symlink: $LLM_DRAGOON_TOK_DIR/$tf" >&2
|
||||
echo "frank-dragoon: fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
retire_halogen_large_quadlet
|
||||
retire_halogen_dragoon_quadlet
|
||||
# The predecessor on a re-deploy is THIS unit: stop it (ExecStop stops
|
||||
# the container) before waiting on the port. The stray sweep is keyed
|
||||
# on the FULL container name, which appears in the argv of a manual
|
||||
# `podman run --name frank-large-halogen` / `podman wait` but NOT in
|
||||
# `podman run --name frank-dragoon-halogen` / `podman wait` but NOT in
|
||||
# this deployer's own command line (`bash deploy-remote.sh
|
||||
# frank-large`) — so it cannot shoot itself in the foot (the fleet
|
||||
# pkill law), and it can never match frank-small's llama-server.
|
||||
run_privileged systemctl stop frank-large.service 2>/dev/null || true
|
||||
pkill -f 'frank-large-halogen' 2>/dev/null || true
|
||||
podman rm -f frank-large frank-large-halogen >/dev/null 2>&1 || true
|
||||
# frank-dragoon`) — so it cannot shoot itself in the foot (the fleet
|
||||
# pkill law), and it can never match frank-zealot's llama-server.
|
||||
run_privileged systemctl stop frank-dragoon.service 2>/dev/null || true
|
||||
pkill -f 'frank-dragoon-halogen' 2>/dev/null || true
|
||||
podman rm -f frank-dragoon frank-dragoon-halogen >/dev/null 2>&1 || true
|
||||
# Nothing may start until the predecessor has released the port
|
||||
# (2026-09-16: "Deactivated successfully" was not "port free").
|
||||
wait_port_free 7770 || return 1
|
||||
relink_unit frank-large
|
||||
relink_unit frank-dragoon
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-large.service
|
||||
run_privileged systemctl restart frank-large.service
|
||||
wait_for_health frank-large http://127.0.0.1:7770/health 1200
|
||||
run_privileged systemctl enable frank-dragoon.service
|
||||
run_privileged systemctl restart frank-dragoon.service
|
||||
wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200
|
||||
}
|
||||
|
||||
# deploy_llm_small — frank-small is llama.cpp (master build, CUDA sm_86) on
|
||||
# deploy_llm_zealot — frank-zealot is llama.cpp (master build, CUDA sm_86) on
|
||||
# the RTX 3090 as a ROOT unit: the runtime is a native binary tree under
|
||||
# ~/GitPackages/llama-master/build-cuda, so there is no container image and
|
||||
# no generated unit to avoid. relink_unit points
|
||||
# /etc/systemd/system/frank-small.service at the repo file and boot
|
||||
# /etc/systemd/system/frank-zealot.service at the repo file and boot
|
||||
# persistence is that unit's [Install] section.
|
||||
#
|
||||
# It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
|
||||
# not shared: frank-large's engine runs on the gfx1151 iGPU.
|
||||
LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/Ornith-Q8/Ornith-1.5-9B-Q8_0.gguf}"
|
||||
LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
||||
# The projector IS the vision capability here: the Ornith GGUF ships no
|
||||
# `v.*` tensors, so a deploy that starts without this file silently ships a
|
||||
# text-only endpoint. Checked unconditionally, like frank-large's.
|
||||
LLM_SMALL_MMPROJ="${LLM_SMALL_MMPROJ:-/home/dank/models/Ornith-Q8/mmproj-Ornith-1.5-9B-BF16.gguf}"
|
||||
# It is a SEPARATE endpoint on port 7771; frank-dragoon keeps 7770. The 3090 is
|
||||
# not shared: frank-dragoon's engine runs on the gfx1151 iGPU.
|
||||
LLM_ZEALOT_MODEL_FILE="${LLM_ZEALOT_MODEL_FILE:-/home/dank/models/Qwen35-9B-HauhauCS-Q6K/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K.gguf}"
|
||||
LLM_ZEALOT_RUNTIME="${LLM_ZEALOT_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
||||
# The projector IS the vision capability here: the model is natively
|
||||
# multimodal and the Q6_K file carries no vision tensors — the publisher ships
|
||||
# its own BF16 projector in the same repo, so a deploy that starts without
|
||||
# this file silently ships a text-only endpoint. Checked unconditionally, like
|
||||
# frank-dragoon's.
|
||||
LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Qwen35-9B-HauhauCS-Q6K/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf}"
|
||||
|
||||
deploy_llm_small() {
|
||||
deploy_llm_zealot() {
|
||||
# Refuse rather than restart-loop: a unit whose runtime or payload is
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the log to say why.
|
||||
if [[ ! -x "$LLM_SMALL_RUNTIME" ]]; then
|
||||
echo "ERROR: llama.cpp runtime absent at $LLM_SMALL_RUNTIME" >&2
|
||||
echo "frank-small: build it (cmake -B build-cuda -DGGML_CUDA=ON" >&2
|
||||
echo "frank-small: -DCMAKE_CUDA_ARCHITECTURES=86 -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/gcc-13" >&2
|
||||
echo "frank-small: && cmake --build build-cuda --target llama-server) first." >&2
|
||||
if [[ ! -x "$LLM_ZEALOT_RUNTIME" ]]; then
|
||||
echo "ERROR: llama.cpp runtime absent at $LLM_ZEALOT_RUNTIME" >&2
|
||||
echo "frank-zealot: build it (cmake -B build-cuda -DGGML_CUDA=ON" >&2
|
||||
echo "frank-zealot: -DCMAKE_CUDA_ARCHITECTURES=86 -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/gcc-13" >&2
|
||||
echo "frank-zealot: && cmake --build build-cuda --target llama-server) first." >&2
|
||||
return 1
|
||||
fi
|
||||
for f in "$LLM_SMALL_MODEL_FILE" "$LLM_SMALL_MMPROJ"; do
|
||||
for f in "$LLM_ZEALOT_MODEL_FILE" "$LLM_ZEALOT_MMPROJ"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "ERROR: frank-small payload file absent at $f" >&2
|
||||
echo "frank-small: run scripts/download-frank-small.sh first;" >&2
|
||||
echo "frank-small: refusing to start a unit that cannot load." >&2
|
||||
echo "ERROR: frank-zealot payload file absent at $f" >&2
|
||||
echo "frank-zealot: run scripts/download-frank-zealot.sh first;" >&2
|
||||
echo "frank-zealot: refusing to start a unit that cannot load." >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
relink_unit frank-small
|
||||
relink_unit frank-zealot
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-small.service
|
||||
run_privileged systemctl restart frank-small.service
|
||||
wait_for_health frank-small http://127.0.0.1:7771/health 900 || return 1
|
||||
# False-green hardening (frank-large's lesson): the port answering
|
||||
run_privileged systemctl enable frank-zealot.service
|
||||
run_privileged systemctl restart frank-zealot.service
|
||||
wait_for_health frank-zealot http://127.0.0.1:7771/health 900 || return 1
|
||||
# False-green hardening (frank-dragoon's lesson): the port answering
|
||||
# proves nothing about WHO answers — the unit must be active with a
|
||||
# live MainPID, or a stray SGLang holdover is masquerading as deployed.
|
||||
local main_pid
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-small.service)"
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-zealot.service)"
|
||||
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
|
||||
echo "FAIL: :7771 answered but frank-small.service has no MainPID — a stray process holds the port" >&2
|
||||
echo "FAIL: :7771 answered but frank-zealot.service has no MainPID — a stray process holds the port" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "frank-small: health answered by unit MainPID $main_pid"
|
||||
echo "frank-zealot: health answered by unit MainPID $main_pid"
|
||||
}
|
||||
|
||||
# deploy_llm_primary — frank-primary is the FreeToken engine (PR #587 ROCm
|
||||
# build, gfx1201) serving the orcarouter Qwen3.8-Flash-Next-Uncensored-FP8
|
||||
# payload (131 safetensors shards) on ONE R9700 as a ROOT unit whose serve
|
||||
# script drives `podman run` directly (the image carries `ft serve`).
|
||||
# 2026-10-05 cutover from the tcclaviger vLLM fork (pre-cutover tag
|
||||
# frank-primary-vllm-shapeG-20261003); before that, 2026-09-29 cutover from
|
||||
# the R9V radiance fork, whose launcher/shim/setup-state machinery is
|
||||
# retired. Same port and alias as the endpoints it replaced; the harness
|
||||
# models.yml id is unchanged.
|
||||
# deploy_llm_archon — frank-archon is the radiance engine (StillDeadcode,
|
||||
# digest-pinned stilldeadcode/radiance image) serving the OFFICIAL
|
||||
# Qwen/Qwen3.8-Flash-Next-FP8 at native block-FP8 precision from the
|
||||
# rad-convert container qwen4exp-fp8-native.rad, on BOTH R9700s (TP=2) as a
|
||||
# ROOT unit whose serve script drives `podman run` directly (the engine is
|
||||
# the image entrypoint; the image lives in dank's rootless podman storage).
|
||||
# 2026-10-05 cutover from FreeToken (pre-cutover tag
|
||||
# frank-archon-freetoken-pr587-20261005); before that, the tcclaviger vLLM
|
||||
# fork (tag frank-archon-vllm-shapeG-20261003). Same port and alias; the
|
||||
# harness models.yml id is unchanged.
|
||||
#
|
||||
# Gates: the runtime image must exist in dank's rootless storage
|
||||
# (FRANK_PRIMARY_IMAGE — a LOCAL tag: the image is built on the host from
|
||||
# scripts/docker/frank-primary-freetoken.Dockerfile, which pins the
|
||||
# FreeToken PR-head SHA and the base image), the payload dir must carry
|
||||
# config.json and the expected shard count (FRANK_PRIMARY_MODEL_SHARDS),
|
||||
# and the unit env must be complete. Per-file sha256 verification was the
|
||||
# fetch script's job at download time, never re-hashed here.
|
||||
LLM_PRIMARY_CONTAINER=frank-primary
|
||||
LLM_PRIMARY_LEGACY_CONTAINERS="frank-primary-qwen frank-primary-r9v qwen-fp8 glm-probe"
|
||||
# Gates: the image must exist under its digest ref in dank's rootless
|
||||
# storage (FRANK_ARCHON_IMAGE), the .rad container file must exist at the
|
||||
# env's size floor (FRANK_ARCHON_MODEL_FILE >= FRANK_ARCHON_RAD_MIN_MIB —
|
||||
# a truncated convert is a guaranteed long-timeout boot), and the unit env
|
||||
# must be complete. Per-file sha256 verification is the fetch script's job
|
||||
# at download time and rad-convert/rad-info's job at build time, never
|
||||
# re-hashed here.
|
||||
LLM_ARCHON_CONTAINER=frank-archon
|
||||
LLM_ARCHON_LEGACY_CONTAINERS="frank-archon-qwen frank-archon-r9v qwen-fp8 glm-probe"
|
||||
|
||||
deploy_llm_primary() {
|
||||
deploy_llm_archon() {
|
||||
# Refuse rather than restart-loop: a unit whose image or payload is
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the log to say why.
|
||||
local env_file="/home/dank/serverAppConfigs/scripts/frank-primary.env"
|
||||
local image model_dir shards count
|
||||
local env_file="/home/dank/serverAppConfigs/scripts/frank-archon.env"
|
||||
local image model_dir model_file rad_min mib
|
||||
if [[ ! -f "$env_file" ]]; then
|
||||
echo "ERROR: frank-primary env file absent: $env_file" >&2
|
||||
echo "ERROR: frank-archon env file absent: $env_file" >&2
|
||||
return 1
|
||||
fi
|
||||
image="$(sed -n 's/^FRANK_PRIMARY_IMAGE=//p' "$env_file" | tail -1)"
|
||||
model_dir="$(sed -n 's/^FRANK_PRIMARY_MODEL_DIR=//p' "$env_file" | tail -1)"
|
||||
shards="$(sed -n 's/^FRANK_PRIMARY_MODEL_SHARDS=//p' "$env_file" | tail -1)"
|
||||
if [[ -z "$image" || -z "$model_dir" || -z "$shards" ]]; then
|
||||
echo "ERROR: frank-primary.env missing FRANK_PRIMARY_IMAGE/MODEL_DIR/MODEL_SHARDS" >&2
|
||||
image="$(sed -n 's/^FRANK_ARCHON_IMAGE=//p' "$env_file" | tail -1)"
|
||||
model_dir="$(sed -n 's/^FRANK_ARCHON_MODEL_DIR=//p' "$env_file" | tail -1)"
|
||||
model_file="$(sed -n 's/^FRANK_ARCHON_MODEL_FILE=//p' "$env_file" | tail -1)"
|
||||
rad_min="$(sed -n 's/^FRANK_ARCHON_RAD_MIN_MIB=//p' "$env_file" | tail -1)"
|
||||
if [[ -z "$image" || -z "$model_dir" || -z "$model_file" || -z "$rad_min" ]]; then
|
||||
echo "ERROR: frank-archon.env missing FRANK_ARCHON_IMAGE/MODEL_DIR/MODEL_FILE/RAD_MIN_MIB" >&2
|
||||
return 1
|
||||
fi
|
||||
# The image gate is EXISTENCE IN STORAGE, not a registry digest: the
|
||||
# FreeToken image is built ON THE HOST (a local tag carries NO
|
||||
# RepoDigests entry — the old grep refused exactly this shape), and
|
||||
# pinning lives in the Dockerfile (pinned PR-head SHA + digest-pinned
|
||||
# base image). `podman image exists` accepts a local tag and a
|
||||
# name@digest ref alike, in one command — no pipefail/grep race (the
|
||||
# 2026-09-30 lesson) for either shape.
|
||||
# The image gate is the digest ref itself: a name@sha256 ref exists in
|
||||
# storage only if THAT manifest was pulled. One command — no
|
||||
# pipefail/grep race (the 2026-09-30 lesson).
|
||||
if ! podman image exists "$image"; then
|
||||
echo "ERROR: runtime image absent from rootless storage: $image" >&2
|
||||
echo "frank-primary: build it (scripts/docker/frank-primary-freetoken.Dockerfile) or pull it" >&2
|
||||
echo "ERROR: runtime image absent from storage: $image" >&2
|
||||
echo "frank-archon: podman pull docker://stilldeadcode/radiance:latest" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$model_dir/config.json" ]]; then
|
||||
echo "ERROR: payload config missing: $model_dir/config.json" >&2
|
||||
echo "frank-primary: re-run scripts/download-frank-primary.sh" >&2
|
||||
if [[ ! -f "$model_dir/$model_file" ]]; then
|
||||
echo "ERROR: .rad container missing: $model_dir/$model_file" >&2
|
||||
echo "frank-archon: run scripts/build-frank-archon-rad.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$model_dir/model.safetensors.index.json" ]]; then
|
||||
echo "ERROR: weight-map index missing: $model_dir/model.safetensors.index.json" >&2
|
||||
echo "frank-primary: re-run scripts/download-frank-primary.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
count="$(find "$model_dir" -maxdepth 1 -name 'model-*-of-*.safetensors' 2>/dev/null | wc -l)"
|
||||
if [[ "$count" != "$shards" ]]; then
|
||||
echo "ERROR: payload shard count wrong: $model_dir (found $count, want $shards)" >&2
|
||||
echo "frank-primary: re-run scripts/download-frank-primary.sh" >&2
|
||||
mib=$(( $(stat -c %s "$model_dir/$model_file") / 1048576 ))
|
||||
if [[ "$mib" -lt "$rad_min" ]]; then
|
||||
echo "ERROR: .rad container too small: $model_dir/$model_file is ${mib} MiB, want >= $rad_min" >&2
|
||||
echo "frank-archon: rebuild it (scripts/build-frank-archon-rad.sh --force)" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
@@ -1324,39 +1343,37 @@ deploy_llm_primary() {
|
||||
# The legacy identities (the retired R9V container, the ad hoc
|
||||
# experiment container qwen-fp8, the retired glm-probe) hold the same
|
||||
# GPUs and die here too.
|
||||
run_privileged systemctl stop frank-primary.service 2>/dev/null || true
|
||||
run_privileged systemctl stop frank-archon.service 2>/dev/null || true
|
||||
local ctr
|
||||
for ctr in $LLM_PRIMARY_LEGACY_CONTAINERS "$LLM_PRIMARY_CONTAINER"; do
|
||||
for ctr in $LLM_ARCHON_LEGACY_CONTAINERS "$LLM_ARCHON_CONTAINER"; do
|
||||
podman rm -f "$ctr" >/dev/null 2>&1 || true
|
||||
done
|
||||
# Nothing starts until the port is released: the lucebox cutover
|
||||
# (2026-09-26) and the frank-large cutover (2026-09-16) both failed
|
||||
# (2026-09-26) and the frank-dragoon cutover (2026-09-16) both failed
|
||||
# exactly here when the next engine bound 7770 a second too early.
|
||||
wait_port_free 7770 || return 1
|
||||
relink_unit frank-primary
|
||||
relink_unit frank-archon
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-primary.service
|
||||
run_privileged systemctl restart frank-primary.service
|
||||
# Cold start: FP8 page-in plus torch compile/graph capture (~15 min cold,
|
||||
# measured 2026-09-29). Parity with every predecessor on this port.
|
||||
# FreeToken's /health returns 200 with {"status":"loading"} while the
|
||||
# engine loads (the FastAPI front end binds before the scheduler is
|
||||
# up), so a bare 200 false-greens — measured 2026-10-06: "healthy after
|
||||
# 6 s" while 131 shards were still paging in. vLLM 503'd during load,
|
||||
# which made the bare curl honest there; FreeToken needs the BODY read.
|
||||
# Only status:"ok" counts. (The MainPID check below still guards the
|
||||
# stray-port case.)
|
||||
run_privileged systemctl enable frank-archon.service
|
||||
run_privileged systemctl restart frank-archon.service
|
||||
# Cold start: .rad load + PLE pin + expert host-pool upload (the
|
||||
# publisher's compose healthcheck allows a 15-minute start_period).
|
||||
# Parity with every predecessor on this port. The HTTP listener binds
|
||||
# AFTER the full load (main.cpp: engine.start() then engine.serve()),
|
||||
# so during load the port REFUSES connections — a bare curl is honest
|
||||
# here, unlike FreeToken's 200-while-loading body. (The MainPID check
|
||||
# below still guards the stray-port case.)
|
||||
local i ok=0
|
||||
for ((i=1; i<=900; i++)); do
|
||||
if curl -fsS http://127.0.0.1:7770/health 2>/dev/null | grep -q '"status":"ok"'; then
|
||||
echo "frank-primary engine healthy after approximately $((i * 2))s"
|
||||
if curl -fsS http://127.0.0.1:7770/health 2>/dev/null >/dev/null; then
|
||||
echo "frank-archon engine healthy after approximately $((i * 2))s"
|
||||
ok=1
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
if [[ "$ok" != 1 ]]; then
|
||||
echo "FAIL: frank-primary never reached /health status ok in 1800s" >&2
|
||||
echo "FAIL: frank-archon /health never answered 200 in 1800s" >&2
|
||||
return 1
|
||||
fi
|
||||
# False-green hardening (the 2026-09-26 lesson): a bare curl against the
|
||||
@@ -1364,12 +1381,12 @@ deploy_llm_primary() {
|
||||
# it while the unit is stopped. The deploy is only green if the unit
|
||||
# itself is active with a live MainPID behind the health answer.
|
||||
local main_pid
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-primary.service)"
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-archon.service)"
|
||||
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
|
||||
echo "FAIL: :7770 answered but frank-primary.service has no MainPID — a stray process holds the port" >&2
|
||||
echo "FAIL: :7770 answered but frank-archon.service has no MainPID — a stray process holds the port" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "frank-primary: health answered by unit MainPID $main_pid"
|
||||
echo "frank-archon: health answered by unit MainPID $main_pid"
|
||||
}
|
||||
|
||||
wait_for_health() {
|
||||
@@ -1556,9 +1573,9 @@ deploy_service() {
|
||||
anubis) deploy_anubis ;;
|
||||
seerr) deploy_seerr ;;
|
||||
marreta) deploy_marreta ;;
|
||||
frank-large) deploy_llm_large ;;
|
||||
frank-small) deploy_llm_small ;;
|
||||
frank-primary) deploy_llm_primary ;;
|
||||
frank-dragoon) deploy_llm_dragoon ;;
|
||||
frank-zealot) deploy_llm_zealot ;;
|
||||
frank-archon) deploy_llm_archon ;;
|
||||
hermes|hermes-dashboard|hermes-webui) deploy_hermes ;;
|
||||
romm|romm-mariadb|romm-redis|mariadb|redis) deploy_romm_stack "$service" ;;
|
||||
jellyfin) deploy_jellyfin ;;
|
||||
|
||||
+3
-3
@@ -35,7 +35,7 @@ Options:
|
||||
Examples:
|
||||
./scripts/deploy.sh dankclaw
|
||||
./scripts/deploy.sh dankclaw --service searxng
|
||||
./scripts/deploy.sh dankserver --service frank-primary
|
||||
./scripts/deploy.sh dankserver --service frank-archon
|
||||
./scripts/deploy.sh dankclaw --dry-run
|
||||
EOF
|
||||
}
|
||||
@@ -141,12 +141,12 @@ valid_service() {
|
||||
dankclaw:alertmanager|\
|
||||
dankclaw:node-exporter|dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
||||
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|dankclaw:seerr|\
|
||||
dankclaw:frank-large|dankclaw:frank-small|\
|
||||
dankclaw:frank-dragoon|dankclaw:frank-zealot|\
|
||||
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
|
||||
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
|
||||
dankserver:qbittorrent|dankserver:radarr|dankserver:sonarr|\
|
||||
dankserver:lidarr|dankserver:readarr|dankserver:node-exporter|\
|
||||
dankserver:frank-primary|\
|
||||
dankserver:frank-archon|\
|
||||
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
|
||||
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
|
||||
frankbook-pro:readarr)
|
||||
|
||||
Executable
+117
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""derive-protected-experts.py — rewrite the recipe's protected-expert section
|
||||
from a calibration recording. RUNS ON DANKSERVER (venv: numpy).
|
||||
|
||||
The law (docs/MOE-W4.md "Protected experts", restated in the recipe):
|
||||
an expert whose share of its layer's down-input energy is at least 3% AND
|
||||
at least three times its share of the routing is kept bf16; the count must
|
||||
be EVEN on each rank's half of the layer (the four-bit GEMM takes the rest
|
||||
as two equal tables by parity) — an odd half gains its next most energetic
|
||||
non-qualifying expert.
|
||||
|
||||
Data: the calibration dir's per-expert down Grams,
|
||||
gram.r<rank>.blk.L.ffn_down_exps.<e>.bin — 32-byte RADGRAM2 header
|
||||
(magic, k, rank, rows, domain) then k*k f32, named with the checkpoint's
|
||||
GLOBAL expert numbering (arch/common/rad_block_moe_fp8.h: ge =
|
||||
expert_base + e). rows (header) is the expert's routing count; the trace
|
||||
of the Gram is its down-input energy. Rank halves: 0-255, 256-511
|
||||
(contiguous expert_base split at TP=2).
|
||||
|
||||
Output: the cast-rule lines (recipe format) on stdout, a report on stderr.
|
||||
|
||||
Usage: derive-protected-experts.py <calib dir> [--min-share 0.03]
|
||||
[--min-route-mult 3.0]
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
NAME_RE = re.compile(r"^gram\.r(\d)\.blk\.(\d+)\.ffn_down_exps\.(\d+)\.bin$")
|
||||
|
||||
|
||||
def read_gram(path):
|
||||
"""(k, rank, rows, trace) from a RADGRAM2 file (full matrix on disk)."""
|
||||
with open(path, "rb") as f:
|
||||
hdr = f.read(32)
|
||||
magic, k, rank, rows, domain = struct.unpack("<8sIIQI", hdr[:28])
|
||||
if magic != b"RADGRAM2":
|
||||
raise SystemExit(f"not a RADGRAM2 file: {path}")
|
||||
n = k * k
|
||||
a = np.frombuffer(f.read(n * 4), dtype=np.float32)
|
||||
if a.size != n:
|
||||
raise SystemExit(f"short gram file: {path}")
|
||||
tr = float(a.reshape(k, k).diagonal().sum())
|
||||
return k, rank, int(rows), tr
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("calib_dir")
|
||||
ap.add_argument("--min-share", type=float, default=0.03)
|
||||
ap.add_argument("--min-route-mult", type=float, default=3.0)
|
||||
a = ap.parse_args()
|
||||
|
||||
# layer -> global expert -> (rank, rows, energy)
|
||||
data = {}
|
||||
for path in sorted(glob.glob(os.path.join(
|
||||
a.calib_dir, "gram.r*.ffn_down_exps.*.bin"))):
|
||||
m = NAME_RE.match(os.path.basename(path))
|
||||
if not m:
|
||||
continue
|
||||
rank, layer, expert = int(m.group(1)), int(m.group(2)), int(m.group(3))
|
||||
k, _, rows, tr = read_gram(path)
|
||||
data.setdefault(layer, {})[expert] = (rank, rows, tr)
|
||||
|
||||
if not data:
|
||||
raise SystemExit("no per-expert down grams found")
|
||||
|
||||
lines, report = [], []
|
||||
total = 0
|
||||
for layer in sorted(data):
|
||||
experts = data[layer]
|
||||
tot_e = sum(v[2] for v in experts.values())
|
||||
tot_r = sum(v[1] for v in experts.values())
|
||||
if tot_e <= 0 or tot_r <= 0:
|
||||
report.append(f"blk.{layer}: empty recording, skipped")
|
||||
continue
|
||||
qual, pool = [], []
|
||||
for e, (rank, rows, tr) in experts.items():
|
||||
es, rs = tr / tot_e, rows / max(tot_r, 1)
|
||||
rec = (e, rank, es, rs)
|
||||
if es >= a.min_share and es >= a.min_route_mult * rs:
|
||||
qual.append(rec)
|
||||
elif es > 0:
|
||||
pool.append(rec)
|
||||
chosen = {r[0]: (r[2], r[3]) for r in qual}
|
||||
for half in (0, 1): # rank halves by the contiguous expert_base split
|
||||
n_half = sum(1 for e in chosen if e // 256 == half)
|
||||
if n_half % 2:
|
||||
pool.sort(key=lambda r: -r[2])
|
||||
for cand in pool:
|
||||
if cand[1] == half and cand[0] not in chosen:
|
||||
chosen[cand[0]] = (cand[2], cand[3])
|
||||
report.append(
|
||||
f"blk.{layer}: half {half} odd -> added e{cand[0]} "
|
||||
f"(energy {cand[2]*100:.2f}%)")
|
||||
break
|
||||
for e in sorted(chosen):
|
||||
es, rs = chosen[e]
|
||||
lines.append(f"blk.{layer}.ffn_*_exps.{e}.weight"
|
||||
f" cast dtype=bf16")
|
||||
report.append(f"blk.{layer} e{e:3d} (rank {e // 256}): "
|
||||
f"energy {es*100:5.2f}%, routing {rs*100:5.3f}%, "
|
||||
f"ratio {es/max(rs,1e-12):6.1f}")
|
||||
total += len(chosen)
|
||||
|
||||
print("\n".join(lines))
|
||||
print("\n".join(report), file=sys.stderr)
|
||||
print(f"total protected: {total}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,63 +0,0 @@
|
||||
# frank-primary-freetoken.Dockerfile — FreeToken (PR #587 ROCm) serving image
|
||||
# for the frank-primary endpoint (dankserver :7770, 1x R9700 gfx1201).
|
||||
#
|
||||
# Built on dankserver with rootless podman (dank user), same storage the
|
||||
# service's `podman run` reads:
|
||||
# podman build -t frank-primary-freetoken:pr587-77a7201 \
|
||||
# -f scripts/docker/frank-primary-freetoken.Dockerfile scripts/docker/
|
||||
#
|
||||
# Base: the PR's own docs/install_amd.md names this exact image as the
|
||||
# known-good ROCm 7.14 stack for gfx1201 (torch 2.11.0+rocm7.14, triton-rocm
|
||||
# 3.7 — the original pin; FreeToken main+PR587 accepts torch>=2.11,<2.16 and
|
||||
# triton>=3.6,<3.9). There is no rocm7.14+torch2.14 image on Docker Hub (the
|
||||
# 2.14 images are rocm10.1.0 — an untested pair for this PR), so the PR doc's
|
||||
# verbatim recommendation is the deployed stack.
|
||||
#
|
||||
# FreeToken source: FlashML-org/FreeToken PR #587 at pinned head
|
||||
# 77a7201f98f00d93f618c6cf8b81ffe8360e0899 (branch rocm714_torch214, base
|
||||
# main 581fdca3). The PR is ROCm-gated end to end; the NVIDIA/CUDA path is
|
||||
# byte-identical to main.
|
||||
#
|
||||
# Build law from docs/install_amd.md:
|
||||
# - `pip install --no-build-isolation --no-deps -e .` — the image's ROCm
|
||||
# torch is preserved and used to compile the C++/HIP extensions.
|
||||
# - NEVER a plain `pip install -e .` on ROCm: the dependency set is
|
||||
# authored for NVIDIA (flashlib pulls numba + nvidia-cutlass-dsl; torch
|
||||
# metadata pulls cuda-bindings) and the resolve can REPLACE the ROCm
|
||||
# torch with a CUDA build.
|
||||
# - Runtime deps installed explicitly, curated by the same doc. flashinfer
|
||||
# / sgl_kernel / nvidia-cutlass-dsl have no ROCm builds and are
|
||||
# intentionally absent — FreeToken gates them off and falls back to its
|
||||
# Triton kernels on ROCm.
|
||||
#
|
||||
# Base is digest-pinned (2026-10-05): sha256:a223aee1… is the manifest this
|
||||
# build was qualified against; a re-pushed tag cannot change the stack. The
|
||||
# readable tag rides along for humans (podman verifies the digest matches).
|
||||
FROM rocm/pytorch:rocm7.14_ubuntu24.04_py3.12_pytorch_release_2.11.0@sha256:a223aee17aef5d21c3b9f63436dd19d27d1c665ec8b2f40011c9546cabae2a80
|
||||
|
||||
# Pin the PR head, not the branch: a future push to rocm714_torch214 would
|
||||
# otherwise silently change the deployed engine.
|
||||
# The pip-ROCm torch in this image ships libhiprtc as the VERSIONED
|
||||
# libhiprtc.so.7 only; the PR's `_cpu_moe` extension links `-lhiprtc`, and
|
||||
# ld looks for the unversioned dev symlink in the ALREADY-LINKED
|
||||
# _rocm_sdk_core/lib dir. Create it there (first build failure 2026-10-05:
|
||||
# `ld: cannot find -lhiprtc`).
|
||||
RUN ln -s libhiprtc.so.7 /opt/venv/lib/python3.12/site-packages/_rocm_sdk_core/lib/libhiprtc.so
|
||||
RUN git clone https://github.com/FlashML-org/FreeToken.git /opt/FreeToken \
|
||||
&& cd /opt/FreeToken \
|
||||
&& git fetch origin pull/587/head \
|
||||
&& git checkout 77a7201f98f00d93f618c6cf8b81ffe8360e0899
|
||||
|
||||
WORKDIR /opt/FreeToken
|
||||
RUN python -m pip install --no-build-isolation --no-deps -e .
|
||||
|
||||
# Curated runtime dependencies (docs/install_amd.md). transformers is pinned
|
||||
# to the pyproject window (>=5.16,<5.17); the rest are the doc's portable set.
|
||||
# torch / torchvision / triton-rocm come from the base image, 1:1 aligned.
|
||||
RUN python -m pip install \
|
||||
"transformers>=5.16,<5.17" tokenizers safetensors huggingface_hub \
|
||||
fastapi uvicorn pydantic openai httpx anyio \
|
||||
einops gguf msgpack pyzmq typer rich apache-tvm-ffi \
|
||||
partial-json-parser \
|
||||
&& python -m pip install --no-deps flashlib==0.3.0 \
|
||||
&& python -m pip install ninja
|
||||
Executable
+113
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env bash
|
||||
# download-frank-archon-swift15.sh — payload fetch/verify for the planned
|
||||
# frank-archon swap to the Swift 1.5 finetune: ukisai/Swift1.5-Qwen3.8-Flash-Next
|
||||
# (BF16 source, 149 files, 360,037,355,038 bytes = 335.3 GiB: 131 safetensors
|
||||
# shards + weight-map index + tokenizer + vision preprocessing configs + chat
|
||||
# template + licenses). Key set verified identical to the official
|
||||
# Qwen/Qwen3.8-Flash-Next base (0 added / 0 missing), so the qwen4exp recipes
|
||||
# apply verbatim; the checkpoint ships BF16 with NO quantization_config.
|
||||
# Repo is UNGATED — no token needed (never put one on the command line).
|
||||
#
|
||||
# bash scripts/download-frank-archon-swift15.sh --fetch resume/complete
|
||||
# bash scripts/download-frank-archon-swift15.sh verify only
|
||||
#
|
||||
# Fetch uses the vendored scripts/hfget.sh (StillDeadcode's Apache-2.0
|
||||
# listing/resume downloader, curl+jq only): per-file size checks against
|
||||
# the Hub listing, `-C -` resume, pickle-format duplicates skipped.
|
||||
# Verify (default) is the AGENTS.md law in three parts: every listed file
|
||||
# present with the listed size, the shard count matches the weight-map
|
||||
# index, and every shard's sha256 equals the Hub's LFS oid. Verify needs
|
||||
# only the public API.
|
||||
set -euo pipefail
|
||||
|
||||
REPO="ukisai/Swift1.5-Qwen3.8-Flash-Next"
|
||||
DEST="/home/dank/models/ukisai/Swift1.5-Qwen3.8-Flash-Next"
|
||||
EXPECTED_SHARDS=131
|
||||
|
||||
mode="verify"
|
||||
[[ "${1:-}" == "--fetch" ]] && mode="fetch"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
if [[ "$mode" == "fetch" ]]; then
|
||||
# HFGET_JOBS=4 is hfget's default (a few streams saturate the link).
|
||||
bash "$SCRIPT_DIR/hfget.sh" "$REPO" "$DEST" main
|
||||
fi
|
||||
|
||||
echo "verifying local payload: $DEST"
|
||||
fail=0
|
||||
|
||||
INDEX="$DEST/model.safetensors.index.json"
|
||||
if [[ ! -f "$INDEX" ]]; then
|
||||
echo " BAD weight-map index missing: $INDEX" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python3 - "$REPO" "$DEST" <<'PY' || fail=1
|
||||
import hashlib, json, os, sys, urllib.request
|
||||
|
||||
repo, dest = sys.argv[1], sys.argv[2]
|
||||
|
||||
def listing():
|
||||
"""The tree API, following the Link cursor exactly like hfget.sh."""
|
||||
out, api, pages = [], (
|
||||
f"https://huggingface.co/api/models/{repo}/tree/main"
|
||||
"?recursive=true&expand=true"), 0
|
||||
while api:
|
||||
with urllib.request.urlopen(api) as r:
|
||||
out += json.load(r)
|
||||
api = None
|
||||
link = r.headers.get("Link", "")
|
||||
for part in link.split(","):
|
||||
if 'rel="next"' in part:
|
||||
api = part[part.find("<") + 1:part.find(">")]
|
||||
pages += 1
|
||||
if pages > 200:
|
||||
raise SystemExit("listing did not terminate")
|
||||
return out
|
||||
|
||||
files = [e for e in listing() if e.get("type") == "file"]
|
||||
sizes = {e["path"]: e.get("size", 0) for e in files}
|
||||
lfs = {e["path"]: e["lfs"]["oid"] for e in files if e.get("lfs")}
|
||||
|
||||
bad = 0
|
||||
for path, want in sorted(sizes.items()):
|
||||
p = os.path.join(dest, path)
|
||||
if not os.path.isfile(p):
|
||||
print(f" BAD {path}: missing")
|
||||
bad += 1
|
||||
elif os.path.getsize(p) != want:
|
||||
print(f" BAD {path}: {os.path.getsize(p)} bytes, listing says {want}")
|
||||
bad += 1
|
||||
print(f" {'OK' if bad == 0 else 'FAIL'}: {len(sizes) - bad}/{len(sizes)} files present at listed size")
|
||||
if bad:
|
||||
raise SystemExit(1)
|
||||
|
||||
# Shard set against the weight-map index.
|
||||
wm = json.load(open(os.path.join(dest, "model.safetensors.index.json")))["weight_map"]
|
||||
shards = sorted(set(wm.values()))
|
||||
print(f" index maps {len(wm)} tensors across {len(shards)} shards")
|
||||
if len(shards) != 131:
|
||||
print(f" BAD shard count {len(shards)}, want 131")
|
||||
raise SystemExit(1)
|
||||
|
||||
# Content: every LFS-tracked file hashed against the Hub oid. The safetensors
|
||||
# shards dominate the bytes; tokenizer/config files ride the same check when
|
||||
# the Hub tracks them in LFS (they are plain blobs here, already size-checked).
|
||||
todo = [p for p in shards if p in lfs]
|
||||
for i, path in enumerate(todo):
|
||||
h = hashlib.sha256()
|
||||
with open(os.path.join(dest, path), "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 24), b""):
|
||||
h.update(chunk)
|
||||
if h.hexdigest() != lfs[path]:
|
||||
print(f" BAD {path}: sha256 mismatch (want {lfs[path]})")
|
||||
bad += 1
|
||||
if (i + 1) % 16 == 0:
|
||||
print(f" ...hashed {i + 1}/{len(todo)}")
|
||||
print(f" {'OK' if bad == 0 else 'FAIL'}: {len(todo) - bad}/{len(todo)} shards sha256-match the Hub")
|
||||
if bad:
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
|
||||
[[ "$fail" -eq 0 ]] || exit 1
|
||||
echo "payload verified: 131 shards + index + tokenizer + configs, sizes and sha256 checked"
|
||||
Executable
+111
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env bash
|
||||
# download-frank-archon.sh — payload fetch/verify for frank-archon:
|
||||
# the OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 checkpoint (131 safetensors
|
||||
# shards + weight-map index + tokenizer + vision preprocessing configs,
|
||||
# ~172.8 GB). 2026-10-05: replaces the orcarouter uncensored payload of the
|
||||
# FreeToken era (that script lived here; its repo was gated and needed
|
||||
# HF_TOKEN — this one is UNGATED and needs no token at all).
|
||||
#
|
||||
# bash scripts/download-frank-archon.sh --fetch resume/complete
|
||||
# bash scripts/download-frank-archon.sh verify only
|
||||
#
|
||||
# Fetch uses the vendored scripts/hfget.sh (StillDeadcode's Apache-2.0
|
||||
# listing/resume downloader, curl+jq only): per-file size checks against
|
||||
# the Hub listing, `-C -` resume, pickle-format duplicates skipped.
|
||||
# Verify (default) is the AGENTS.md law in three parts: every listed file
|
||||
# present with the listed size, the shard count matches the weight-map
|
||||
# index, and every shard's sha256 equals the Hub's LFS oid. Verify needs
|
||||
# only the public API.
|
||||
set -euo pipefail
|
||||
|
||||
REPO="Qwen/Qwen3.8-Flash-Next-FP8"
|
||||
DEST="/home/dank/models/Qwen/Qwen3.8-Flash-Next-FP8"
|
||||
EXPECTED_SHARDS=131
|
||||
|
||||
mode="verify"
|
||||
[[ "${1:-}" == "--fetch" ]] && mode="fetch"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
if [[ "$mode" == "fetch" ]]; then
|
||||
# HFGET_JOBS=4 is hfget's default (a few streams saturate the link).
|
||||
bash "$SCRIPT_DIR/hfget.sh" "$REPO" "$DEST" main
|
||||
fi
|
||||
|
||||
echo "verifying local payload: $DEST"
|
||||
fail=0
|
||||
|
||||
INDEX="$DEST/model.safetensors.index.json"
|
||||
if [[ ! -f "$INDEX" ]]; then
|
||||
echo " BAD weight-map index missing: $INDEX" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python3 - "$REPO" "$DEST" <<'PY' || fail=1
|
||||
import hashlib, json, os, sys, urllib.request
|
||||
|
||||
repo, dest = sys.argv[1], sys.argv[2]
|
||||
|
||||
def listing():
|
||||
"""The tree API, following the Link cursor exactly like hfget.sh."""
|
||||
out, api, pages = [], (
|
||||
f"https://huggingface.co/api/models/{repo}/tree/main"
|
||||
"?recursive=true&expand=true"), 0
|
||||
while api:
|
||||
with urllib.request.urlopen(api) as r:
|
||||
out += json.load(r)
|
||||
api = None
|
||||
link = r.headers.get("Link", "")
|
||||
for part in link.split(","):
|
||||
if 'rel="next"' in part:
|
||||
api = part[part.find("<") + 1:part.find(">")]
|
||||
pages += 1
|
||||
if pages > 200:
|
||||
raise SystemExit("listing did not terminate")
|
||||
return out
|
||||
|
||||
files = [e for e in listing() if e.get("type") == "file"]
|
||||
sizes = {e["path"]: e.get("size", 0) for e in files}
|
||||
lfs = {e["path"]: e["lfs"]["oid"] for e in files if e.get("lfs")}
|
||||
|
||||
bad = 0
|
||||
for path, want in sorted(sizes.items()):
|
||||
p = os.path.join(dest, path)
|
||||
if not os.path.isfile(p):
|
||||
print(f" BAD {path}: missing")
|
||||
bad += 1
|
||||
elif os.path.getsize(p) != want:
|
||||
print(f" BAD {path}: {os.path.getsize(p)} bytes, listing says {want}")
|
||||
bad += 1
|
||||
print(f" {'OK' if bad == 0 else 'FAIL'}: {len(sizes) - bad}/{len(sizes)} files present at listed size")
|
||||
if bad:
|
||||
raise SystemExit(1)
|
||||
|
||||
# Shard set against the weight-map index.
|
||||
wm = json.load(open(os.path.join(dest, "model.safetensors.index.json")))["weight_map"]
|
||||
shards = sorted(set(wm.values()))
|
||||
print(f" index maps {len(wm)} tensors across {len(shards)} shards")
|
||||
if len(shards) != 131:
|
||||
print(f" BAD shard count {len(shards)}, want 131")
|
||||
raise SystemExit(1)
|
||||
|
||||
# Content: every LFS-tracked file hashed against the Hub oid. The safetensors
|
||||
# shards dominate the bytes; tokenizer/config files ride the same check when
|
||||
# the Hub tracks them in LFS (they are plain blobs here, already size-checked).
|
||||
todo = [p for p in shards if p in lfs]
|
||||
for i, path in enumerate(todo):
|
||||
h = hashlib.sha256()
|
||||
with open(os.path.join(dest, path), "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 24), b""):
|
||||
h.update(chunk)
|
||||
if h.hexdigest() != lfs[path]:
|
||||
print(f" BAD {path}: sha256 mismatch (want {lfs[path]})")
|
||||
bad += 1
|
||||
if (i + 1) % 16 == 0:
|
||||
print(f" ...hashed {i + 1}/{len(todo)}")
|
||||
print(f" {'OK' if bad == 0 else 'FAIL'}: {len(todo) - bad}/{len(todo)} shards sha256-match the Hub")
|
||||
if bad:
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
|
||||
[[ "$fail" -eq 0 ]] || exit 1
|
||||
echo "payload verified: 131 shards + index + tokenizer + configs, sizes and sha256 checked"
|
||||
Executable
+119
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env bash
|
||||
# download-frank-dragoon-flashnext.sh — fetch frank-dragoon's flash-server v2
|
||||
# payload: the Quat3rnion abliterated Swift1.5 checkpoint of peonist's v2
|
||||
# .hgn, the REQUIRED lookup table and the optional vision tower from
|
||||
# peonist-ai/halogen-qwen3.8-flash-next (both at the revisions the model
|
||||
# card pins), and the flat tokenizer the two repos share. All public, no
|
||||
# token — and per the fleet secret law, none would ever touch argv.
|
||||
#
|
||||
# The fetch law, same as every sibling script: curl -f (an access error
|
||||
# must not become a "successful" 153-byte artifact), exact byte count from
|
||||
# the Hub tree API, LFS sha256 verification, --verify-only to re-check an
|
||||
# existing payload without downloading.
|
||||
#
|
||||
# File manifest (verified against the Hub tree API on 2026-10-06):
|
||||
# qwen38-flash-next-v2-swift15-abliterated.hgn 66,687,678,464 B sha256
|
||||
# 9c880b8ba48deabfa06172eff1e7572e4fe465e09e97faec6e50c3f6fdcac037
|
||||
# (Quat3rnion main; compacted r2 of the card's 89,775,868,032 B
|
||||
# append-only original — byte-identical live tensors, dropped padding)
|
||||
# qwen38-flash-next-ngram.hgn 51,200,246,144 B sha256
|
||||
# 3450acada94e19aabad88bd49b45eb70304820949c2240fc178067b20ae5dffe
|
||||
# (peonist @5cc17cea; REQUIRED — the engine reads it beside the
|
||||
# checkpoint)
|
||||
# qwen38-flash-next-vision.hgn 897,916,416 B sha256
|
||||
# d62e0ae553fe88afd3833733d4a4c669f34d20fd8dfce4b9610525bed2134b10
|
||||
# (peonist @e053f488; optional, enables image input)
|
||||
# tokenizer/* @e053f488 — byte-identical to the 27B-era tokenizer
|
||||
# already on disk (tokenizer.json LFS
|
||||
# 0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3);
|
||||
# fetched fresh rather than symlinked, per the no-symlink law.
|
||||
set -euo pipefail
|
||||
|
||||
DEST="${FRANK_DRAGOON_MODEL_DIR:-$HOME/models/halogen-flashnext-v2}"
|
||||
ABLIT="https://huggingface.co/Quat3rnion/halogen-swift1.5-qwen3.8-flash-next-v2-abliterated/resolve/main"
|
||||
PEONIST="https://huggingface.co/peonist-ai/halogen-qwen3.8-flash-next/resolve"
|
||||
NGRAM_REV="5cc17cea1a10b502b4a5db41d8d2b1a0f22dba26"
|
||||
TOK_REV="e053f488b120b99ed2525e6ac99f68c51b3b6179"
|
||||
|
||||
FILES=(qwen38-flash-next-v2-swift15-abliterated.hgn
|
||||
qwen38-flash-next-ngram.hgn
|
||||
qwen38-flash-next-vision.hgn
|
||||
tokenizer/tokenizer.json
|
||||
tokenizer/chat_template.jinja
|
||||
tokenizer/generation_config.json
|
||||
tokenizer/merges.txt
|
||||
tokenizer/vocab.json
|
||||
tokenizer/tokenizer_config.json)
|
||||
SIZES=(66687678464
|
||||
51200246144
|
||||
897916416
|
||||
12809320
|
||||
8952
|
||||
202
|
||||
3353259
|
||||
6722759
|
||||
17928)
|
||||
SHAS=(9c880b8ba48deabfa06172eff1e7572e4fe465e09e97faec6e50c3f6fdcac037
|
||||
3450acada94e19aabad88bd49b45eb70304820949c2240fc178067b20ae5dffe
|
||||
d62e0ae553fe88afd3833733d4a4c669f34d20fd8dfce4b9610525bed2134b10
|
||||
0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3
|
||||
- - - - - -)
|
||||
|
||||
# Fetch source per file: repo base + revision (empty = ABLIT @main).
|
||||
SRCS=("$ABLIT"
|
||||
"$PEONIST/$NGRAM_REV"
|
||||
"$PEONIST/$TOK_REV"
|
||||
"$PEONIST/$TOK_REV"
|
||||
"$PEONIST/$TOK_REV"
|
||||
"$PEONIST/$TOK_REV"
|
||||
"$PEONIST/$TOK_REV"
|
||||
"$PEONIST/$TOK_REV"
|
||||
"$PEONIST/$TOK_REV")
|
||||
|
||||
VERIFY_ONLY=0
|
||||
[ "${1:-}" = "--verify-only" ] && VERIFY_ONLY=1
|
||||
|
||||
mkdir -p "$DEST"
|
||||
|
||||
for i in "${!FILES[@]}"; do
|
||||
f="${FILES[$i]}"; want="${SIZES[$i]}"; sha="${SHAS[$i]}"; base="${SRCS[$i]}"
|
||||
path="$DEST/$f"
|
||||
mkdir -p "$(dirname "$path")"
|
||||
if [ "$VERIFY_ONLY" -eq 0 ]; then
|
||||
# A file already at the published size skips the fetch entirely:
|
||||
# resuming it would ask the CDN for a range past EOF (416 -> curl
|
||||
# error under set -e), and re-pulling 62 GiB to prove a point is
|
||||
# not what --verify-only is for. sha256 below still decides.
|
||||
if [ -f "$path" ] && [ "$(stat -c %s "$path")" = "$want" ]; then
|
||||
echo "HAVE $f ($want bytes) — skipping fetch"
|
||||
else
|
||||
echo "FETCH $f -> $path"
|
||||
# HF answers /resolve/ with a 302 to the Xet CDN and curl DROPS
|
||||
# the Range header across redirects — a naive -C - then
|
||||
# "resumes" by pulling the ~1 KB redirect body (measured
|
||||
# 2026-10-05 on the 27B fetch: file frozen 14 MiB short, curl
|
||||
# exit 0). Resolve the effective URL first, then resume
|
||||
# against the CDN, which honors Range.
|
||||
url=$(curl -sfIL -o /dev/null -w '%{url_effective}' "$base/$f") \
|
||||
|| { echo "cannot resolve $base/$f" >&2; exit 1; }
|
||||
curl -fL --retry 5 -C - -o "$path" "$url"
|
||||
fi
|
||||
fi
|
||||
[ -f "$path" ] || { echo "MISSING $path (run without --verify-only)" >&2; exit 1; }
|
||||
got="$(stat -c %s "$path")"
|
||||
[ "$got" = "$want" ] || { echo "SIZE $path: $got != $want" >&2; exit 1; }
|
||||
echo "OK $f ($got bytes)"
|
||||
done
|
||||
|
||||
echo "byte counts match; verifying sha256 where the Hub publishes an LFS oid..."
|
||||
for i in "${!FILES[@]}"; do
|
||||
f="${FILES[$i]}"; sha="${SHAS[$i]}"
|
||||
if [ "$sha" = "-" ]; then
|
||||
echo "OK $f (plain git blob, size-checked)"
|
||||
continue
|
||||
fi
|
||||
got="$(sha256sum "$DEST/$f" | cut -d' ' -f1)"
|
||||
[ "$got" = "$sha" ] || { echo "SHA256 $f: $got != $sha" >&2; exit 1; }
|
||||
echo "OK $f sha256"
|
||||
done
|
||||
echo "DOWNLOAD_DONE $DEST"
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# download-frank-large-halogen.sh — fetch frank-large's halogen payload:
|
||||
# download-frank-dragoon-halogen.sh — fetch frank-dragoon's halogen payload:
|
||||
# the qwen3.8-27b .hgn checkpoint and the export tokenizer, from the
|
||||
# PUBLIC peonist-ai/halogen-qwen3.8-27b repo (no token — and per the fleet
|
||||
# secret law, none would ever touch argv).
|
||||
@@ -10,7 +10,7 @@
|
||||
# existing payload without downloading.
|
||||
set -euo pipefail
|
||||
|
||||
DEST="${FRANK_LARGE_MODEL_DIR:-$HOME/models/halogen-qwen3.8-27b}"
|
||||
DEST="${FRANK_DRAGOON_MODEL_DIR:-$HOME/models/halogen-qwen3.8-27b}"
|
||||
BASE="https://huggingface.co/peonist-ai/halogen-qwen3.8-27b/resolve/main"
|
||||
|
||||
FILES=(qwen3.8-27b-p1w4d-d2.hgn
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF for
|
||||
# frank-large (Dankclaw iGPU). Public repo — no auth header, and never a token
|
||||
# frank-dragoon (Dankclaw iGPU). Public repo — no auth header, and never a token
|
||||
# on the command line (it is world-readable in /proc/<pid>/cmdline).
|
||||
#
|
||||
# The payload is the repo's Q6 AGENT rung:
|
||||
@@ -10,7 +10,7 @@
|
||||
# names for MTP serving ("Take Q6 AGENT (114) if you want protected
|
||||
# heads"). Like every ROCmFPX file it loads on nothing but the fork.
|
||||
#
|
||||
# The other two files frank-large needs are NOT re-downloaded:
|
||||
# The other two files frank-dragoon needs are NOT re-downloaded:
|
||||
# * mtp-Qwen3.8-27B-Q4_0.gguf — the uncensored repo redistributes the
|
||||
# ALIGNED repo's draft head byte for byte (identical Hub LFS oid
|
||||
# 051a1764…), so it is hardlinked out of the aligned payload dir when that
|
||||
@@ -25,7 +25,7 @@
|
||||
# byte count and the Hub LFS sha256. Idempotent: complete files are skipped.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-large-kingjones-uncensored.sh [DEST_DIR] [--verify-only]
|
||||
# scripts/download-frank-dragoon-kingjones-uncensored.sh [DEST_DIR] [--verify-only]
|
||||
set -euo pipefail
|
||||
|
||||
REPO="kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF"
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF for frank-large
|
||||
# Download kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF for frank-dragoon
|
||||
# (Dankclaw iGPU). Public repo — no auth header, and never a token on the
|
||||
# command line (it is world-readable in /proc/<pid>/cmdline).
|
||||
#
|
||||
@@ -24,7 +24,7 @@
|
||||
# come from the Hub LFS oids (api/models/<repo>/tree/main).
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-large-kingjones.sh [DEST_DIR] [--verify-only]
|
||||
# scripts/download-frank-dragoon-kingjones.sh [DEST_DIR] [--verify-only]
|
||||
set -euo pipefail
|
||||
|
||||
REPO="kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF"
|
||||
@@ -1,9 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download and assemble jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca (the frank-large
|
||||
# Download and assemble jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca (the frank-dragoon
|
||||
# model) on Dankclaw.
|
||||
#
|
||||
# bash scripts/download-frank-large-orca.sh
|
||||
# bash scripts/download-frank-large-orca.sh --verify-only
|
||||
# bash scripts/download-frank-dragoon-orca.sh
|
||||
# bash scripts/download-frank-dragoon-orca.sh --verify-only
|
||||
#
|
||||
# The package is 136 GB of Hub-sized transport parts (HF caps new uploads at
|
||||
# 50 GB, so the 79 GB main GGUF, the 52 GB PLE payload and the 4 GB MTP head
|
||||
@@ -105,4 +105,4 @@ fi
|
||||
echo "verifying assembled files against SHA256SUMS"
|
||||
(cd "$MODEL_DIR" && sha256sum -c SHA256SUMS)
|
||||
(cd "$MODEL_DIR/vision" && sha256sum -c SHA256SUMS)
|
||||
echo "frank-large payload complete: $MODEL_DIR"
|
||||
echo "frank-dragoon payload complete: $MODEL_DIR"
|
||||
@@ -1,9 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the Q4_K_XL frank-large payload on Dankclaw:
|
||||
# Download the Q4_K_XL frank-dragoon payload on Dankclaw:
|
||||
# unsloth/Qwen3.8-27B-GGUF -> Qwen3.8-27B-UD-Q4_K_XL.gguf
|
||||
#
|
||||
# bash scripts/download-frank-large-q4.sh
|
||||
# bash scripts/download-frank-large-q4.sh --verify-only
|
||||
# bash scripts/download-frank-dragoon-q4.sh
|
||||
# bash scripts/download-frank-dragoon-q4.sh --verify-only
|
||||
#
|
||||
# Integrity: the file is checked against its declared byte count and sha256
|
||||
# (the LFS oid from the Hub tree API, 2026-10-01). NO AUTH HEADER IS USED:
|
||||
@@ -21,12 +21,12 @@
|
||||
# x 2 (K,V) x 2 B), so 4 sessions x 163840 ctx = 640,000 tokens = 39.06 GiB.
|
||||
#
|
||||
# The speculative drafter is UNCHANGED and still fetched by its own script
|
||||
# (scripts/download-frank-large.sh, z-lab's Hub repo). That name must not
|
||||
# appear here at all — the frank-large seam greps THIS file for it to prove
|
||||
# (scripts/download-frank-dragoon.sh, z-lab's Hub repo). That name must not
|
||||
# appear here at all — the frank-dragoon seam greps THIS file for it to prove
|
||||
# the drafter was not folded into the payload fetch.
|
||||
#
|
||||
# NOTE ON MTP: this payload has no baked-in MTP head (the Hub repo ships one
|
||||
# separately as MTP/mtp-Qwen3.8-27B-Q4_0.gguf). frank-large does not need it —
|
||||
# separately as MTP/mtp-Qwen3.8-27B-Q4_0.gguf). frank-dragoon does not need it —
|
||||
# spec decode here is the block-diffusion drafter, not MTP.
|
||||
set -euo pipefail
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the Swift-1.5 frank-large candidate on Dankclaw:
|
||||
# Download the Swift-1.5 frank-dragoon candidate on Dankclaw:
|
||||
# ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF
|
||||
# Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf
|
||||
#
|
||||
# bash scripts/download-frank-large-swift.sh
|
||||
# bash scripts/download-frank-large-swift.sh --verify-only
|
||||
# bash scripts/download-frank-dragoon-swift.sh
|
||||
# bash scripts/download-frank-dragoon-swift.sh --verify-only
|
||||
#
|
||||
# Integrity: the file is checked against its declared byte count and sha256
|
||||
# (the LFS oid from the Hub, 2026-09-28). NO AUTH HEADER IS USED: the repo is
|
||||
@@ -15,9 +15,9 @@
|
||||
# Layout after a successful run (MODEL_DIR):
|
||||
# Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf 31457990272
|
||||
#
|
||||
# This is the CANDIDATE payload, not the deployed one. frank-large still
|
||||
# serves scripts/download-frank-large.sh's output; do not repoint
|
||||
# frank-large.env at MODEL_DIR until the gufo load test passes.
|
||||
# This is the CANDIDATE payload, not the deployed one. frank-dragoon still
|
||||
# serves scripts/download-frank-dragoon.sh's output; do not repoint
|
||||
# frank-dragoon.env at MODEL_DIR until the gufo load test passes.
|
||||
set -euo pipefail
|
||||
|
||||
MODEL_DIR="/home/dank/models/Swift-1.5-Qwen3.8-27B-Uncensored-MTP-UD-Q8_K_XL"
|
||||
@@ -1,9 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download unsloth/Qwen3.8-Flash-Next-GGUF:UD-IQ4_XS (the
|
||||
# frank-large model) on Dankclaw.
|
||||
# frank-dragoon model) on Dankclaw.
|
||||
#
|
||||
# bash scripts/download-frank-large-unsloth.sh
|
||||
# bash scripts/download-frank-large-unsloth.sh --verify-only
|
||||
# bash scripts/download-frank-dragoon-unsloth.sh
|
||||
# bash scripts/download-frank-dragoon-unsloth.sh --verify-only
|
||||
#
|
||||
# The UD-IQ4_XS GGUF ships as three llama.cpp split parts (HF caps uploads at
|
||||
# 50 GB). The parts ARE the canonical payload: the serve script points -m at
|
||||
@@ -112,4 +112,4 @@ if [[ "$mode" == "download" ]]; then
|
||||
[[ -e "$MODEL_DIR/vision/mmproj-F16.gguf" ]] || mv "$MODEL_DIR/mmproj-F16.gguf" "$MODEL_DIR/vision/"
|
||||
fi
|
||||
|
||||
echo "frank-large payload complete: $MODEL_DIR"
|
||||
echo "frank-dragoon payload complete: $MODEL_DIR"
|
||||
@@ -1,10 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the frank-large payload on Dankclaw:
|
||||
# Download the frank-dragoon payload on Dankclaw:
|
||||
# - unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL (the 27B dense main GGUF)
|
||||
# - z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M (the DFlash2 drafter)
|
||||
#
|
||||
# bash scripts/download-frank-large.sh
|
||||
# bash scripts/download-frank-large.sh --verify-only
|
||||
# bash scripts/download-frank-dragoon.sh
|
||||
# bash scripts/download-frank-dragoon.sh --verify-only
|
||||
#
|
||||
# Integrity: every file is checked against its declared byte count and sha256
|
||||
# (the values below are the LFS oids from the Hub, 2026-09-21). An HF token
|
||||
@@ -86,4 +86,4 @@ if [[ "$mode" == "download" ]]; then
|
||||
done
|
||||
fi
|
||||
|
||||
echo "frank-large payload complete: $MODEL_DIR"
|
||||
echo "frank-dragoon payload complete: $MODEL_DIR"
|
||||
@@ -1,88 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Payload inventory/verification for frank-primary:
|
||||
# orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8 (compressed-tensors FP8, 131
|
||||
# shards + weight-map index, ~172.8 GiB) — the tcclaviger-stack payload that
|
||||
# became frank-primary on 2026-09-29.
|
||||
#
|
||||
# bash scripts/download-frank-primary.sh --verify-only (the ONLY mode)
|
||||
# bash scripts/download-frank-primary.sh --fetch
|
||||
#
|
||||
# THIS REPO IS GATED (401 anonymous — measured 2026-09-29): anonymous fetch
|
||||
# is impossible and the tree API masks LFS hashes, so a byte-exact upstream
|
||||
# manifest cannot be built without the authed endpoint. The payload on
|
||||
# /home/dank/models was fetched with the operator's HF token (out of band:
|
||||
# HF_TOKEN in the environment, never in argv) at bring-up time.
|
||||
#
|
||||
# Modes:
|
||||
# --fetch resume any missing/truncated shard via `hf download` with
|
||||
# HF_TOKEN from the environment (NEVER passed on the command
|
||||
# line — a -H auth header is world-readable in
|
||||
# /proc/<pid>/cmdline, the 2026-09-16 leak).
|
||||
# --verify-only default: local integrity check — shard count vs the
|
||||
# weight-map index, every file nonzero and its size equal to
|
||||
# the index's declared total, plus sha256 of every shard
|
||||
# (slow, ~10 min for 173 GiB).
|
||||
set -euo pipefail
|
||||
|
||||
REPO="orcarouter/Qwen3.8-Flash-Next-Uncensored-FP8"
|
||||
DEST="/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8"
|
||||
EXPECTED_SHARDS=131
|
||||
|
||||
mode="verify"
|
||||
[[ "${1:-}" == "--fetch" ]] && mode="fetch"
|
||||
|
||||
if [[ "$mode" == "fetch" ]]; then
|
||||
: "${HF_TOKEN:?--fetch needs HF_TOKEN in the environment (never argv)}"
|
||||
export HF_TOKEN
|
||||
mkdir -p "$DEST"
|
||||
hf download "$REPO" --local-dir "$DEST"
|
||||
fi
|
||||
|
||||
echo "verifying local payload: $DEST"
|
||||
fail=0
|
||||
|
||||
# The weight-map index is the source of truth for the shard set.
|
||||
INDEX="$DEST/model.safetensors.index.json"
|
||||
if [[ ! -f "$INDEX" ]]; then
|
||||
echo " BAD weight-map index missing: $INDEX" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Every index entry must exist, nonzero, and sized equal to the map's total.
|
||||
python3 - "$INDEX" "$DEST" <<'PY' || fail=1
|
||||
import json, os, sys
|
||||
index, dest = sys.argv[1], sys.argv[2]
|
||||
wm = json.load(open(index))["weight_map"]
|
||||
files = sorted(set(wm.values()))
|
||||
print(f" index maps {len(wm)} tensors across {len(files)} files")
|
||||
bad = 0
|
||||
total = 0
|
||||
for f in files:
|
||||
p = os.path.join(dest, f)
|
||||
if not os.path.isfile(p) or os.path.getsize(p) == 0:
|
||||
print(f" BAD {f}: missing or zero bytes" )
|
||||
bad += 1
|
||||
else:
|
||||
total += os.path.getsize(p)
|
||||
print(f" {'OK' if bad == 0 else 'FAIL'}: {len(files) - bad}/{len(files)} files present, {total/2**30:.1f} GiB total")
|
||||
raise SystemExit(1 if bad else 0)
|
||||
PY
|
||||
|
||||
# Shard count gate (what the serve script and deployer check).
|
||||
n="$(find "$DEST" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l)"
|
||||
if [[ "$n" != "$EXPECTED_SHARDS" ]]; then
|
||||
echo " BAD shard count: found $n, want $EXPECTED_SHARDS" >&2
|
||||
fail=1
|
||||
else
|
||||
echo " OK shard count $n"
|
||||
fi
|
||||
|
||||
if [[ "${2:-}" == "--hash" || "${VERIFY_HASH:-0}" == "1" ]]; then
|
||||
echo "hashing every shard (slow)..."
|
||||
cd "$DEST"
|
||||
sha256sum -c --quiet model.safetensors.index.json 2>/dev/null || \
|
||||
sha256sum model-*-of-00131.safetensors > /dev/null # presence check; content hash list printed on demand
|
||||
fi
|
||||
|
||||
[[ "$fail" -eq 0 ]] || exit 1
|
||||
echo "payload inventory OK: $EXPECTED_SHARDS shards + index present"
|
||||
@@ -1,87 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the frank-small payload for llama.cpp on the RTX 3090:
|
||||
# ornith-ai/Ornith-1.5-9B-GGUF:Q8_0 plus its vision projector
|
||||
# (the GGUF carries no `v.*` tensors — the tower lives only in the mmproj, so
|
||||
# without this file the endpoint is text-only by construction). Public repo —
|
||||
# no auth header, and never a token on the command line (it is world-readable
|
||||
# in /proc/<pid>/cmdline).
|
||||
#
|
||||
# Fresh downloads, no resume (`curl -C -` once produced a 213 MB oversized file
|
||||
# on a 31 GiB payload); -f/--fail-with-body so an HTTP 401 is not a success,
|
||||
# then byte count and the Hub LFS sha256. Idempotent: complete files are
|
||||
# skipped.
|
||||
#
|
||||
# 2026-10-05 (Frank): the rung is Ornith-1.5-9B Q8_0. The payload moved off
|
||||
# MiMo-V2.6 (Q6_K) to Ornith because it is the fleet's own model and it is the
|
||||
# only one the harness's own prompt recipes were written against; Q8_0 was the
|
||||
# rung chosen so the quant does not cost the throughput this endpoint already
|
||||
# runs at. The Q5_K_M file this script served from 2026-10-02 stays on disk at
|
||||
# /home/dank/models/MiMo-Q5KM/ (6876124704 B, LFS oid
|
||||
# 04cf46f2b5584ef922697d6c063f94879434fb86c467f1a09e52daf0cddaf35f) as the
|
||||
# rollback target; re-fetch it from the same repo with the same recipe.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-small.sh [DEST_DIR] [--verify-only]
|
||||
set -euo pipefail
|
||||
|
||||
REPO="ornith-ai/Ornith-1.5-9B-GGUF"
|
||||
BASE="https://huggingface.co/${REPO}/resolve/main"
|
||||
|
||||
DEST=""
|
||||
VERIFY_ONLY=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--verify-only) VERIFY_ONLY=1 ;;
|
||||
*) DEST="$arg" ;;
|
||||
esac
|
||||
done
|
||||
DEST="${DEST:-/home/dank/models/Ornith-Q8}"
|
||||
|
||||
# name bytes sha256 (the sha256 is the LFS oid in the repo's pointer file)
|
||||
FILES=(
|
||||
"Ornith-1.5-9B-Q8_0.gguf 9786060384 22086870b009dbe9815ee752c48a82de930118a7c5ce5599590892ae03b8b010"
|
||||
"mmproj-Ornith-1.5-9B-BF16.gguf 921704672 626f9f90627402a6bf4a999111d0fbd69b5fcca7aa8ba089d69e5f10e8858e1d"
|
||||
)
|
||||
|
||||
mkdir -p "$DEST"
|
||||
status=0
|
||||
for entry in "${FILES[@]}"; do
|
||||
read -r name size sha <<<"$entry"
|
||||
f="$DEST/$name"
|
||||
have=0
|
||||
if [[ -f "$f" ]] && [[ "$(stat -c%s "$f")" == "$size" ]]; then
|
||||
if sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
have=1
|
||||
fi
|
||||
fi
|
||||
if (( have )); then
|
||||
echo "OK $name"
|
||||
continue
|
||||
fi
|
||||
if (( VERIFY_ONLY )); then
|
||||
echo "MISSING $name"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
echo "FETCHING $name"
|
||||
tmp="$f.part"
|
||||
curl -fsSL --fail-with-body -o "$tmp" "$BASE/$name"
|
||||
mv "$tmp" "$f"
|
||||
if [[ "$(stat -c%s "$f")" != "$size" ]]; then
|
||||
echo "SIZE FAIL $name ($(stat -c%s "$f") != $size)"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
if ! sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
echo "SHA FAIL $name"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
echo "VERIFIED $name"
|
||||
done
|
||||
if (( status == 0 )); then
|
||||
echo "DOWNLOAD_DONE $DEST"
|
||||
else
|
||||
echo "DOWNLOAD_FAILED $DEST" >&2
|
||||
fi
|
||||
exit "$status"
|
||||
Executable
+90
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the frank-zealot payload for llama.cpp on the RTX 3090:
|
||||
# HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive at **Q6_K** plus the
|
||||
# PUBLISHER's OWN vision projector (the BF16 mmproj — the model is natively
|
||||
# multimodal, so this file is the endpoint's eyes). Public repo — no auth
|
||||
# header, and never a token on the command line (it is world-readable in
|
||||
# /proc/<pid>/cmdline).
|
||||
#
|
||||
# Fresh downloads, no resume (`curl -C -` once produced a 213 MB oversized
|
||||
# file on a 31 GiB payload); -f/--fail-with-body so an HTTP error is not a
|
||||
# success, then byte count and sha256. The Hub exposes no LFS oid for this
|
||||
# repo (Xet storage), so the sha256 values below were computed locally after
|
||||
# the first verified fetch — byte counts match the Hub's declared sizes.
|
||||
# A sha of "-" means "first fetch: size-verified, sha printed for pinning".
|
||||
# Idempotent: complete files are skipped.
|
||||
#
|
||||
# 2026-10-07 (Frank): the payload moved to Qwen3.5-9B-Uncensored Q6_K at
|
||||
# 2 x 200,704 (qwen35 hybrid: 32 layers, 8 full-attention + 24 Gated
|
||||
# DeltaNet, no MTP tensors in this file). Rollback payloads stay on disk:
|
||||
# /home/dank/models/Gemma4-26B-A4B/ (Q4_K_M, the 2026-10-07 era being
|
||||
# replaced), Swift15-UncIQ3XXS/, MiMo-Q6K/, MiMo-Q5KM/, Ornith-Q8/.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-zealot.sh [DEST_DIR] [--verify-only]
|
||||
set -euo pipefail
|
||||
|
||||
DEST=""
|
||||
VERIFY_ONLY=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--verify-only) VERIFY_ONLY=1 ;;
|
||||
*) DEST="$arg" ;;
|
||||
esac
|
||||
done
|
||||
DEST="${DEST:-/home/dank/models/Qwen35-9B-HauhauCS-Q6K}"
|
||||
|
||||
# repo | hub-name | dest-name | bytes | sha256 (computed locally; Xet repo)
|
||||
FILES=(
|
||||
"HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive|Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K.gguf|Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K.gguf|7359259008|c0ba7beb68fd3fe47891bd549486d38dcf62d00817296ea314ad37017f5a4986"
|
||||
"HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive|mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf|mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf|921704448|05f662501f8bd45607b079723a3e238a4e888fd085a10a53f4057a0e250f6934"
|
||||
)
|
||||
|
||||
status=0
|
||||
for entry in "${FILES[@]}"; do
|
||||
IFS='|' read -r repo hubname destname size sha <<<"$entry"
|
||||
url="https://huggingface.co/${repo}/resolve/main/${hubname}"
|
||||
f="$DEST/$destname"
|
||||
have=0
|
||||
if [[ -f "$f" ]] && [[ "$(stat -c%s "$f")" == "$size" ]]; then
|
||||
if [[ "$sha" == "-" ]]; then
|
||||
have=1
|
||||
elif sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
have=1
|
||||
fi
|
||||
fi
|
||||
if (( have )); then
|
||||
echo "OK $destname"
|
||||
continue
|
||||
fi
|
||||
if (( VERIFY_ONLY )); then
|
||||
echo "MISSING $destname"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
echo "FETCHING $destname (from $repo)"
|
||||
tmp="$f.part"
|
||||
curl -fsSL --fail-with-body -o "$tmp" "$url"
|
||||
mv "$tmp" "$f"
|
||||
if [[ "$(stat -c%s "$f")" != "$size" ]]; then
|
||||
echo "SIZE FAIL $destname ($(stat -c%s "$f") != $size)"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
if [[ "$sha" == "-" ]]; then
|
||||
echo "VERIFIED $destname (size only) sha256=$(sha256sum "$f" | cut -d' ' -f1)"
|
||||
continue
|
||||
fi
|
||||
if ! sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then
|
||||
echo "SHA FAIL $destname"
|
||||
status=1
|
||||
continue
|
||||
fi
|
||||
echo "VERIFIED $destname"
|
||||
done
|
||||
if (( status == 0 )); then
|
||||
echo "DOWNLOAD_DONE $DEST"
|
||||
else
|
||||
echo "DOWNLOAD_FAILED $DEST" >&2
|
||||
fi
|
||||
exit "$status"
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env bash
|
||||
# fleet-probes-swift15.sh — RUNS ON DANKSERVER. The cutover contract probes.
|
||||
set -uo pipefail
|
||||
B=http://localhost:7770/v1
|
||||
|
||||
echo "== vision (solid red 512px, expect 'Red')"
|
||||
python3 - <<'PY'
|
||||
import base64, json, urllib.request, struct, zlib
|
||||
def png(color):
|
||||
w = h = 512
|
||||
raw = b"".join(b"\x00" + bytes(color) * w for _ in range(h))
|
||||
def chunk(t, d):
|
||||
c = t + d
|
||||
return struct.pack(">I", len(d)) + c + struct.pack(">I", zlib.crc32(c))
|
||||
return (b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0))
|
||||
+ chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b""))
|
||||
b64 = base64.b64encode(png((255, 0, 0))).decode()
|
||||
body = json.dumps({"model": "frank-archon", "max_tokens": 256, "temperature": 0,
|
||||
"messages": [{"role": "user", "content": [
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64}},
|
||||
{"type": "text", "text": "What color is this image? Answer with one word."}]}]}).encode()
|
||||
r = urllib.request.Request("http://localhost:7770/v1" + "/chat/completions",
|
||||
data=body, headers={"Content-Type": "application/json"})
|
||||
d = json.load(urllib.request.urlopen(r, timeout=300))
|
||||
m = d["choices"][0]["message"]
|
||||
ans = (m.get("content") or m.get("reasoning_content") or "")
|
||||
print("VISION ANSWER:", ans[:80].replace("\n", " "))
|
||||
PY
|
||||
|
||||
echo "== tools (expect tool_calls)"
|
||||
curl -s -m 120 $B/chat/completions -H 'Content-Type: application/json' -d '{
|
||||
"model":"frank-archon","temperature":0,"max_tokens":128,
|
||||
"messages":[{"role":"user","content":"What is 23 times 47? Use the calculator tool."}],
|
||||
"tools":[{"type":"function","function":{"name":"calculator","description":"evaluate arithmetic",
|
||||
"parameters":{"type":"object","properties":{"expr":{"type":"string"}},"required":["expr"]}}}]}' \
|
||||
| python3 -c 'import json,sys; d=json.load(sys.stdin); m=d["choices"][0]["message"]; print("TOOL_CALLS:", m.get("tool_calls"))'
|
||||
|
||||
echo "== reasoning_effort fold (xhigh/medium/low/high all 200 on froggeric v22.5)"
|
||||
for lvl in xhigh medium low high; do
|
||||
code=$(curl -s -o /dev/null -w "%{http_code}" -m 120 $B/chat/completions \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"model\":\"frank-archon\",\"max_tokens\":8,\"reasoning_effort\":\"$lvl\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]}")
|
||||
echo " reasoning_effort=$lvl -> HTTP $code"
|
||||
done
|
||||
|
||||
echo "== reasoning_content channel"
|
||||
curl -s -m 120 $B/chat/completions -H 'Content-Type: application/json' \
|
||||
-d '{"model":"frank-archon","max_tokens":64,"messages":[{"role":"user","content":"Is 97 prime? Think briefly."}]}' \
|
||||
| python3 -c 'import json,sys; m=json.load(sys.stdin)["choices"][0]["message"]; print("has reasoning_content:", bool(m.get("reasoning_content")), "| content:", (m.get("content") or "")[:60].replace(chr(10)," "))'
|
||||
|
||||
echo "== response_format json_object"
|
||||
curl -s -m 120 $B/chat/completions -H 'Content-Type: application/json' \
|
||||
-d '{"model":"frank-archon","max_tokens":64,"response_format":{"type":"json_object"},"messages":[{"role":"user","content":"Return JSON {\"ok\": true}"}]}' \
|
||||
| python3 -c 'import json,sys; d=json.load(sys.stdin); print("JSON MODE:", (d["choices"][0]["message"].get("content") or "")[:60])'
|
||||
echo "PROBES_DONE"
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure per-block / per-expert expert-routing activity on live frank-primary.
|
||||
"""Measure per-block / per-expert expert-routing activity on live frank-archon.
|
||||
|
||||
Method: AMD IBS op sampling (data addresses, `perf record -e ibs_op// --data`)
|
||||
on the llama-server process while concurrent generations of mixed content run.
|
||||
@@ -0,0 +1,286 @@
|
||||
# frank-archon.env — radiance (StillDeadcode, Apache-2.0) serving
|
||||
# ukisai/Swift1.5-Qwen3.8-Flash-Next converted to GPTQ w4 (MODEL_FILE below,
|
||||
# built by scripts/convert-swift15-w4dn8.sh) on BOTH R9700s (TP=2),
|
||||
# dankserver :7770.
|
||||
#
|
||||
# SHAPE (2026-10-13): THREE decode slots x 200,000 tokens, fp8 (quantized) KV,
|
||||
# with the per-card budget spent on resident experts: 25,600 MiB of slab +
|
||||
# 5,120 MiB of pool = 30,720 MiB, inside the 30,969 MiB ceiling the engine's own
|
||||
# budget table allows (card 32,624 MiB, settle floor 209 MiB). The pool is sized
|
||||
# to the contract and nothing more (3 x 200,000 needs 5,063 MiB at fp8; 5,120
|
||||
# predicts 608,027 tokens, 1.3 % slack), because every MiB taken from the slab
|
||||
# is prefill and TTFT (2026-10-11 A/B: slab -14.8 % cost prefill -29 %, TTFT
|
||||
# +25 % at the full window) while KV beyond the contract buys nothing.
|
||||
# Consequence, deliberate: a FOURTH full-height conversation does not fit this
|
||||
# pair, and the engine refuses it — it never truncates. See the KV block for the
|
||||
# arithmetic and the measured dtype law. The fp8-native payload this file was
|
||||
# written for is the rollback target and keeps its own trio under the
|
||||
# fp8-native runbook.
|
||||
#
|
||||
# 2026-10-05 cutover from FreeToken (PR #587 ROCm, one card, experts in
|
||||
# pinned host RAM; git tag frank-archon-freetoken-pr587-20261005 is the
|
||||
# pre-cutover state — its unit/env/serve trio and Dockerfile are all in the
|
||||
# tag and rebuildable). Before that: tcclaviger vLLM fork (tag
|
||||
# frank-archon-vllm-shapeG-20261003).
|
||||
#
|
||||
# WHY RADIANCE: the engine is C++/HIP with libr4d kernels built for
|
||||
# gfx1201, drives `podman run` directly from the digest-pinned
|
||||
# stilldeadcode/radiance image (287 MiB; host needs only the amdgpu driver),
|
||||
# and its qwen4exp plugin reads the modelopt FP8 encoding NATIVELY
|
||||
# (`X.weight_scale_inv` is the scale plane — checkpoint.h). The PLE n-gram
|
||||
# table (51B params, 47.7 GiB) is served from pinned host RAM via io_uring
|
||||
# reads — Frank's call: "Native FP8. Experts from ram. Pin the PLE to ram."
|
||||
# The 4-bit-codebook publisher shape (qwen3.8-next-flash-fp8-iq4r-moe.rad,
|
||||
# --host-pool-mib 12288) remains the documented fallback if the native
|
||||
# container proves unbootable.
|
||||
#
|
||||
# PAYLOAD: /home/dank/models/rad/qwen4exp-fp8-native.rad — a rad-convert
|
||||
# NO-RECIPE pure copy of the official 131-shard checkpoint (every weight
|
||||
# keeps its block-FP8 encoding; ~161 GiB). The convert runs against a
|
||||
# staging dir whose config.json drops `quantization_config`: the descriptor
|
||||
# string is only the plugin SELECTION key (spec §2.4 — exact match, no
|
||||
# fallback), and no shipped plugin claims (qwen4exp, "fp8"); the shipped
|
||||
# qwen4exp plugin claims (qwen4exp, "") which is what recipe-built
|
||||
# containers carry. Per-weight encodings are data, not the descriptor —
|
||||
# removing the key changes nothing about what the weights hold. Built by
|
||||
# scripts/build-frank-archon-rad.sh (staging + convert + rad-info).
|
||||
#
|
||||
# SERVING FACTS (docs/GUIDE.md, docs/DOCKER.md, compose/flashnext.yaml):
|
||||
# - The HTTP listener binds AFTER the full load (main.cpp: engine.start()
|
||||
# then engine.serve()) — during load the port refuses connections, so a
|
||||
# bare curl /health gate is honest. /health == /ping: 200 JSON while
|
||||
# serving, 503 only while stopping (core/server/admin.cpp:385).
|
||||
# - Reasoning: --reasoning-format defaults to auto, emitting thinking in
|
||||
# `reasoning_content` (config.cpp). Tool calls: the qwen3.8 dialect, from
|
||||
# the chat format. Both need no parser flags.
|
||||
# - Sampling: the OpenAI endpoint accepts temperature, top_p, top_k, min_p,
|
||||
# typical_p, presence_penalty, frequency_penalty, repetition_penalty,
|
||||
# repeat_last_n, chat_template_kwargs, reasoning_effort,
|
||||
# enable_thinking, preserve_thinking (core/server/oai.cpp:194-205) —
|
||||
# models.yml's extraBody set is fully inside it. Unknown params are
|
||||
# refused (400), which is why nothing unverified rides the wire.
|
||||
# - MTP: the official checkpoint carries the mtp.* head (3,101 tensors,
|
||||
# mtp_num_hidden_layers=1) and the plugin probes draft depth 3 — the
|
||||
# publisher's throughput peak for this model. Stated explicitly with
|
||||
# --num-speculative-tokens 3.
|
||||
# - The fused TP all-reduce is declared over max-num-seqs x
|
||||
# (1 + --num-speculative-tokens) rows; 2 x 3 = 6 rows, under the 25-row
|
||||
# exactness bound of the wht6 wire.
|
||||
# Engine pin: Codeberg tag v1.0.13 (commit 53282fa), Docker Hub tag 1.0.13,
|
||||
# index digest below. REJECTED UPGRADE: 1.2.0 (sha256:665775ed1d90de6dbb
|
||||
# 25558e07fc60d20328f1346f96f3c6e00845eb763f01fa, tag 1.2.0, commit
|
||||
# e95d397) was benched 2026-10-06 on this exact shape (fp8-native rad,
|
||||
# vram pair, spec 2, seqs 3, 43.6k greedy probe): decode 29.21 t/s and
|
||||
# draft acceptance 0.7123 vs the 1.0.13 control's 30.08 and 0.7549.
|
||||
# Acceptance on the greedy harness is deterministic (the control
|
||||
# reproduced the recorded 0.755 exactly), so the drop is a real numerics
|
||||
# change — the 1.2.0 MoE weight-rotation change (92b8aaf) is the named
|
||||
# suspect; decode loss tracks the acceptance ratio (0.943 vs 0.945).
|
||||
# The vram-pair translation itself is exact (1.2.0 boot ledger took
|
||||
# vram_weights 25.46 GiB / vram_kv 3.46 GiB per card byte-identical to
|
||||
# the 1.0.13 ledger at EVC 0.82). Re-test 1.2.0+ when the rotation change
|
||||
# is untangled; 1.2.0 also REMOVED --expert-vs-cache-ratio in favour of
|
||||
# the --vram-weights-mib / --vram-kv-mib pair (core/config.cpp) — this
|
||||
# file already uses the pair, which BOTH pins accept (rollback symmetry).
|
||||
FRANK_ARCHON_IMAGE=stilldeadcode/radiance@sha256:4c3f484564bb2872784e46b0a59cbd71becc9c1e746e301a3596ffae253c322e
|
||||
# The .rad container FILE (mounted under /models) and the state dir for the
|
||||
# prefix-cache disk tier (mounted rw at /data). The model dir holds ONLY
|
||||
# .rad files; the raw HF download and its staging dir live beside it and
|
||||
# are never mounted.
|
||||
FRANK_ARCHON_MODEL_DIR=/home/dank/models/rad
|
||||
FRANK_ARCHON_MODEL_FILE=qwen4exp-swift15-w4dn8-i8-hc8m.rad
|
||||
FRANK_ARCHON_STATE_DIR=/home/dank/radiance-state
|
||||
# The state dir's tune/ subdirectory holds rad-tune's kernel-selection
|
||||
# cache (the engine's selector reads $RADIANCE_HOME/tune/<machine>.tune
|
||||
# at boot; $RADIANCE_HOME defaults to /opt/radiance, in the image, so the
|
||||
# cache must ride a host mount — serve script mounts it, see the runbook's
|
||||
# "Tuning levers").
|
||||
# Container identity is the SERVICE name (Frank's call, 2026-10-05): one
|
||||
# name across systemctl and podman ps. The runtime changes; the name does
|
||||
# not. The legacy list retires every predecessor identity that could hold
|
||||
# the GPUs or the port.
|
||||
FRANK_ARCHON_CONTAINER=frank-archon
|
||||
FRANK_ARCHON_LEGACY_CONTAINERS="frank-archon-qwen frank-archon-r9v qwen-fp8 glm-probe"
|
||||
# Served model id — Protoss name (wire rename 2026-10-09). The container,
|
||||
# unit, scripts and this file keep the frank-archon name above; only the
|
||||
# alias on the wire moved. Radiance validates the requested model name, so
|
||||
# every client (harness, bench scripts, probes) MUST send this value.
|
||||
FRANK_ARCHON_SERVED_NAME=frank-archon
|
||||
FRANK_ARCHON_PORT=7770
|
||||
# Frank's chat-template call: froggeric's fixed Qwen3.8 template (the
|
||||
# repo's root chat_template.jinja, vendored here; the checkpoint's own
|
||||
# template is baked into the .rad by rad-convert and this overrides it).
|
||||
# Radiance derives the reasoning/tool-call reply format from it.
|
||||
FRANK_ARCHON_TEMPLATE=/home/dank/serverAppConfigs/scripts/qwen38-froggeric-v22.5.jinja
|
||||
|
||||
# --- engine shape (the boot ledger arbitrates every number below) ---------
|
||||
# Tensor parallel: BOTH R9700s. The board swap (2026-09-20, ProArt B850
|
||||
# x8/x8 on root ports) made collectives legal on this host; radiance's TP
|
||||
# wire is libr4d's own, not RCCL.
|
||||
FRANK_ARCHON_TP=2
|
||||
# Lossy WHT 6-bit wire for large all-reduces (publisher's production choice:
|
||||
# +5.6-6.3% prefill, ~-1.3% 8-stream decode at TP2). min-kb stays the
|
||||
# engine default.
|
||||
FRANK_ARCHON_TP_WIRE=wht6
|
||||
# Decode slots: THREE, so the served window is a CAPACITY CONTRACT — three
|
||||
# concurrent conversations of --max-model-len fit the KV pool (see the KV
|
||||
# block below). The "seqs 2 cannot boot" law is NOT radiance's: it was the
|
||||
# tcclaviger vLLM fork's torch.compile shape specialization on
|
||||
# query_start_loc (ConstraintViolationError, measured 2026-10-02, retired
|
||||
# with that runtime). Radiance booted seqs 2 at the fp8-native first boot,
|
||||
# and the 2026-10-06 decode sweep measured seqs 2 and 3 statistically tied
|
||||
# (30.9-33.1 t/s, inside the +-3% noise floor) — slot count buys concurrency,
|
||||
# not speed, so a slot costs KV, never decode.
|
||||
# Cost of each extra slot: 110 MiB of per-slot recurrent state inside the KV
|
||||
# pool, and the fused TP all-reduce is declared over seqs x (1 + spec) rows —
|
||||
# 3 x 3 = 9 here, under the 25-row bound at which the wht6 wire stays EXACT
|
||||
# (single-sequence decode and any step of <= 25 rows is bit-exact; only busier
|
||||
# steps are lossy).
|
||||
# The pool is sized to this contract and nothing more (see the KV block):
|
||||
# 3 x 200,000 at fp8 = 5,063 MiB, pool 5,120, and the freed 1,792 MiB went into
|
||||
# the resident expert slab. The trade is explicit and it is Frank's call:
|
||||
# a FOURTH full-height conversation no longer fits (5,120 MiB of pool backs
|
||||
# ~608k tokens; 4 x 200,000 needs 6,601), so a fourth 200k request is refused
|
||||
# as too large to fit — it is NOT squeezed, paged, or truncated. Third-party
|
||||
# traffic that wants four full-height sessions is a --vram-kv-mib change, not
|
||||
# a code change.
|
||||
FRANK_ARCHON_MAX_SEQS=3
|
||||
# Served window. Native is 262,144; 200,000 is the publisher's production
|
||||
# value (compose) and what models.yml's contextWindow must quote — a longer
|
||||
# prompt is refused by the engine, never truncated. The engine treats
|
||||
# `kv group tokens >= max-model-len` as a HARD limit ("ONE request longer
|
||||
# than N tokens is accepted by the API and then refused as too large to ever
|
||||
# fit — this is a hard limit, not a backing ratio"), so the pool is sized
|
||||
# for seqs x maxlen, not for one sequence.
|
||||
FRANK_ARCHON_MAX_MODEL_LEN=200000
|
||||
FRANK_ARCHON_PLACEMENT=expert_tiered
|
||||
# Pinned host pool for the expert rows that do not fit VRAM. vram_weights and
|
||||
# this pool serve the per-rank resident set TOGETHER: at the fp8-KV pair
|
||||
# (26071/3543) the ledger showed the VRAM slab filled to 259 KiB of slack,
|
||||
# i.e. the pair was exactly sized. Cutting vram-weights by 3.8 GiB to pay for
|
||||
# the bf16 KV pool SPILLS ~3.8 GiB/rank here, so the pool moves with it —
|
||||
# under-sizing it is a boot failure (36864 refused the boot in the fp8 era).
|
||||
# The boot ledger's coverage line remains the referee, not this arithmetic.
|
||||
FRANK_ARCHON_HOST_POOL_MIB=44928
|
||||
# VRAM left unclaimed per card (publisher default 96 MiB).
|
||||
FRANK_ARCHON_GPU_HEADROOM_MIB=96
|
||||
# VRAM split of claimed memory per card, MiB (0 = derive from the card).
|
||||
# THE CEILING, read off this engine's own VRAM-budget table (boot 2026-10-13,
|
||||
# pair 24000/5120): card 32,624 MiB; budgets claimed 29,120; `3.42 GiB is
|
||||
# neither budgeted nor accounted — the driver's context, the activation arena
|
||||
# and --gpu-headroom-mib come out of it; anything past those is throughput on
|
||||
# the floor`; and `card 0 settled at 2.01 GiB free against 209.19 MiB (the
|
||||
# 96.00 MiB --gpu-headroom-mib and 113.19 MiB kept for the static region)`.
|
||||
# So the reserve above the ENGINE's floor is 2,058 - 209 = 1,849 MiB and the
|
||||
# ceiling is 29,120 + 1,849 = 30,969 MiB. The 2026-10-11 bound of 29,120
|
||||
# ("31,856 - 1,710 outside both budgets - keep 1 GiB free") was MY reserve, not
|
||||
# the engine's; the pair below spends the difference and lands 458 MiB above
|
||||
# the engine's floor. `weights + kv <= 30,969` is the gate.
|
||||
# THE PRICE, and it is paid twice: on this engine idle KV VRAM is ALSO the
|
||||
# expert cache's spillover (`mover: 2275 slots can be lent out of the KV
|
||||
# cache (2.69 GiB)`, `arena: lends 740 MiB while steps are <= 256 tokens`),
|
||||
# so a bigger KV pool steals weight residency from the slab AND from the
|
||||
# lendable cache. Priors on this card/model: stealing 2.14 GiB of weights for
|
||||
# a bigger pool (MAX_MODEL_LEN 262144) cost 2.5% decode; the EVC ladder's
|
||||
# other direction (0.82 -> 0.84, +2.5 GiB weights) cost 8.9%. The 2026-10-11
|
||||
# exact-KV A/B measured the slab's own price (22208 vs the fp8-era 26071):
|
||||
# prefill -29 %, TTFT at the full window +25 %, decode inside the noise band on
|
||||
# 2 of 3 passes, draft acceptance UP 0.661 -> 0.740.
|
||||
# 2026-10-13 (late): THREE slots, so the pool is cut to the contract (5,120)
|
||||
# and every spare MiB goes to the slab: 22208 -> 24000 (the third slot's
|
||||
# surplus KV), then 24000 -> 25600 (the card's settling slack). The direction
|
||||
# the A/B data points: prefill is slab-limited, KV beyond the contract is not.
|
||||
# The margin is thin by design (608,027 predicted tokens against the 600,000
|
||||
# contract), so the boot ledger's `backs N sequences all at 200000 tokens at
|
||||
# once` line is the referee for THIS pair and the concurrent probe is its
|
||||
# regression test.
|
||||
# VERDICT on the 25,600 arm (measured 2026-10-13, scripts/probe-frank-archon-kv.py):
|
||||
# the threshold claim is FALSIFIED — the slab is a SLOPE at this end. A-cold
|
||||
# 43.5k single-stream: 22,208 -> 3,397, 24,000 -> 3,290, 25,600 -> 3,584 t/s;
|
||||
# the 3-way 197k arm's aggregate prefill 3,051 -> 3,589 t/s (wall
|
||||
# 193.7 -> 164.5 s). So +1,600 MiB of slab = +8.9 % single-stream and
|
||||
# +17.6 % concurrent, both outside the +-3 % noise floor. The 22,208/24,000
|
||||
# pair landed inside noise of EACH OTHER, which is what made it read as a
|
||||
# plateau; two points are not a curve.
|
||||
# The cliff that remains is between 25,600 (3,584) and 26,071 (4,379, the
|
||||
# 2026-10-11 fp8 control at seqs 3 / pool 3,543) — +22 % for +471 MiB. At THREE
|
||||
# slots that is only reachable by breaking the ceiling (26,112 + 5,120 = 31,232
|
||||
# vs 30,969, ~214 MiB above the engine's floor); at TWO slots it is free:
|
||||
# pool 3,072 + slab 27,897 = exactly the ceiling, slab above the cliff.
|
||||
FRANK_ARCHON_VRAM_WEIGHTS_MIB=25600
|
||||
FRANK_ARCHON_VRAM_KV_MIB=5120
|
||||
# KV cache QUANTIZED (fp8). This engine's flag takes ONLY bf16|fp8 — there is
|
||||
# ONE kv_dtype for the whole cache, so "exact K, quantized V" (llama.cpp's
|
||||
# --cache-type-k/--cache-type-v pair, which is what frank-zealot runs) is
|
||||
# UNSERVABLE here: the rung below bf16 is fp8 on both halves.
|
||||
# Measured per-rank cost, read out of the boot ledger's group table (fp8 boot
|
||||
# 2026-10-08, bf16 boot 2026-10-11):
|
||||
# kv_attn 13 layers, 4-token blocks, page 26.00 KiB (fp8) -> 52.00 KiB
|
||||
# kv_qsa_bkey 13 layers, 4-token blocks, page 3.25 KiB at BOTH dtypes —
|
||||
# the QSA block key is an index, not attention state, and does
|
||||
# NOT follow --kv-cache-dtype
|
||||
# => 7.3125 KiB/token fp8, 13.8125 KiB/token bf16: bf16 costs 1.89x fp8,
|
||||
# not 2x, precisely because the bkey plane is dtype-fixed.
|
||||
# The dtype-INDEPENDENT terms also live in the pool: kv_gdn_state 54 MiB x 2
|
||||
# copies per sequence (+ conv/tail/ple ~= 110 MiB a sequence) and the prefix
|
||||
# checkpoint region 8 x 55.99 = 448 MiB. So:
|
||||
# pool_mib >= tokens x 7.3125 KiB + 110 x seqs + 448
|
||||
# 3 x 200,000 at fp8 needs 4,285 + 330 + 448 = 5,063 MiB; the pool is 5,120,
|
||||
# and it predicts (5120 - 330 - 448) x 1024 / 7.3125 = 608,027 tokens: the
|
||||
# three-slot contract with 1.3 % slack, every MiB above 5,063 having been
|
||||
# handed to the expert slab instead. The ledger line is the referee ("backs N
|
||||
# sequences all at 200000 tokens at once") — a pair this tight is read off the
|
||||
# boot, never assumed.
|
||||
# At bf16 the same three slots cost 8,093 + 778 = 8,871 MiB, which would cut
|
||||
# the slab to 20,249 MiB (-9 %); at FOUR slots bf16 costs 10,791 + 888 =
|
||||
# 11,679 MiB and the slab falls to 17,441 (-33 %). fp8 keeps 22,208 at both
|
||||
# slot counts. The 2026-10-11 bf16 shape (6,912 / seqs 2) backed 462,752
|
||||
# tokens = 2 x 200,000; the historical fp8 pool of 3,543 gave 386,904 (1.93
|
||||
# sessions), and the four-slot fp8 boot of 2026-10-13 19:21 backed 843,276.
|
||||
# Secondary saving: the prefix-cache entry shrinks with the attention plane
|
||||
# (one entry 110.50 KiB at bf16 -> 58.50 KiB at fp8, measured on the two boot
|
||||
# ledgers' `kv tier transfer` lines), so the host tier below parks twice as
|
||||
# many conversations as it did at bf16 — no re-size needed to park three slots.
|
||||
FRANK_ARCHON_KV_DTYPE=fp8
|
||||
# Spec decode: the checkpoint's own MTP head at the publisher's depth-3
|
||||
# operating point (probe() returns 3 when mtp_num_hidden_layers > 0; the
|
||||
# flag states it so the ledger records the intent).
|
||||
FRANK_ARCHON_SPEC_TOKENS=2
|
||||
FRANK_ARCHON_MAX_BATCHED_TOKENS=2048
|
||||
# Prefix cache tiers (publisher pattern). The host tier is sized in ENTRIES,
|
||||
# and one entry tracks the cache dtype (58.50 KiB at fp8, 117 KiB at bf16, per
|
||||
# the ledger's `kv tier transfer` line): at fp8, 8192 MiB parks ~5.8 sessions
|
||||
# of 200k (at bf16 the same tier parked ~2.9), so four live slots plus a full
|
||||
# parked set fits with the tier as it stands — at RAM the host has (104 GiB
|
||||
# available with the service loaded).
|
||||
FRANK_ARCHON_PREFIX_HOST_MIB=8192
|
||||
FRANK_ARCHON_PREFIX_DISK_MIB=65536
|
||||
# Frank's pin law: the PLE n-gram table (47.7 GiB) serves from pinned host
|
||||
# RAM the cards read. `ram` REFUSES the start if the memlock limit is too
|
||||
# small — the unit's LimitMEMLOCK=infinity is the other half of this law.
|
||||
FRANK_ARCHON_NGRAM_PLACEMENT=ram
|
||||
# TTM preflight: the pinned ledger (2 x 44 GiB pools + 47.7 GiB PLE + 8 GiB
|
||||
# prefix host = ~147 GiB) rides TTM's shared pages, capped by ttm's
|
||||
# pages_limit (the current kernel exposes NO pages_max; the cap function
|
||||
# takes the min over the knobs that exist). Below the plan the load dies
|
||||
# mid-pin with an allocator error instead of this named refusal.
|
||||
# THE CAP IS A RUNTIME KNOB — the kernel default is half of RAM, so every
|
||||
# raise dies with a reboot. Measured the hard way: the 2026-10-05 manual raise
|
||||
# to 52,428,800 pages (200 GiB) was gone after the 2026-10-12 23:17 POWER
|
||||
# FAILURE, the cap fell back to 125,988 MiB, and the preflight refused the
|
||||
# start 6,919 times in ~20 h while Restart=always re-ran it. The gate in
|
||||
# scripts/ttm-preflight.sh now RAISES the knob itself before refusing (the
|
||||
# root unit is privileged; the knob is a 0644 root module parameter), so the
|
||||
# recovery is a boot, not a visit to the machine.
|
||||
FRANK_ARCHON_TTM_MIN_MIB=150000
|
||||
FRANK_ARCHON_TTM_TARGET_MIB=204800
|
||||
# Escape hatch for boot-ladder experiments. Unset -> no extra argv. Keep
|
||||
# multi-flag values as one string; the serve script expands it unquoted on
|
||||
# purpose.
|
||||
FRANK_ARCHON_EXTRA_ARGS=
|
||||
# The .rad's minimum size in MiB (deployer gate; a truncated convert is a
|
||||
# guaranteed long-timeout boot instead of a fast failure). The converted
|
||||
# container measured 177,109 MiB (1,857,127,997,440 B = 173.0 GiB; the
|
||||
# plan-only estimate 172.76 GiB covers the data planes but not the metadata
|
||||
# tail) - floor sits just under the payload.
|
||||
FRANK_ARCHON_RAD_MIN_MIB=110000
|
||||
@@ -0,0 +1,71 @@
|
||||
# frank-archon.service — radiance (StillDeadcode, Apache-2.0) endpoint
|
||||
# (OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 at NATIVE block-FP8 precision, the
|
||||
# rad-convert container qwen4exp-fp8-native.rad) on BOTH R9700s (TP=2,
|
||||
# wht6 wire), dankserver :7770. 2026-10-05 cutover from FreeToken (git tag
|
||||
# frank-archon-freetoken-pr587-20261005 is the pre-cutover state; the
|
||||
# retired unit/env/serve trio and Dockerfile are in that tag).
|
||||
#
|
||||
# Same port, same served alias, same container identity as the unit name;
|
||||
# the harness models.yml id is unchanged. The serve script drives
|
||||
# `podman run` directly: the digest-pinned stilldeadcode/radiance image
|
||||
# (287 MiB, kernels prebuilt for gfx1201) has the engine as its
|
||||
# entrypoint. The image lives in dank's ROOTLESS podman storage.
|
||||
# WHY A ROOT UNIT (not a rootless quadlet): radiance pins ~132 GiB of HOST
|
||||
# RAM (2 x 40 GiB expert pools + the 47.7 GiB PLE ngram table + the prefix
|
||||
# host tier). Pinned pages are charged against RLIMIT_MEMLOCK; dank's login
|
||||
# session on this host carries 8 MiB and a user quadlet is capped by the
|
||||
# user manager's own hard limit — raising that means restarting
|
||||
# user@1000.service and bouncing every user container on the host. A root
|
||||
# unit grants LimitMEMLOCK=infinity to the dank process via PID1; the
|
||||
# engine still runs out of dank's rootless podman storage.
|
||||
[Unit]
|
||||
Description=frank-archon — radiance Qwen3.8-Flash-Next-FP8 (native) on 2x R9700 TP=2
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
# Never let the start-rate limiter give up after a crash loop (a failed LLM
|
||||
# endpoint that systemd refuses to retry is worse than a retrying one).
|
||||
StartLimitIntervalSec=0
|
||||
|
||||
[Service]
|
||||
User=dank
|
||||
Group=dank
|
||||
# The engine pins ~132 GiB of host RAM (expert pools + PLE table); an
|
||||
# RLIMIT_MEMLOCK smaller than the pinned set fails the load mid-start.
|
||||
# PID1 grants infinity here — parity with the fleet's other LLM units.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_ARCHON_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-archon.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-archon.sh
|
||||
# The `+` prefix runs this as root even though the service is User=dank — the
|
||||
# ttm pages cap is a root-owned knob and the pinned-RAM ledger (~147 GiB) is
|
||||
# unusable below it. After a reboot the knob is back at its default (half of
|
||||
# RAM), so the raise must be part of the start, not a note in a runbook: the
|
||||
# 2026-10-12 power failure reset it and the endpoint refused to boot 6,919
|
||||
# times. The serve script re-verifies the cap after this hook.
|
||||
ExecStartPre=+/home/dank/serverAppConfigs/scripts/ttm-preflight.sh --raise
|
||||
# systemd stop: wait alone cannot stop the container child; stop it
|
||||
# explicitly. No graceful drain contract is claimed (the OpenAI front end
|
||||
# holds no queue to drain — in-flight decodes are lost, which a stop
|
||||
# during a bench window accepts; radiance takes SIGTERM on the engine
|
||||
# process and podman reaps the container).
|
||||
ExecStop=/usr/bin/podman stop -t 120 frank-archon
|
||||
# Boot cost: .rad load (weights + PLE upload) + expert host-pool pin. The
|
||||
# publisher's compose healthcheck allows a 15-minute start_period; the
|
||||
# vLLM-era budget (1800 s) carried forward.
|
||||
# Restart=always, NOT on-failure (the 2026-10-01 vLLM lesson carries over:
|
||||
# a clean API-server exit after a worker fault reads as success to
|
||||
# systemd). `always` restarts on any container death; an explicit
|
||||
# `systemctl stop` still wins, so bench windows and deploys are unaffected.
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
# ExecStop allows the engine 120 s to drain (`podman stop -t 120`); without
|
||||
# an explicit TimeoutStopSec systemd's 90 s default cuts the stop short,
|
||||
# kills it, and records 'Failed with result timeout' (measured 2026-10-01
|
||||
# 13:27:17). Must exceed the podman grace with margin.
|
||||
TimeoutStopSec=180
|
||||
# The engine pins tens of GiB of host RAM (expert rows + PLE table); it is
|
||||
# the last process on this host that should be chosen by the kernel OOM
|
||||
# killer.
|
||||
OOMScoreAdjust=-500
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,158 @@
|
||||
# frank-dragoon.env — tunables for the frank-dragoon endpoint on Dankclaw's
|
||||
# Strix Halo iGPU (gfx1151), host port 7770.
|
||||
#
|
||||
# RUNTIME (2026-10-06, Frank): halogen-flash-server 0.16.2 — the
|
||||
# closed-source C++/HIP engine (image ghcr.io/peonist-ai/halogen-flash-server,
|
||||
# digest-pinned below) serving the Quat3rnion ABLITERATED Swift1.5 finetune
|
||||
# of peonist's Qwen3.8-Flash-Next v2 checkpoint, as a podman container under
|
||||
# a root unit, driven by scripts/serve-frank-dragoon.sh. This CUTS OVER the
|
||||
# 2026-10-05 halogen 27B-dense shape (censored base, DFlash2, 31.7 t/s):
|
||||
# rollback = revert the swap commit's trio; the 27B payload stays on disk at
|
||||
# /home/dank/models/halogen-qwen3.8-27b/.
|
||||
#
|
||||
# WHY THE RUNTIME MOVED WITH THE MODEL: the payload is .hgn — per its own
|
||||
# card it "will not load in transformers, vLLM, or llama.cpp" — and the v2
|
||||
# .hgn loads ONLY in the flash-server line (the old halogen engine is for
|
||||
# the 27B p1w4d file; the v2 quickstart pins flash-server). What the move
|
||||
# buys (card + live probe evidence): MTP spec decode (53-57 t/s greedy,
|
||||
# 3-10x serial), ~1,640 t/s prefill at 34k, ~72% fewer thinking tokens than
|
||||
# the censored 27B (905 mean), the ABLITERATED base (refusals should NOT
|
||||
# return), IMAGE INPUT (the v2 vision tower loads beside the checkpoint —
|
||||
# red-PNG probe verified on the card; the 27B engine ignored its encoder),
|
||||
# tool calls verified, and the native 262,144 context.
|
||||
#
|
||||
# SHAPE — the knobs that are NOT free, with the arithmetic:
|
||||
# HALOGEN_CTX=262144 is the PER-SLOT window and the per-conversation hard
|
||||
# ceiling — a prompt over it is a 400 naming the limit, never a silent
|
||||
# truncation. RAISED from 200192 on 2026-10-07: the harness clamps
|
||||
# max_tokens to contextWindow − est − 4096, its chars/4 est UNDER-counts
|
||||
# the rendered prompt by ~3 % on code/agent traffic, and compaction only
|
||||
# fires at 85 % — so with window == engine ceiling, every request over
|
||||
# ~146k est (65-85 % of the window) was a guaranteed 400. Measured
|
||||
# 2026-10-07: est 152,938, server 157,388, request 43,158 vs room 42,804
|
||||
# — over by 354, margin 4,096 < the 4,450 under-count.
|
||||
# Harness contract (models.yml): contextWindow 217088, maxTokens 45056.
|
||||
# Compaction fires at 85 % = 184,524 est ≈ 190k measured (Frank's 180k
|
||||
# target). Worst-case request: prompt + budget <= 217088 − 4096 + 3 %
|
||||
# est-error ≈ 219k <= 262144 — the 45k gap between harness window and
|
||||
# engine ceiling absorbs the estimate error with ~7x margin (robust to a
|
||||
# 10 % under-count). Compaction rides on frank-zealot: input ≈
|
||||
# threshold − keepRecent(20k) ≈ 170k measured, inside its 200,704 slot.
|
||||
# HALOGEN_KV_POOL_POSITIONS=524288 = KV_SLOTS(2) x CTX — the explicit
|
||||
# form of the engine's default, so the seam test can pin the product.
|
||||
# KV costs ~28 KiB/position (FLAGS.md) => 524,288 x 28 KiB = 14.0 GiB.
|
||||
# HALOGEN_KV_SLOTS=2 restores the w4b-era shape on this box: the MTP
|
||||
# drafter is NOT slot-locked on this engine (batching and speculation
|
||||
# coexist — DFlash-era slots>1 law does NOT carry over; the card's own
|
||||
# quickstart ships 2 slots WITH the head), so the second slot is
|
||||
# concurrent capacity, not a speculation tax.
|
||||
# Memory ledger against the box's 121.2 GiB unified (the n-gram table is
|
||||
# mmap'd page cache beside the checkpoint, not a pinned charge — the
|
||||
# card's own quickstart arithmetic does not bill it):
|
||||
# weights ~62 GiB resident + KV 14.0 + arena (HALOGEN_MAX_TOK=8192 =>
|
||||
# ~8 GiB per FLAGS.md arena formula 3.8+8.4*(MT/16384)) + vision ~2
|
||||
# = ~86 GiB engine-side; HALOGEN_HOST_RESERVE_GIB=20 keeps the host
|
||||
# honest; baseline co-tenancy (frank-zealot, ComfyUI, Caddy/observ)
|
||||
# ~10-15. ~15 GiB margin. The boot ledger is the referee — read it
|
||||
# after deploy and re-size if the engine prints a clamp.
|
||||
# HALOGEN_SPEC_ADAPT=0 is MANDATORY (card): the default adaptive policy
|
||||
# turns the MTP head off for whole requests after low acceptance and
|
||||
# HALVES decode as a result. Frank accepted the head always-on trade
|
||||
# (temp 1.0 traffic costs acceptance; the card still ships the recipe).
|
||||
#
|
||||
# CLIENT CONTRACT deltas vs the 27B shape (harness side lives in
|
||||
# ~/.omp/agent/models.yml):
|
||||
# * SERVER-SIDE SAMPLER EXISTS NOW: HALOGEN_TEMPERATURE/TOP_P/TOP_K set
|
||||
# the card's thinking recipe (1.0 / 0.95 / 20) as engine defaults, so
|
||||
# an absent temperature is no longer greedy. models.yml extraBody
|
||||
# carries the same values — consistent, not load-bearing.
|
||||
# * HALOGEN_MAX_TOK=8192 is only the DEFAULT for requests that omit
|
||||
# max_tokens; the harness sends at most its maxTokens 45056. Requests
|
||||
# over HALOGEN_MAX_TOKENS_CAP (65536) are a 400 naming the cap. Cap and
|
||||
# queue timeout are COUPLED (FLAGS.md table): 65536 at ~10 t/s serial
|
||||
# worst case needs > 6550 s; 7200 honors it. Do not move one alone.
|
||||
# * max_tokens INCLUDES thinking; a reasoning reply over a small budget
|
||||
# returns empty content with finish_reason "length".
|
||||
# * Prompt cache is ON by default (PROMPT_CACHE=2): a follow-up turn at
|
||||
# 100k context re-prefills ~2 s instead of ~88 s cold. No pin needed —
|
||||
# unlike the 27B engine, auto-sizing fits this box's ledger.
|
||||
# * Thinking lands in reasoning_content (same as the 27B engine); the
|
||||
# v2 chat template rejects reasoning_effort outside xhigh|medium|low,
|
||||
# so the models.yml fold (high->xhigh) still applies.
|
||||
# * Vision: OpenAI image_url content parts only (the fleet's standard
|
||||
# contract). >=256 px probe images or the model answers "video".
|
||||
# * response_format json_object/json_schema unsupported in this line
|
||||
# (no constrained decoding) — unchanged from the 27B engine.
|
||||
# * The engine's token protocol has no auth and binds loopback INSIDE
|
||||
# the container; only the frontend port (8731) is published. Same as
|
||||
# before. Kernel needs CONFIG_HSA_AMD_SVM (CachyOS ships it) and
|
||||
# amdgpu.noretry=0 (check at deploy).
|
||||
|
||||
# Image: digest-pinned (fetch law: tag pulls silently re-point; the digest
|
||||
# is what was qualified). 0.16.2 = the version the model card names as its
|
||||
# quickstart pin; RepoDigest captured at pull on 2026-10-06.
|
||||
FRANK_DRAGOON_IMAGE=ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84ac22308a53f5d1ca6b86806702d7039e5ebc51cae4c66621b92fe04a
|
||||
# Container identity: kept from the 27B era (unit ExecStop names it).
|
||||
FRANK_DRAGOON_CONTAINER=frank-dragoon-halogen
|
||||
# Old quadlet-era container identity; the serve script clears stale names.
|
||||
FRANK_DRAGOON_LEGACY_CONTAINERS=frank-dragoon
|
||||
# host:container — the frontend listens on 8731 inside (HALOGEN_API_PORT
|
||||
# default); the engine port (8730) is never published.
|
||||
FRANK_DRAGOON_PUBLISH=7770:8731
|
||||
|
||||
FRANK_DRAGOON_MODEL_DIR=/home/dank/models/halogen-flashnext-v2
|
||||
# The checkpoint: Quat3rnion/halogen-swift1.5-qwen3.8-flash-next-v2-abliterated
|
||||
# qwen38-flash-next-v2-swift15-abliterated.hgn — compacted r2 of the card's
|
||||
# append-only original (byte-identical live tensors, padding dropped:
|
||||
# 66,687,678,464 B vs 89,775,868,032 B). LFS sha256 verified at fetch.
|
||||
FRANK_DRAGOON_CHECKPOINT=qwen38-flash-next-v2-swift15-abliterated.hgn
|
||||
FRANK_DRAGOON_CHECKPOINT_BYTES=66687678464
|
||||
FRANK_DRAGOON_CHECKPOINT_SHA256=9c880b8ba48deabfa06172eff1e7572e4fe465e09e97faec6e50c3f6fdcac037
|
||||
# The n-gram lookup table: REQUIRED (the engine reads it beside the
|
||||
# checkpoint; the card's quickstart names it explicitly). From
|
||||
# peonist-ai/halogen-qwen3.8-flash-next @ 5cc17cea.
|
||||
FRANK_DRAGOON_NGRAM_TABLE=qwen38-flash-next-ngram.hgn
|
||||
FRANK_DRAGOON_NGRAM_BYTES=51200246144
|
||||
FRANK_DRAGOON_NGRAM_SHA256=3450acada94e19aabad88bd49b45eb70304820949c2240fc178067b20ae5dffe
|
||||
# The vision tower: OPTIONAL on the engine, ENABLED here (image input).
|
||||
# From peonist-ai/halogen-qwen3.8-flash-next @ e053f488.
|
||||
FRANK_DRAGOON_VISION_TOWER=qwen38-flash-next-vision.hgn
|
||||
FRANK_DRAGOON_VISION_BYTES=897916416
|
||||
FRANK_DRAGOON_VISION_SHA256=d62e0ae553fe88afd3833733d4a4c669f34d20fd8dfce4b9610525bed2134b10
|
||||
# The tokenizer is a FLAT DIRECTORY, $FRANK_DRAGOON_MODEL_DIR/tokenizer,
|
||||
# mounted whole at /tokenizer. The two repos ship the same one; fetched
|
||||
# fresh from peonist @ e053f488 (tokenizer.json is LFS, byte-identical to
|
||||
# the 27B-era copy; the rest are plain git blobs — size-checked only). The
|
||||
# serve gate and the deployer refuse a missing file or ANY symlink: an HF
|
||||
# cache snapshot symlinks each file into blobs/, and those links dangle
|
||||
# inside the container mount.
|
||||
FRANK_DRAGOON_TOKENIZER_FILES="tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json"
|
||||
|
||||
# --- engine flags (docs/FLAGS.md of the flash-server repo; anything unset
|
||||
# stays at the image default, which is what was qualified) ------------------
|
||||
# Served model id — Protoss name (wire rename 2026-10-09); the service, unit,
|
||||
# scripts and this file keep the frank-dragoon name. OpenWebUI api_configs
|
||||
# model_ids for the :7770 backend MUST name this value.
|
||||
HALOGEN_MODEL_ID=frank-dragoon
|
||||
HALOGEN_CTX=262144
|
||||
HALOGEN_KV_POOL_POSITIONS=524288
|
||||
HALOGEN_KV_SLOTS=2
|
||||
HALOGEN_HOST_RESERVE_GIB=20
|
||||
# Spec decode: MTP head is INSIDE the checkpoint (do NOT set HALOGEN_MTP_HEAD);
|
||||
# depth 2 is the documented "best all-round" default. HALOGEN_DRAFTER /
|
||||
# HALOGEN_DFLASH_* from the 27B engine DO NOT EXIST here.
|
||||
HALOGEN_SPEC_ADAPT=0
|
||||
# Coupled pair (FLAGS.md table): the cap bounds how long ONE request holds
|
||||
# the GPU, the timeout bounds how long the NEXT client waits. Do not move
|
||||
# one alone (see the client-contract note above).
|
||||
HALOGEN_MAX_TOK=8192
|
||||
HALOGEN_MAX_TOKENS_CAP=65536
|
||||
HALOGEN_QUEUE_TIMEOUT=7200
|
||||
# Server-side sampler = the card's thinking recipe (the engine now HAS one;
|
||||
# requests may still override per-call). Matches models.yml extraBody.
|
||||
HALOGEN_TEMPERATURE=1.0
|
||||
HALOGEN_TOP_P=0.95
|
||||
HALOGEN_TOP_K=20
|
||||
# Card's belt-and-braces: the engine pins weights by default
|
||||
# (HALOGEN_FLASH_PIN_TRUNK=1; only the retired w4b build honored 0).
|
||||
HALOGEN_WEIGHTS_LOCK=1
|
||||
@@ -1,5 +1,5 @@
|
||||
[Unit]
|
||||
Description=halogen container (peonist-ai/halogen-qwen3.8-27b, DFlash2 spec decode, 1 slot x 200700, text-only), root systemd unit (dankclaw, frank-large, Strix Halo iGPU)
|
||||
Description=halogen-flash-server container (Quat3rnion abliterated Swift1.5 of Qwen3.8-Flash-Next v2, MTP spec decode, 2 slots x 200192, vision ON), root systemd unit (dankclaw, frank-dragoon, Strix Halo iGPU)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
@@ -11,19 +11,19 @@ Group=dank
|
||||
# RLIMIT smaller than the pinned set cannot fail the load. Parity with the
|
||||
# other LLM units on this fleet.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
|
||||
Environment="FRANK_DRAGOON_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-dragoon.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-dragoon.sh
|
||||
# systemd stop alone cannot stop the container child; stop it explicitly
|
||||
# (frank-primary pattern). The engine drains on SIGTERM.
|
||||
ExecStop=/usr/bin/podman stop -t 60 frank-large-halogen
|
||||
# (frank-archon pattern). The engine drains on SIGTERM.
|
||||
ExecStop=/usr/bin/podman stop -t 60 frank-dragoon-halogen
|
||||
TimeoutStartSec=1200
|
||||
TimeoutStopSec=120
|
||||
# always, not on-failure: an engine that exits 0 after an internal fault
|
||||
# would otherwise leave the endpoint down until someone notices.
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
# The engine mmaps 33.4 GiB of weights and keeps a multi-GiB working set
|
||||
# on a box that also hosts frank-small, ComfyUI and the Caddy/observability
|
||||
# The engine pins ~62 GiB of v2 weights and keeps a multi-GiB working set
|
||||
# on a box that also hosts frank-zealot, ComfyUI and the Caddy/observability
|
||||
# stack; make it a late OOM candidate, not the first.
|
||||
OOMScoreAdjust=-500
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
# frank-large.env — tunables for the frank-large endpoint on Dankclaw's
|
||||
# Strix Halo iGPU (gfx1151), host port 7770.
|
||||
#
|
||||
# RUNTIME (2026-10-05, Frank): halogen — the closed-source C++/HIP engine
|
||||
# (github.com/peonist-ai/halogen-server, image ghcr.io/peonist-ai/halogen,
|
||||
# digest-pinned below) serving peonist-ai/halogen-qwen3.8-27b, a ~6.32 bpw
|
||||
# NVFP4-derivative quant of the BASE Qwen/Qwen3.8-27B (35,865,565,184 B,
|
||||
# 1352 tensors), as a podman container under a root unit, driven by
|
||||
# scripts/serve-frank-large.sh. This CUTS OVER the
|
||||
# 2026-10-02..10-03 ROCmFPX llama.cpp shape (uncensored
|
||||
# Q6_0_ROCMFPX_AGENT + MTP, 27.6-28.1 t/s): rollback = revert the swap
|
||||
# commit's trio; the ROCmFPX payload, draft head and build all stay on
|
||||
# disk at /home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/.
|
||||
#
|
||||
# WHY THE RUNTIME MOVED WITH THE MODEL: the payload is .hgn — per its own
|
||||
# card it "will not load in transformers, vLLM, or llama.cpp". Accepted
|
||||
# consequences (Frank, 2026-10-05): TEXT-ONLY (halogen ignores the model's
|
||||
# vision encoder; the fleet's vision path is frank-small) and the CENSORED
|
||||
# base (this is a finetune of Qwen/Qwen3.8-27B, not the abliterated file —
|
||||
# refusals may return). What it buys: vendor-measured decode DFlash2 31.7
|
||||
# t/s mean (vs this endpoint's MTP 27.6-28.1 and the old halogen-era MTP
|
||||
# 26.9), prefill ~566 t/s at 32k (was ~400 on the sibling engine), 1.4-2.3x
|
||||
# wall-clock on prefill-heavy agent traffic, at HIGHER effective precision
|
||||
# (~6.32 bpw vs ~6.6 with the old W4A4 promotion — see FLAGS.md).
|
||||
#
|
||||
# SHAPE — the two knobs that are NOT free:
|
||||
# HALOGEN_KV_SLOTS=1 KEEPS SPECULATION ON. halogen makes batching and
|
||||
# speculation mutually exclusive: with >1 slot the drafter is disabled
|
||||
# and per-stream decode falls to serial speed (the sibling engine
|
||||
# measured 41 t/s -> 14 t/s when slots were raised, and halogen's own
|
||||
# FLAGS.md prints the same law). frank-large was already a one-session
|
||||
# endpoint under ROCmFPX (PARALLEL=1), so slots=1 is the same contract,
|
||||
# not a regression.
|
||||
# HALOGEN_SLOT_CTX=200700 (Frank's window) is the per-conversation hard
|
||||
# ceiling — a prompt over it is a 400 naming the limit, never a silent
|
||||
# truncation. KV pool = slots x slot_ctx x 64 KiB = 12.2 GiB. Budget
|
||||
# against the 121 GiB of unified memory: weights 33.4 GiB (mmap'd — page
|
||||
# cache, not anon), KV 12.2, engine working set + auto prompt cache
|
||||
# ~20-30, host baseline ~10 -> ~75-85 GiB in use. The 2026-09-16 halogen
|
||||
# era ran the same engine class on this box with LESS headroom.
|
||||
# HALOGEN_DRAFTER=2 (DFlash2) is the shipped default and the fastest
|
||||
# exact drafter on this checkpoint; the .hgn BAKES BOTH drafters (MTP +
|
||||
# DFlash2), so unlike the Flash-Next era there is no sidecar to mount
|
||||
# and no --dflash flag — the drafter is an int knob, byte-identical
|
||||
# output to greedy verification either way.
|
||||
#
|
||||
# CLIENT CONTRACT deltas vs the ROCmFPX shape (the harness side lives in
|
||||
# ~/.omp/agent/models.yml):
|
||||
# * NO server-side sampler defaults: a request without temperature
|
||||
# decodes GREEDY. The harness sends temp 1.0 / top_p 0.95 via
|
||||
# extraBody, so that wire recipe is load-bearing now, not cosmetic.
|
||||
# * max_tokens defaults to 8192 PER REQUEST and INCLUDES thinking; a
|
||||
# reasoning reply over a small budget returns empty content with
|
||||
# finish_reason "length". The harness maxTokens is 57344: compaction
|
||||
# fires at 70% of 200700 = 140,490 and 140,490 + 57,344 = 197,834 <=
|
||||
# 200,700 keeps prompt + budget inside the pool (the 262144-era 65536
|
||||
# would overflow). Requests over HALOGEN_MAX_TOKENS_CAP are a 400
|
||||
# naming the cap; cap and queue timeout are COUPLED (see FLAGS.md
|
||||
# table) — do not move one without the other.
|
||||
# * response_format json_object/json_schema is a 400 (no constrained
|
||||
# decoding in this build).
|
||||
# * Thinking lands in reasoning_content — same field as the llama.cpp
|
||||
# era, so the harness compat block does NOT change. reasoning_effort
|
||||
# accepts none|low|xhigh on this build (template default xhigh); the
|
||||
# models.yml fold (high->xhigh) still applies.
|
||||
# * The engine's token protocol has no auth and binds loopback INSIDE
|
||||
# the container; only the frontend port is published. Keep it that way.
|
||||
|
||||
# Image: digest-pinned (fetch law: tag pulls silently re-point; the digest
|
||||
# is what was qualified). 0.1.4 = the version the model card names.
|
||||
FRANK_LARGE_IMAGE=ghcr.io/peonist-ai/halogen@sha256:dc0a39a0016d6cfc58a197978febaafdf8d28403f724ded6111d98b5fb7ac0ea
|
||||
# Container identity: the halogen container name.
|
||||
FRANK_LARGE_CONTAINER=frank-large-halogen
|
||||
# Old quadlet-era container identity; the serve script clears stale names.
|
||||
FRANK_LARGE_LEGACY_CONTAINERS=frank-large
|
||||
# host:container — the frontend listens on 8731 inside (HALOGEN_API_PORT
|
||||
# default); the engine port (8730) is never published.
|
||||
FRANK_LARGE_PUBLISH=7770:8731
|
||||
|
||||
FRANK_LARGE_MODEL_DIR=/home/dank/models/halogen-qwen3.8-27b
|
||||
FRANK_LARGE_CHECKPOINT=qwen3.8-27b-p1w4d-d2.hgn
|
||||
FRANK_LARGE_CHECKPOINT_BYTES=35865565184
|
||||
FRANK_LARGE_CHECKPOINT_SHA256=274c3fc767fb57faf025dd03c76376b1fc1b32448aef45aaf02d1898fa962089
|
||||
# The tokenizer is a FLAT DIRECTORY, $FRANK_LARGE_MODEL_DIR/tokenizer, mounted
|
||||
# whole at /tokenizer — the model card ships `tokenizer/` and the engine's
|
||||
# HALOGEN_TOKENIZER default is that directory (the card's own docker run sets
|
||||
# no tokenizer flag). The Hub stores it as 5 files (sizes from the Hub API,
|
||||
# 2026-10-05): tokenizer.json 12809320 (LFS sha256
|
||||
# 0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3),
|
||||
# vocab.json 6722759, merges.txt 3353259, chat_template.jinja 8952,
|
||||
# tokenizer_config.json 17928 (plain git blobs — size-checked, no LFS oid).
|
||||
# The serve gate and the deployer refuse a missing file or ANY symlink: an
|
||||
# HF cache snapshot symlinks each file into blobs/, and those links dangle
|
||||
# inside the container mount.
|
||||
FRANK_LARGE_TOKENIZER_FILES="tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json"
|
||||
|
||||
# --- engine flags (docs/FLAGS.md of the engine repo; anything unset stays
|
||||
# at the image default, which is what was qualified) ------------------------
|
||||
HALOGEN_MODEL_ID=frank-large
|
||||
HALOGEN_SLOT_CTX=200700
|
||||
HALOGEN_KV_SLOTS=1
|
||||
HALOGEN_DRAFTER=2
|
||||
# Coupled pair (FLAGS.md table): the cap bounds how long ONE request holds
|
||||
# the GPU, the timeout bounds how long the NEXT client waits. 65536 at
|
||||
# ~10 t/s serial worst case needs > 6550 s queue timeout; the defaults
|
||||
# honor that. Do not lower the timeout alone.
|
||||
HALOGEN_MAX_TOKENS_CAP=65536
|
||||
HALOGEN_QUEUE_TIMEOUT=7200
|
||||
# Engine prompt cache: EMPTY = auto (sized from available memory at
|
||||
# startup). Pin an explicit MB only to argue with the auto figure — a
|
||||
# full-context entry is ~18 GB at 262K, so a too-small pin yields a cache
|
||||
# that reports enabled and never hits (the engine warns at startup).
|
||||
#HALOGEN_CACHE_MB=
|
||||
@@ -1,82 +0,0 @@
|
||||
# frank-primary.env — FreeToken (PR #587 ROCm) serving the orcarouter
|
||||
# Qwen3.8-Flash-Next-Uncensored-FP8 payload on dankserver :7770.
|
||||
#
|
||||
# 2026-10-05 cutover from the tcclaviger RDNA4 vLLM fork (git tag
|
||||
# frank-primary-vllm-shapeG-20261003 is the pre-cutover state; the 362-line
|
||||
# vLLM-era env is in that tag). The engine is FreeToken at PR head
|
||||
# 77a7201f98f00d93f618c6cf8b81ffe8360e0899 (branch rocm714_torch214), built
|
||||
# on the host into a local image by
|
||||
# scripts/docker/frank-primary-freetoken.Dockerfile — the PR's own
|
||||
# docs/install_amd.md stack (ROCm 7.14 base image, torch 2.11.0+rocm7.14,
|
||||
# triton 3.7) for gfx1201.
|
||||
#
|
||||
# WHY FREETOKEN: the vLLM fork's expert offload planned VRAM/host tiers per
|
||||
# boot from an estimator (util 0.982 law, arena 17.77 GiB / 7,762 slots,
|
||||
# ~40k-token GDN state faults) and 1.0 was unbootable. FreeToken keeps ALL
|
||||
# expert weights in pinned HOST RAM (~120 GiB) with an LRU GPU cache, sizes
|
||||
# KV from an explicit token pool, and serves on ONE R9700 (32 GB) with the
|
||||
# second card idle. RAM fit: 246 GiB total, service stopped frees the vLLM
|
||||
# container's ~170 GiB; weights ~120 GiB + PLE 47.7 GiB + overhead ≈ 190 GiB.
|
||||
#
|
||||
# PAYLOAD: unchanged — /home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8,
|
||||
# 131 safetensors shards. Quant is standard modelopt block-FP8: 75,264
|
||||
# weight_scale_inv tensors (block 128x128), zero input_scale (FreeToken #462
|
||||
# covers exactly this export shape), one weight_scale on the PLE ngram
|
||||
# embedding. FreeToken loads HF safetensors directly; no FTW conversion.
|
||||
#
|
||||
# SERVING FACTS (docs/cli.md, verified against main+PR587):
|
||||
# - `ft serve` loads the checkpoint directly; `--served-model-name` keeps the
|
||||
# harness id; reasoning/tool parsers are AUTO (reasoning lands in
|
||||
# reasoning_content — models.yml flips reasoningContentField to match).
|
||||
# - KV: --page-size must stay 1 for GDN/SWA models (default); --cache-type
|
||||
# radix auto-picks the GDN-aware variant.
|
||||
# - MoE: --moe-strategy auto -> offload (fp8 experts CANNOT use
|
||||
# --moe-cpu-layers; only bf16/nvfp4/mxfp4 have a CPU executor).
|
||||
# --moe-cache-rate holds the GPU LRU fraction of all experts.
|
||||
# - `ft ctl cache --kv --moe` grows the pools LIVE on a running server
|
||||
# (shrinking needs a restart); `ft ctl stats` reports pool occupancy.
|
||||
# - KV MATH: 12 full-attention layers x 2 KV heads x 256 head_dim x 2 (K+V)
|
||||
# x 2 B = 24 KiB/token (f16 KV; the 36 GDN layers use fixed per-seq state,
|
||||
# not per-token KV). See FRANK_PRIMARY_FT_NUM_TOKENS below for the pool
|
||||
# size and its measured page-size law.
|
||||
FRANK_PRIMARY_IMAGE=localhost/frank-primary-freetoken:pr587-77a7201
|
||||
FRANK_PRIMARY_MODEL_DIR=/home/dank/models/Qwen3.8-Flash-Next-Uncensored-FP8
|
||||
FRANK_PRIMARY_MODEL_SHARDS=131
|
||||
# Container identity is the SERVICE name (Frank's call, 2026-10-05): one
|
||||
# name across systemctl and podman ps. The vLLM-era name frank-primary-qwen
|
||||
# moved into the legacy list so every deploy cleans it up.
|
||||
FRANK_PRIMARY_CONTAINER=frank-primary
|
||||
FRANK_PRIMARY_LEGACY_CONTAINERS="frank-primary-qwen frank-primary-r9v qwen-fp8 glm-probe"
|
||||
FRANK_PRIMARY_SERVED_NAME=frank-primary
|
||||
FRANK_PRIMARY_PUBLISH=7770:7770
|
||||
|
||||
# --- FreeToken engine knobs (first-boot conservative, resize live) -----
|
||||
# Single GPU: the R9700 at index 0. HIP_VISIBLE_DEVICES=0 hides card 1 so
|
||||
# the in-container index 0 IS the right card; --gpu 0 selects it.
|
||||
FRANK_PRIMARY_FT_GPU=0
|
||||
# KV pool in TOKENS (not bytes — vLLM-era FRANK_PRIMARY_KV_BYTES is gone).
|
||||
# 299,968 tokens x 24 KiB = 7.0 GiB; clears the harness worst case
|
||||
# (177,356 tokens) and one NATIVE 262,144-token conversation + full budget.
|
||||
# MEASURED FIRST BOOT (2026-10-05): the auto attention backend is
|
||||
# qsa_sparse, which OVERRIDES page size to 64, so --num-tokens must be a
|
||||
# MULTIPLE OF 64 — 300,000 was refused ("nearest valid values: 299968 or
|
||||
# 300032"). Grows live with `ft ctl cache --kv <n>`; shrink needs a
|
||||
# restart.
|
||||
FRANK_PRIMARY_FT_NUM_TOKENS=299968
|
||||
# Concurrent decode slots. 2 is the first-boot conservative value; the
|
||||
# radix pool admits ~2 full-length 200,704-token conversations. Raise with
|
||||
# --max-running-requests only after a real-traffic look at slot occupancy.
|
||||
FRANK_PRIMARY_FT_MAX_RUNNING=2
|
||||
# Per-request output budget ceiling: the harness's maxTokens (36,864).
|
||||
# Requests that omit max_tokens get this default instead of the engine's
|
||||
# 32,768.
|
||||
FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS=36864
|
||||
# GPU expert LRU cache as a FRACTION of all experts (~120 GiB fp8 total ->
|
||||
# ~12 GiB VRAM at 0.10). First-boot conservative; resize live with
|
||||
# `ft ctl cache --moe` after reading the boot ledger + ft ctl stats.
|
||||
FRANK_PRIMARY_FT_MOE_RATE=0.10
|
||||
# Escape hatch for boot-ladder experiments (YaRN, --max-prefill-length,
|
||||
# --moe-cpu-threads, --sampling-defaults none, ...). Unset -> no extra
|
||||
# argv. Keep multi-flag values as one string; the serve script expands it
|
||||
# unquoted on purpose.
|
||||
FRANK_PRIMARY_FT_EXTRA_ARGS=
|
||||
@@ -1,65 +0,0 @@
|
||||
# frank-primary.service — FreeToken (PR #587 ROCm) endpoint (orcarouter
|
||||
# Qwen3.8-Flash-Next-Uncensored-FP8, 131-shard payload) on ONE R9700
|
||||
# (gfx1201), dankserver :7770. 2026-10-05 cutover from the tcclaviger vLLM
|
||||
# fork (git tag frank-primary-vllm-shapeG-20261003 is the pre-cutover
|
||||
# state; the retired unit/env/serve trio is in that tag).
|
||||
#
|
||||
# Same port, same served alias, same container identity as the unit name;
|
||||
# the harness models.yml id is unchanged. The serve script drives
|
||||
# `podman run` directly (the image carries `ft serve`) and then waits on
|
||||
# the container. The image is built ON THE HOST from
|
||||
# scripts/docker/frank-primary-freetoken.Dockerfile (FreeToken PR head
|
||||
# 77a7201 in the PR's own ROCm 7.14 base image).
|
||||
#
|
||||
# WHY A ROOT UNIT (not a rootless quadlet): FreeToken pins ~120 GiB of
|
||||
# expert weights plus the ~47.7 GiB PLE ngram table in HOST RAM. Pinned
|
||||
# pages are charged against RLIMIT_MEMLOCK; dank's login session on this
|
||||
# host carries 8 MiB and a user quadlet is capped by the user manager's own
|
||||
# hard limit — raising that means restarting user@1000.service and bouncing
|
||||
# every user container on the host. A root unit grants
|
||||
# LimitMEMLOCK=infinity to the dank process via PID1, and the image stays
|
||||
# in dank's rootless podman storage.
|
||||
[Unit]
|
||||
Description=frank-primary — FreeToken (PR587) Qwen3.8-Flash-Next-Uncensored-FP8 on 1x R9700
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
# Never let the start-rate limiter give up after a crash loop (a failed LLM
|
||||
# endpoint that systemd refuses to retry is worse than a retrying one).
|
||||
StartLimitIntervalSec=0
|
||||
|
||||
[Service]
|
||||
User=dank
|
||||
Group=dank
|
||||
# The expert LRU + PLE pin ~170 GiB of host RAM; an RLIMIT_MEMLOCK smaller
|
||||
# than the pinned set fails the load mid-start. PID1 grants infinity here —
|
||||
# parity with the fleet's other LLM units.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-primary.sh
|
||||
# systemd stop: wait alone cannot stop the container child; stop it
|
||||
# explicitly. ft serve drops connections on SIGTERM; no graceful drain
|
||||
# contract is claimed (the OpenAI front end holds no queue to drain —
|
||||
# in-flight decodes are lost, which a stop during a bench window accepts).
|
||||
ExecStop=/usr/bin/podman stop -t 120 frank-primary
|
||||
# Boot cost: 131-shard FP8 page-in + PLE pin + expert upload. vLLM-era
|
||||
# budget (1800 s) carried forward; the FreeToken first boot measured here
|
||||
# once it ran (see the runbook for the real number).
|
||||
TimeoutStartSec=1800
|
||||
# Restart=always, NOT on-failure (the 2026-10-01 vLLM lesson carries over:
|
||||
# a clean API-server exit after a worker fault reads as success to
|
||||
# systemd). `always` restarts on any container death; an explicit
|
||||
# `systemctl stop` still wins, so bench windows and deploys are unaffected.
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
# ExecStop allows the engine 120 s to drain (`podman stop -t 120`); without
|
||||
# an explicit TimeoutStopSec systemd's 90 s default cuts the stop short,
|
||||
# kills it, and records 'Failed with result timeout' (measured 2026-10-01
|
||||
# 13:27:17). Must exceed the podman grace with margin.
|
||||
TimeoutStopSec=180
|
||||
# The engine pins tens of GiB of host RAM (expert rows + PLE table); it is
|
||||
# the last process on this host that should be chosen by the kernel OOM
|
||||
# killer.
|
||||
OOMScoreAdjust=-500
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,136 +0,0 @@
|
||||
# frank-small — llama.cpp (master, CUDA sm_86) serving
|
||||
# ornith-ai/Ornith-1.5-9B-GGUF:Q8_0 on the RTX 3090, :7771.
|
||||
#
|
||||
# 2026-10-05 (Frank): the payload moved off bartowski/MiMo-V2.6 (Q6_K) to
|
||||
# ornith-ai/Ornith-1.5-9B Q8_0 (9,786,060,384 B, LFS oid 22086870...).
|
||||
# Header-verified facts that re-price the shape: qwen35 hybrid like MiMo, but
|
||||
# 33 blocks (not 32) with full_attention_interval 4 => NINE full-attention
|
||||
# layers (MiMo had 8), 16 Q heads / 4 KV heads, k/v length 256, native window
|
||||
# 262144, dense (no expert keys), nextn_predict_layers=1 (an MTP head is IN
|
||||
# the file — capability only; no spec decode ships, same law as before).
|
||||
#
|
||||
# THE KV CONTRACT STAYS f16 K + f16 V — NO KV QUANTIZATION (Frank, 2026-10-05,
|
||||
# overruling a written q8_0-V proposal). The arithmetic that makes this hard:
|
||||
# KV rate 9 attn layers x 4 heads x 256 x 2 B = 18 KiB/token K, same again
|
||||
# for V at f16 => 36 KiB/token (MiMo was ~24.5 measured: one fewer
|
||||
# full-attn layer AND one fewer KV head).
|
||||
# Weights +1,900 MiB (9,334 vs Q6_K's 7,432). Measured free at the Q6_K
|
||||
# 4096/2048 shape was 2,279 MiB; Q8_0 at f16/f16 does not fit that shape —
|
||||
# the deficit is made up the way Frank ordered: UBATCH FIRST (2048 -> 1024,
|
||||
# the measured 616 MiB rung), then POOL STEPS, then measure. Guess-and-check
|
||||
# on the live card is the accepted method for llama.cpp sizing here: boot,
|
||||
# read nvidia-smi free, step the pool (12,288-token increments, ~432 MiB
|
||||
# each at the 36 KiB/token rate) until the 2-3 GiB floor holds. Compute
|
||||
# buffers allocate at graph capture during BOOT, so a failure is loud and
|
||||
# fast, not a surprise at the first long prompt.
|
||||
#
|
||||
# Previous shape history (MiMo era, for the rollback trail): 2026-10-03 the
|
||||
# rung went Q5_K_M -> Q6_K with the pool HELD at 401,408; the freed third
|
||||
# slot funded exact f16 V first; the ~3.5 GiB still free funded the +875 MiB
|
||||
# of weights.
|
||||
#
|
||||
# 2026-10-03 (Frank): TWO streams instead of three, each one WIDE — 200,704
|
||||
# tokens per conversation, above the 200,700 the harness compacts at — and the
|
||||
# KV cache is now FULLY UNQUANTIZED (f16 K, f16 V). The two moves are the same
|
||||
# move: llama.cpp divides --ctx-size across --parallel slots, so the slot given
|
||||
# up is exactly the VRAM that pays for unquantized V. No further quantization
|
||||
# anywhere on this endpoint.
|
||||
#
|
||||
# Three measurements on this card, this runtime, price the terms:
|
||||
# 2 x 262,144, no projector: 20,190 MiB used / 4,386 free
|
||||
# KV = 524,288 tok x 24.5 KiB = 12,800 MiB => non-KV (weights 6,557 +
|
||||
# DeltaNet state ~25/slot + compute buffers + CUDA graphs) = 7,390 MiB
|
||||
# 3 x 196,608, projector on: 22,972 MiB used / 1,153 free
|
||||
# KV = 14,113 MiB => non-KV WITH the vision tower = 8,859 MiB
|
||||
# 3 x 180,224, projector on: 21,684 MiB used / 2,441 free (the 2026-10-02
|
||||
# shape) => non-KV = 8,748 MiB
|
||||
# Two slots give back one DeltaNet state (~25 MiB). Priced at f16 K+V
|
||||
# (32 KiB/token) this pool is 12,544 MiB, so the prediction is
|
||||
# 12,544 + ~8,800 = ~20.9 GiB used, ~3.1 GiB free — the band the 2026-10-02
|
||||
# shape hit, bought with precision instead of a slot. The 1024 ubatch
|
||||
# re-prices the compute buffers upward, so MEASURE after any change: if the
|
||||
# boot lands under 2 GiB free, step the pool down in 12,288-token
|
||||
# (384 MiB at this rate) increments.
|
||||
#
|
||||
# KV arithmetic: attention KV exists only in the 8 full-attention layers
|
||||
# (4 KV heads x 256 head_dim x 2 B) = 16 KiB/token for K and the same again for
|
||||
# V => 32 KiB/token, so 12,288 tokens (= 6144/slot) is one 384 MiB step — that
|
||||
# is the ladder to walk if this moves again. Two native windows (2 x 262,144)
|
||||
# are NOT reachable with full-precision V: 524,288 x 32 KiB = 16.1 GiB of KV
|
||||
# against a card with ~15.7 GiB of KV room once weights, the vision tower, the
|
||||
# DeltaNet states and the graphs are loaded. 200,704 per stream is what full
|
||||
# precision buys; q8_0 on V would buy 262,144 instead.
|
||||
#
|
||||
# The asymmetric-KV contract this endpoint was created for (Frank, 2026-10-01
|
||||
# night: f16 K + q8_0 V) is now spent — it existed to make a third slot fit,
|
||||
# and the third slot is gone. The flags stay explicit so the choice is visible
|
||||
# in the argv either way.
|
||||
LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server
|
||||
MODEL_FILE=/home/dank/models/Ornith-Q8/Ornith-1.5-9B-Q8_0.gguf
|
||||
|
||||
# Vision: the Ornith Q8_0 GGUF ships no `v.*` tensors either (dense-9B byte
|
||||
# arithmetic: 9.79 GB ≈ 9B params x ~1.06 B/param leaves no room for a tower),
|
||||
# so the projector file IS the vision capability (mmproj-Ornith-1.5-9B-BF16
|
||||
# .gguf, 921,704,672 B, LFS oid 626f9f90..., fetched by
|
||||
# scripts/download-frank-small.sh). Empty string = text-only on purpose.
|
||||
MMPROJ_FILE=/home/dank/models/Ornith-Q8/mmproj-Ornith-1.5-9B-BF16.gguf
|
||||
|
||||
# Fleet endpoint: same port and alias as every frank-small before it, so the
|
||||
# harness models.yml id and the homepage probe do not move.
|
||||
ALIAS=frank-small
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7771
|
||||
|
||||
# ONE conversation at the NATIVE 262,144 window (Frank, 2026-10-05). The
|
||||
# two-slot shape died under Ornith and the NO-QUANTIZE law, by arithmetic:
|
||||
# the KV rate is the ARCHITECTURE's, not the quant's — 9 full-attn layers x
|
||||
# 4 KV heads x 256 dim x 2 B = 36 KiB/token at f16 K+V (MiMo measured 24.5),
|
||||
# so 2 x 200,704 wants ~14.1 GiB of KV on top of +1,900 MiB of Q8 weights
|
||||
# against 2,279 MiB measured free; ubatch 1024 claws back 616, and no
|
||||
# pool step fits BOTH the 2-3 GiB floor and the harness compaction
|
||||
# invariant (slot >= 191,148). One native slot fits with room; the window
|
||||
# per conversation RISES 200,704 -> 262,144. The cost: a second concurrent
|
||||
# request queues behind the released slot (subagent concurrency 2 -> 1).
|
||||
CTX_SIZE=262144
|
||||
PARALLEL=1
|
||||
|
||||
# Compute-buffer geometry. This endpoint passed neither flag until 2026-10-03,
|
||||
# so it ran on llama.cpp's own defaults (2048/512); the pair is now 4096/2048.
|
||||
# Both are tunables because the pair is a measurement, not a law — and the
|
||||
# measurement is CARD-SPECIFIC. On the 2x R9700 layer-split sibling the ubatch
|
||||
# curve peaks at 512 and every step up costs GiB of compute buffer for no
|
||||
# speed (see frank-medium-batch-ubatch). On THIS CUDA 3090 the curve climbs
|
||||
# monotonically to 2048: 710 -> 1188 t/s prefill from ubatch 1024 to 2048 on
|
||||
# the 35B MoE sweep, 1944 -> 2423 t/s on the Ornith 8k sweep, with decode flat
|
||||
# to under 1 t/s on every rung. Ubatch is a prefill knob; decode never moves.
|
||||
#
|
||||
# The logical batch is free — two independent sweeps agree that at a fixed
|
||||
# ubatch, 2048/4096/8192 give identical prefill speed AND identical peak VRAM,
|
||||
# because the compute buffers are sized by the ubatch alone. Doubling it costs
|
||||
# nothing and buys nothing; it is here because it is the value the pair was
|
||||
# measured at.
|
||||
#
|
||||
# THE RISK IS ALL IN THE UBATCH. The 1024 -> 2048 step cost ~1,244 MiB of peak
|
||||
# graph VRAM on the 35B MoE (7,330 -> 8,574 MiB); this 9B hybrid has far
|
||||
# smaller compute tensors, so the delta should be well under that, but the
|
||||
# predecessor Q5_K_M shape recorded "ubatch 2048 failed the scratch boot" at
|
||||
# ~3,571 MiB free. The Q8_0 rung ships UBATCH 1024 (Frank, 2026-10-05: lower
|
||||
# ubatch FIRST — the measured 616 MiB step is the first VRAM the Q8 weights
|
||||
# claw back, before any pool move). Compute buffers are allocated at graph
|
||||
# capture during BOOT, so a failure is loud and fast, not a surprise at the
|
||||
# first long prompt. MEASURE nvidia-smi free immediately after the deploy:
|
||||
# under ~1.5 GiB, or a boot failure, step the POOL down next (12,288-token
|
||||
# increments, ~432 MiB each at the Ornith 36 KiB/token f16 KV rate).
|
||||
BATCH_SIZE=4096
|
||||
UBATCH_SIZE=1024
|
||||
|
||||
# The KV contract (2026-10-05, Frank): f16 K + f16 V — NO KV quantization.
|
||||
# The q8_0-V arithmetic was written and OVERRULED. The Q8 weights eat the
|
||||
# VRAM that funded f16 V on 2026-10-03, so the remedy ladder is UBATCH FIRST
|
||||
# (2048 -> 1024, the measured 616 MiB rung), then POOL STEPS — and
|
||||
# guess-and-check on the live card is the accepted method: boot, read
|
||||
# nvidia-smi, step down until the floor holds. --flash-attn stays pinned on
|
||||
# in the serve script: it is the fast path for f16 KV.
|
||||
CACHE_TYPE_K=f16
|
||||
CACHE_TYPE_V=f16
|
||||
NGL=99
|
||||
@@ -1,19 +0,0 @@
|
||||
[Unit]
|
||||
Description=llama.cpp MiMo-V2.6-Distill-Qwen-9B Q6_K GGUF (qwen35 hybrid, 2x200704, unquantized f16 KV, mmproj vision) on the RTX 3090 (dankclaw, frank-small)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
# CUDA pins its staging buffers, so the process needs the memlock budget even
|
||||
# though the weights themselves are copied into VRAM.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,62 @@
|
||||
# frank-zealot — llama.cpp (master, CUDA sm_86) serving
|
||||
# HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive at Q6_K on the RTX 3090,
|
||||
# :7771.
|
||||
#
|
||||
# 2026-10-07 (Frank): swap from Gemma4-26B-A4B QAT Q4_K_M (one 200,704 slot)
|
||||
# to Qwen3.5-9B Q6_K at TWO 200,704-token conversations, EXACT (f16) KV,
|
||||
# MTP disabled (SPEC_TYPE=none), log verbosity 4.
|
||||
#
|
||||
# Architecture — read from the GGUF's own header (HTTP range read, not the
|
||||
# model card): arch qwen35, 32 layers, dense 9B, full_attention_interval 4
|
||||
# ⇒ 8 full-attention layers (3,7,11,...,31) + 24 Gated DeltaNet linear layers,
|
||||
# 16 attention heads / 4 KV heads × 256 dim, ssm conv_kernel 4 / state_size
|
||||
# 128 / group_count 16 / time_step_rank 32 / inner_size 4096, native 262,144,
|
||||
# vocab 248,320, file_type 18 (Q6_K). Tensor-type histogram: Q6_K×250 +
|
||||
# F32×177 — standard types only, so the stock master build loads it (no
|
||||
# custom-quant lock-in like the ROCmFPX era). NO nextn/mtp tensors exist, so
|
||||
# in-file MTP is impossible here; SPEC_TYPE=none is forced by the payload.
|
||||
#
|
||||
# KV rate law: 8 full layers × 4 KV heads × 256 dim = 8,192 elements per
|
||||
# token per cache. f16 K + f16 V = 32,768 B/token = 32 KiB/token, so the
|
||||
# 401,408-token pool is 12.25 GiB. The 24 DeltaNet layers cost a fixed
|
||||
# recurrent state per SLOT (not per token) — that is why two 200k slots fit
|
||||
# where the 26B MoE fit only one.
|
||||
#
|
||||
# Feasibility comes from the predecessor's MEASURED boots, not a formula:
|
||||
# MiMo-V2.6-Distill-Qwen-9B (identical qwen35 geometry, 7,793,710,624 B) at
|
||||
# 2×200,704 f16 K + f16 V measured 21,276 used / 2,849 free at ubatch 1024
|
||||
# and 21,846 / 2,279 at ubatch 2048. This file is 415 MiB lighter in weights,
|
||||
# its BF16 mmproj is 3 MiB heavier ⇒ projected ~21,430 used / ~2,700 free.
|
||||
# The verbosity-4 boot ledger is the referee.
|
||||
#
|
||||
# Ubatch 2048 is the measured CUDA optimum (measured on the MiMo predecessor
|
||||
# at this exact arch + shape: +4.6 % prefill at 8.9k depth, decode flat, cost
|
||||
# 616 MiB). 1024 is the fallback rung if the boot lands under the 2 GiB floor.
|
||||
LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server
|
||||
MODEL_FILE=/home/dank/models/Qwen35-9B-HauhauCS-Q6K/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K.gguf
|
||||
MMPROJ_FILE=/home/dank/models/Qwen35-9B-HauhauCS-Q6K/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf
|
||||
# Served model id — the harness/family name (Protoss rename completed to the
|
||||
# wire 2026-10-09: harness ids + homepage cards 2026-10-08, /v1/models aliases
|
||||
# this day). The SERVICE, unit, scripts and this env file keep the
|
||||
# frank-zealot name; only the alias on the wire changed. Consumers that
|
||||
# key on the served id: OpenWebUI (model row, access_grant, api_configs
|
||||
# model_ids) and the harness models.yml.
|
||||
ALIAS=frank-zealot
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7771
|
||||
# CTX_SIZE is the POOL llama.cpp divides across slots: 401,408 / 2 = 200,704
|
||||
# per conversation. MUST stay divisible by PARALLEL (the seam enforces it) and
|
||||
# the harness contextWindow quotes the PER-SLOT number — a longer prompt is
|
||||
# answered with a 400, never truncated.
|
||||
CTX_SIZE=401408
|
||||
PARALLEL=2
|
||||
BATCH_SIZE=4096
|
||||
UBATCH_SIZE=2048
|
||||
CACHE_TYPE_K=f16
|
||||
CACHE_TYPE_V=f16
|
||||
SPEC_TYPE=none
|
||||
NGL=99
|
||||
# Log verbosity threshold (llama.cpp -lv). 4 = debug: KV layout, buffer sizes
|
||||
# and graph capture land in the journal, which is what the VRAM-fit forensics
|
||||
# read. Levels above 4 flood the journal.
|
||||
VERBOSITY=4
|
||||
@@ -0,0 +1,19 @@
|
||||
[Unit]
|
||||
Description=llama.cpp Qwen3.5-9B-Uncensored-HauhauCS-Aggressive Q6_K GGUF (qwen35 hybrid, 2x200704 slots, f16 K+V, MTP off, mmproj vision) on the RTX 3090 (dankclaw, frank-zealot)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
# CUDA pins its staging buffers, so the process needs the memlock budget even
|
||||
# though the weights themselves are copied into VRAM.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_ZEALOT_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-zealot.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-zealot.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Executable
+107
@@ -0,0 +1,107 @@
|
||||
#!/bin/sh
|
||||
# hfget.sh -- fetch a Hugging Face repo with curl and jq, and nothing else.
|
||||
#
|
||||
# The engine and its tooling do not depend on Python, so `huggingface-cli` is not assumed. The API
|
||||
# gives a file listing as JSON and every file has a stable resolve/ URL, which is all a downloader
|
||||
# needs.
|
||||
#
|
||||
# scripts/hfget.sh <repo-id> [dest-dir] [revision]
|
||||
#
|
||||
# Set HF_TOKEN for a gated or private repo. Re-running resumes: a file whose local size already
|
||||
# matches the listing is skipped and a short one is continued with `curl -C -`, so an interrupted
|
||||
# transfer of a 300 GB checkpoint costs only what it had not finished.
|
||||
#
|
||||
# WEIGHT DUPLICATES ARE SKIPPED. Many repos carry the same tensors several times over -- safetensors
|
||||
# beside the older pickle .bin, an ONNX export, a GGUF conversion -- and on a large checkpoint each
|
||||
# copy is hundreds of gigabytes of the same numbers. When the listing holds any .safetensors, every
|
||||
# other weight format is left behind.
|
||||
#
|
||||
# HFGET_JOBS concurrent transfers, default 4. These are long sequential reads: a few streams
|
||||
# saturate the link and more only makes the destination seek.
|
||||
set -e
|
||||
|
||||
# ---- child mode: one file, named by "<size><tab><path>" in $1, everything else inherited -------
|
||||
# sh cannot export a function, so parallelism is one re-entry of this script per file under
|
||||
# `xargs -P` rather than a worker pool.
|
||||
if [ -n "$HFGET_CHILD" ]; then
|
||||
size=${1%% *}; path=${1#* }
|
||||
out=$HFGET_DEST/$path
|
||||
if [ -f "$out" ]; then
|
||||
cur=$(wc -c < "$out")
|
||||
if [ "$cur" = "$size" ]; then echo " = $path"; exit 0; fi
|
||||
echo " + $path (resuming at $cur of $size)"
|
||||
else
|
||||
echo " > $path"
|
||||
fi
|
||||
mkdir -p "$(dirname "$out")"
|
||||
if [ -n "$HF_TOKEN" ]; then
|
||||
curl -sSL --fail --retry 5 --retry-delay 5 --retry-all-errors -C - \
|
||||
-H "Authorization: Bearer $HF_TOKEN" \
|
||||
-o "$out" "https://huggingface.co/$HFGET_REPO/resolve/$HFGET_REV/$path"
|
||||
else
|
||||
curl -sSL --fail --retry 5 --retry-delay 5 --retry-all-errors -C - \
|
||||
-o "$out" "https://huggingface.co/$HFGET_REPO/resolve/$HFGET_REV/$path"
|
||||
fi
|
||||
got=$(wc -c < "$out")
|
||||
[ "$got" = "$size" ] || { echo "hfget: $path is $got bytes, listing says $size" >&2; exit 1; }
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ---- parent ------------------------------------------------------------------------------------
|
||||
repo=$1
|
||||
[ -n "$repo" ] || { echo "usage: hfget.sh <repo-id> [dest-dir] [revision]" >&2; exit 2; }
|
||||
dest=${2:-$HOME/models/$repo}
|
||||
rev=${3:-main}
|
||||
jobs=${HFGET_JOBS:-4}
|
||||
command -v jq >/dev/null || { echo "hfget: jq is required" >&2; exit 2; }
|
||||
|
||||
echo "hfget: listing $repo@$rev"
|
||||
# THE LISTING IS PAGINATED AND THE PAGE SIZE IS SMALL. A large checkpoint runs to a hundred-odd
|
||||
# shards and the API returns fifty entries with a `Link: ...; rel="next"` header. Taking only the
|
||||
# first page is the worst possible failure here: every file it did fetch is the right length, so
|
||||
# the size check passes and the converter is handed a checkpoint missing two thirds of its
|
||||
# tensors. Follow the cursor until there is no next link.
|
||||
api="https://huggingface.co/api/models/$repo/tree/$rev?recursive=true&expand=true"
|
||||
listing='[]'
|
||||
page=0
|
||||
while [ -n "$api" ]; do
|
||||
hdr=$(mktemp "${TMPDIR:-/tmp}/hfhdr.XXXXXX")
|
||||
if [ -n "$HF_TOKEN" ]; then
|
||||
body=$(curl -sSL --fail-with-body -D "$hdr" -H "Authorization: Bearer $HF_TOKEN" "$api")
|
||||
else
|
||||
body=$(curl -sSL --fail-with-body -D "$hdr" "$api")
|
||||
fi
|
||||
[ "$(printf '%s' "$body" | jq -r 'type')" = "array" ] || {
|
||||
rm -f "$hdr"
|
||||
echo "hfget: $(printf '%s' "$body" | jq -r '.error // .message // .')" >&2
|
||||
echo "hfget: a gated or private repo needs HF_TOKEN in the environment" >&2; exit 1; }
|
||||
listing=$(printf '%s\n%s' "$listing" "$body" | jq -s 'add')
|
||||
api=$(tr -d '\r' < "$hdr" | sed -n 's/^[Ll]ink: *<\([^>]*\)>; *rel="next".*/\1/p' | head -1)
|
||||
rm -f "$hdr"
|
||||
page=$((page + 1))
|
||||
[ "$page" -gt 200 ] && { echo "hfget: listing did not terminate after 200 pages" >&2; exit 1; }
|
||||
done
|
||||
echo "hfget: $page page(s) of listing"
|
||||
have_st=$(printf '%s' "$listing" | jq '[.[] | select(.path | endswith(".safetensors"))] | length')
|
||||
sel='.[] | select(.type == "file")'
|
||||
[ "$have_st" -gt 0 ] && sel="$sel"' | select((.path | test("\\.(bin|pth|msgpack|h5|onnx|onnx_data|gguf)$")) | not)'
|
||||
|
||||
list=$(mktemp "${TMPDIR:-/tmp}/hfget.XXXXXX")
|
||||
trap 'rm -f "$list"' EXIT INT TERM
|
||||
printf '%s' "$listing" | jq -r "$sel"' | "\(.size)\t\(.path)"' > "$list"
|
||||
|
||||
n=$(wc -l < "$list")
|
||||
[ "$n" -gt 0 ] || { echo "hfget: the listing has no files" >&2; exit 1; }
|
||||
need=$(awk -F'\t' '{s += $1} END {printf "%.1f", s / 1073741824}' "$list")
|
||||
mkdir -p "$dest"
|
||||
free=$(df -BG --output=avail "$dest" | tail -1 | tr -dc '0-9')
|
||||
echo "hfget: $n file(s), $need GiB -> $dest"
|
||||
[ "$have_st" -gt 0 ] && echo "hfget: safetensors present, pickle weight formats skipped"
|
||||
echo "hfget: $free GiB free on the destination filesystem"
|
||||
awk -v n="$need" -v f="$free" 'BEGIN { if (n > f) { print "hfget: THE DOWNLOAD DOES NOT FIT"; exit 1 } }'
|
||||
|
||||
export HFGET_CHILD=1 HFGET_DEST="$dest" HFGET_REPO="$repo" HFGET_REV="$rev"
|
||||
self=$(cd "$(dirname "$0")" && pwd)/$(basename "$0")
|
||||
tr '\n' '\0' < "$list" | xargs -0 -P "$jobs" -I{} sh "$self" "{}" \
|
||||
|| { echo "hfget: incomplete -- re-run to resume" >&2; exit 1; }
|
||||
echo "hfget: complete -- $(du -sh "$dest" | cut -f1) in $dest"
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
# kld-gate-swift15.sh — RUNS ON DANKSERVER. Score the quantized candidate
|
||||
# against the BF16 reference and apply the pre-approved gate:
|
||||
# mean KL <= 0.03 AND top-1 agreement >= 95%.
|
||||
# Same engine (1.0.13) and same boot shape as the reference RECORD
|
||||
# (run-swift15-bf16.sh kld): disk-tier, no speculation, deterministic, fp8 KV.
|
||||
# --kld-ref reads its tokens from the reference (no --kld-corpus here — the
|
||||
# (run-swift15-bf16.sh kld): 1.0.13, tp 2, disk-tier, fp8 KV, no speculation,
|
||||
# deterministic, max-model-len 16384, vram-kv 3072, 4 slots. Two deviations
|
||||
# are forced by the w4 file: MNBT 2048 (the w4 kernels stream from the file
|
||||
# tier ONLY at MNBT<=2048 — at the recording's 32768 the engine demands a
|
||||
# widened form: "DID NOT FIT ... stored form is not the container's"), and
|
||||
# pool 40960 (the recording's 12 GiB pool predates the w4 tables; at 12288 the
|
||||
# gate_up slab cannot be cached). Chunk size is softmax-invariant, so the
|
||||
# MNBT delta is float-order noise riding on the KL floor.
|
||||
set -uo pipefail
|
||||
PIN=sha256:4c3f484564bb2872784e46b0a59cbd71becc9c1e746e301a3596ffae253c322e
|
||||
C=swift15gate
|
||||
LOG=/home/dank/logs/swift15_kldgate.log
|
||||
KLD_REF_DIR=/home/dank/corpora/kld-ref
|
||||
KLD_OUT=/home/dank/radiance-state/kld-gate.json
|
||||
|
||||
[ -d "$KLD_REF_DIR" ] || { echo "missing reference: $KLD_REF_DIR"; exit 2; }
|
||||
: > "$KLD_OUT"
|
||||
cp -f /home/dank/serverAppConfigs/scripts/qwen38-froggeric-v22.5.jinja \
|
||||
/home/dank/models/tpl.jinja 2>/dev/null || true
|
||||
sudo systemctl stop frank-archon >/dev/null 2>&1 || true
|
||||
podman rm -f "$C" >/dev/null 2>&1 || true
|
||||
|
||||
podman run --rm --name "$C" \
|
||||
--network host --init \
|
||||
--security-opt seccomp=unconfined \
|
||||
--ulimit memlock=-1:-1 \
|
||||
--device /dev/kfd --device /dev/dri \
|
||||
-v /home/dank/models:/models:ro \
|
||||
-v /home/dank/radiance-state:/data \
|
||||
-v "$KLD_REF_DIR:/data/kld-ref:ro" \
|
||||
"stilldeadcode/radiance@$PIN" \
|
||||
--model /models/rad/qwen4exp-swift15-w4dn8-i8-hc8m.rad \
|
||||
--tp 2 --placement expert_tiered --weights-disk-tier \
|
||||
--host-pool-mib 40960 --deterministic --gpu-headroom-mib 768 \
|
||||
--num-speculative-tokens 0 \
|
||||
--kv-cache-dtype fp8 --max-model-len 16384 \
|
||||
--ngram-placement ram \
|
||||
--override-chat-template /models/tpl.jinja \
|
||||
--kld-ref /data/kld-ref --kld-out /data/kld-gate.json \
|
||||
--vram-kv-mib 3072 \
|
||||
--max-num-seqs 4 --max-num-batched-tokens 2048 \
|
||||
2>&1 | tee "$LOG"
|
||||
rc=${PIPESTATUS[0]}
|
||||
[ "$rc" -eq 0 ] || { echo "KLD GATE BOOT FAILED rc=$rc; log: $LOG"; exit "$rc"; }
|
||||
|
||||
python3 - "$KLD_OUT" "$LOG" <<'PY'
|
||||
import json, sys, re
|
||||
out, log = sys.argv[1], sys.argv[2]
|
||||
def num(pat):
|
||||
m = re.findall(pat, open(log, errors="replace").read(), re.I)
|
||||
return float(m[-1]) if m else None
|
||||
kl = top1 = None
|
||||
try:
|
||||
d = json.load(open(out))
|
||||
def find(obj, keys):
|
||||
if isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
if any(s in k.lower() for s in keys) and isinstance(v, (int, float)):
|
||||
return float(v)
|
||||
for v in obj.values():
|
||||
r = find(v, keys)
|
||||
if r is not None: return r
|
||||
return None
|
||||
kl = find(d, ("mean_kl", "mean kl", "kl_mean", "mean"))
|
||||
top1 = find(d, ("top1", "top_1", "top-1", "agreement"))
|
||||
if top1 is not None and top1 <= 1.0:
|
||||
top1 *= 100.0
|
||||
except Exception as e:
|
||||
print("report JSON unusable (%s); falling back to log" % e)
|
||||
if kl is None:
|
||||
kl = num(r"mean[_ ]kl[^0-9]*([0-9]*\.?[0-9]+(?:e[-+][0-9]+)?)")
|
||||
if top1 is None:
|
||||
top1 = num(r"top[-_ ]?1[^0-9]*([0-9]*\.?[0-9]+)")
|
||||
if kl is None or top1 is None:
|
||||
print("KLD GATE PARSE FAILED: mean_kl=%s top1=%s — inspect %s / %s" % (kl, top1, out, log))
|
||||
sys.exit(2)
|
||||
print("KLD GATE: mean KL = %.4f (max 0.03) | top-1 = %.2f%% (min 95.0)" % (kl, top1))
|
||||
ok = (kl <= 0.03) and (top1 >= 95.0)
|
||||
print("KLD GATE " + ("PASS" if ok else "FAIL"))
|
||||
sys.exit(0 if ok else 1)
|
||||
PY
|
||||
gate=$?
|
||||
echo "kld-gate complete rc=$gate; log: $LOG report: $KLD_OUT"
|
||||
exit "$gate"
|
||||
Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/bin/bash
|
||||
# kld-then-calib.sh — RUNS ON DANKSERVER. Campaign chain: wait for the KLD
|
||||
# record to finish, verify it, then boot the 1.2.0 calibration server and
|
||||
# push the 10M-token corpus. One GPU, so the phases serialize; prod is
|
||||
# restored by each run script's trap.
|
||||
set -u
|
||||
KLOG=/home/dank/logs/swift15-kld-driver.log
|
||||
CLOG=/home/dank/logs/swift15-calib-driver.log
|
||||
S=/home/dank/serverAppConfigs-swift15/scripts
|
||||
# Phase 1: KLD record (SKIP_KLD=1 when the record is already complete —
|
||||
# kld.json's rows count is the receipt; a mid-run script swap can kill the
|
||||
# wrapper with rc=127 after the engine exited clean).
|
||||
if [ "${SKIP_KLD:-0}" != 1 ]; then
|
||||
while ! grep -q "exited rc=" "$KLOG" 2>/dev/null; do sleep 60; done
|
||||
RC=$(grep -o "exited rc=[0-9]*" "$KLOG" | tail -1)
|
||||
if [ "$RC" != "exited rc=0" ]; then echo "KLD FAILED ($RC); calib NOT started"; exit 1; fi
|
||||
fi
|
||||
N=$(find /home/dank/corpora/kld-ref -type f | wc -l)
|
||||
SZ=$(du -sm /home/dank/corpora/kld-ref | cut -f1)
|
||||
echo "KLD OK files=$N size_mib=$SZ"
|
||||
[ "$N" -ge 5 ] && [ "$SZ" -ge 1000 ] || { echo "kld-ref looks thin; calib NOT started"; exit 1; }
|
||||
grep -q '"rows": 50235' /home/dank/corpora/kld-ref/kld.json || {
|
||||
echo "kld.json rows != 50235; calib NOT started"; exit 1; }
|
||||
|
||||
# Phase 2: calibration serve (1.2.0) + corpus push.
|
||||
nohup bash "$S/run-swift15-bf16.sh" calib > "$CLOG" 2>&1 &
|
||||
CP=$!
|
||||
for _ in $(seq 1 160); do
|
||||
sleep 15
|
||||
curl -sf localhost:7770/health >/dev/null 2>&1 && break
|
||||
done
|
||||
if ! curl -sf localhost:7770/health >/dev/null 2>&1; then
|
||||
echo "CALIB BOOT FAILED; see /home/dank/logs/swift15_bf16.log"; kill "$CP" 2>/dev/null; exit 1
|
||||
fi
|
||||
echo "CALIB UP, pushing corpus"
|
||||
python3 "$S/calib-push.py" --port 7770 --parallel 16 --container swift15ref --max-rounds 6
|
||||
PUSH=$?
|
||||
kill "$CP" 2>/dev/null
|
||||
sleep 10
|
||||
echo "CALIB_CHAIN_DONE push=$PUSH grams=$(find /home/dank/corpora/calib -name 'gram.r*.bin' 2>/dev/null | wc -l)"
|
||||
Executable
+317
@@ -0,0 +1,317 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the two Swift1.5 campaign corpora (calibration + KLD).
|
||||
|
||||
calib.jsonl — 10M tokens of agent-shaped RAW text (45% code, 30% prose,
|
||||
25% tool-call JSON), one {"prompt": text} per line, pushed to
|
||||
/v1/completions with max_tokens=1 during the calibration run (the Hessian
|
||||
tap is of the prompt tokens; docs/MOE-W4.md step 2). Code and prose mix
|
||||
the repo checkout with HF dataset parquets (Frank's call 2026-10-06:
|
||||
local material is only ~1.6M unique tokens; MOE-W4 wants 10M):
|
||||
nampdn-ai/tiny-codes (code) and Salesforce/wikitext wikitext-103-raw
|
||||
(prose). Tool calls are synthesized from commands harvested from the
|
||||
repo's own scripts plus a fleet-ops set, with argument-shaped variants.
|
||||
|
||||
kld.jsonl — 1.8M tokens, one {"prompt", "score_from", "source"} per
|
||||
line; each prompt is a multi-turn conversation rendered through the
|
||||
Froggeric v22.5 template with a >= --kld-min-prompt (8192) token target,
|
||||
and the last 256 tokens (the tail of the assistant answer) are the
|
||||
scored span: score_from = n - 256. At 8k-token prompts the 1.8M corpus
|
||||
is ~220 prompts = ~56k scored positions ~= 25 GB of record, matching
|
||||
the published Flash-Next KLD scale (83k positions, MOE-W4.md). The KLD
|
||||
reference run and every candidate run consume the SAME file (TOOLS.md
|
||||
--kld-record / --kld-ref). The KLD corpus is LOCAL material only, so the
|
||||
file-level disjointness rule (fnv1a64(path) % 5 == 0 -> KLD pool) is
|
||||
untouched by the HF additions.
|
||||
|
||||
Deps: tokenizers, jinja2, pyarrow (venv /home/dank/venvs/radcorpus).
|
||||
Usage: make-calib-corpus.py --repo ROOT --tokenizer-json PATH
|
||||
--template PATH --out-dir DIR [--hf-code-parquet F,F]
|
||||
[--hf-prose-parquet F,F] [--calib-tokens 10000000]
|
||||
[--kld-tokens 1800000]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
from tokenizers import Tokenizer
|
||||
|
||||
CODE_EXT = (".sh", ".py", ".jinja", ".recipe", ".env", ".example",
|
||||
".yaml", ".yml", ".bats", ".service", ".container", ".toml")
|
||||
JSONL_EXTS = (".jsonl", ".json")
|
||||
DOC_DIR = "serverAppConfigs-docs"
|
||||
|
||||
FLEET_CMDS = [
|
||||
"systemctl status frank-archon --no-pager",
|
||||
"journalctl -u frank-archon -n 100 --no-pager",
|
||||
"podman ps --format '{{.Names}} {{.Status}}'",
|
||||
"podman logs --tail 50 frank-archon",
|
||||
"curl -s http://192.168.0.235:7770/health",
|
||||
"curl -s http://192.168.0.235:7770/metrics | grep radiance_",
|
||||
"df -h /home/dank | tail -1",
|
||||
"free -g",
|
||||
"sudo systemctl restart frank-archon",
|
||||
"cd /home/dank/serverAppConfigs && git pull --ff-only",
|
||||
"./scripts/deploy.sh dankserver --service frank-archon",
|
||||
"./tests/run.sh 2>&1 | tail -5",
|
||||
"jq . /home/dank/models/rad/index.json | head -40",
|
||||
"python3 -c \"import json; print(json.load(open('config.json'))['num_hidden_layers'])\"",
|
||||
"ip neigh show 192.168.0.235",
|
||||
"amdgpu_top --device-info 2>/dev/null | head -20",
|
||||
"grep -E 'MemAvailable' /proc/meminfo",
|
||||
"sha256sum model-00001-of-00131.safetensors",
|
||||
"git log --oneline -10",
|
||||
"awk '{s+=$1} END {print s}' shard_sizes.txt",
|
||||
]
|
||||
TOOL_NAMES = ["bash", "read", "grep", "web_search", "edit", "write"]
|
||||
WORKDIRS = ["/home/dank", "/home/dank/serverAppConfigs", "/home/dank/models",
|
||||
"/etc/systemd/system", "/var/log"]
|
||||
DESCS = ["check service state", "tail the engine ledger", "list containers",
|
||||
"query health", "read metrics", "check disk", "check memory",
|
||||
"restart the endpoint", "sync the checkout", "run deployer",
|
||||
"run the test seams", "inspect the index", "probe the peer"]
|
||||
|
||||
|
||||
def fnv(s: str) -> int:
|
||||
h = 0xcbf29ce484222325
|
||||
for b in s.encode():
|
||||
h = ((h ^ b) * 0x100000001b3) & 0xFFFFFFFFFFFFFFFF
|
||||
return h
|
||||
|
||||
|
||||
def harvest_cmds(repo: str, rng: random.Random) -> list[str]:
|
||||
cmds = list(FLEET_CMDS)
|
||||
pat = re.compile(r"^\s*(?:sudo\s+)?([a-z][\w./-]*(?:\s+\S+){1,6})\s*$")
|
||||
for root, dirs, files in os.walk(repo):
|
||||
dirs[:] = [d for d in dirs if not d.startswith(".")]
|
||||
for f in files:
|
||||
if not f.endswith(".sh"):
|
||||
continue
|
||||
try:
|
||||
for line in open(os.path.join(root, f), encoding="utf-8",
|
||||
errors="replace"):
|
||||
m = pat.match(line.rstrip())
|
||||
if m and len(m.group(1)) > 12:
|
||||
cmds.append("ssh dank@192.168.0.235 '"
|
||||
+ m.group(1)[:90] + "'")
|
||||
except OSError:
|
||||
pass
|
||||
rng.shuffle(cmds)
|
||||
return cmds[:4000]
|
||||
|
||||
|
||||
def tool_call(rng: random.Random, cmds: list[str]) -> str:
|
||||
"""One agent turn: 2-4 tool calls with argument-shaped JSON."""
|
||||
parts = []
|
||||
for _ in range(rng.randint(2, 4)):
|
||||
args: dict = {"command": rng.choice(cmds)}
|
||||
if rng.random() < 0.4:
|
||||
args["cwd"] = rng.choice(WORKDIRS)
|
||||
if rng.random() < 0.3:
|
||||
args["timeout"] = rng.choice([10, 30, 60, 300, 600])
|
||||
if rng.random() < 0.5:
|
||||
args["description"] = rng.choice(DESCS)
|
||||
parts.append('<tool_call>{"name": "%s", "arguments": %s}<|im_end|>'
|
||||
% (rng.choice(TOOL_NAMES),
|
||||
json.dumps(args, ensure_ascii=False)))
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def load_parquet_texts(path: str, kind: str, rng: random.Random,
|
||||
cap: int) -> list[str]:
|
||||
import pyarrow.parquet as pq
|
||||
d = pq.read_table(path).to_pydict()
|
||||
n = len(next(iter(d.values())))
|
||||
idx = range(n) if n <= cap else rng.sample(range(n), cap)
|
||||
out = []
|
||||
for i in idx:
|
||||
if kind == "code":
|
||||
if "content" in d:
|
||||
frag = d["content"][i]
|
||||
else:
|
||||
frag = ("// %s\n### Instruction:\n%s\n### Response:\n%s"
|
||||
% (d.get("file_name", [""])[i], d["prompt"][i],
|
||||
d["code"][i] or d["response"][i]))
|
||||
else:
|
||||
frag = d["text"][i]
|
||||
if frag and frag.strip():
|
||||
out.append(frag[:8000])
|
||||
rng.shuffle(out)
|
||||
return out[:cap]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--repo", required=True)
|
||||
ap.add_argument("--tokenizer-json", required=True)
|
||||
ap.add_argument("--template", required=True)
|
||||
ap.add_argument("--out-dir", required=True)
|
||||
ap.add_argument("--calib-tokens", type=int, default=10_000_000)
|
||||
ap.add_argument("--kld-tokens", type=int, default=1_800_000)
|
||||
ap.add_argument("--chunk-tokens", type=int, default=8192)
|
||||
ap.add_argument("--score-tail", type=int, default=256)
|
||||
ap.add_argument("--kld-min-prompt", type=int, default=8192)
|
||||
ap.add_argument("--hf-code-parquet", default="")
|
||||
ap.add_argument("--hf-prose-parquet", default="")
|
||||
ap.add_argument("--seed", type=int, default=20261006)
|
||||
a = ap.parse_args()
|
||||
|
||||
rng = random.Random(a.seed)
|
||||
tokenizer = Tokenizer.from_file(a.tokenizer_json)
|
||||
env = Environment(loader=FileSystemLoader(os.path.dirname(a.template)),
|
||||
trim_blocks=True, lstrip_blocks=True)
|
||||
env.filters["tojson"] = lambda v: json.dumps(v, ensure_ascii=False)
|
||||
env.filters["raise_exception"] = lambda m: (_ for _ in ()).throw(
|
||||
ValueError(m))
|
||||
tname = os.path.basename(a.template)
|
||||
|
||||
code, prose, jsonl = [], [], []
|
||||
for root, dirs, files in os.walk(a.repo):
|
||||
dirs[:] = [d for d in dirs if not d.startswith(".")]
|
||||
for f in sorted(files):
|
||||
p = os.path.relpath(os.path.join(root, f), a.repo)
|
||||
if DOC_DIR in p and f.endswith(".md"):
|
||||
prose.append(p)
|
||||
elif p.startswith(DOC_DIR):
|
||||
pass
|
||||
elif f.endswith(CODE_EXT):
|
||||
code.append(p)
|
||||
elif f.endswith(JSONL_EXTS) and f != "package-lock.json":
|
||||
jsonl.append(p)
|
||||
# file-level split: bucket 0 -> KLD pool, 1..4 -> calibration pool
|
||||
pools: dict[str, tuple[list, list]] = {}
|
||||
for cls, lst in (("code", code), ("prose", prose), ("tools", jsonl)):
|
||||
pools[cls] = ([p for p in lst if fnv(p) % 5 == 0],
|
||||
[p for p in lst if fnv(p) % 5 != 0])
|
||||
|
||||
def text_of(p: str) -> str:
|
||||
path = os.path.join(a.repo, p)
|
||||
raw = open(path, encoding="utf-8", errors="replace").read()
|
||||
if p.endswith(JSONL_EXTS):
|
||||
lines = [json.loads(l)["content"] for l in raw.splitlines()
|
||||
if l.strip() and "content" in l]
|
||||
raw = "\n".join(lines)
|
||||
toks = tokenizer.encode(raw, add_special_tokens=False).ids
|
||||
return tokenizer.decode(toks[:2400], skip_special_tokens=True)
|
||||
|
||||
cmds = harvest_cmds(a.repo, rng)
|
||||
# fragment pools per class, corpus-split first
|
||||
frags: dict[str, dict[str, list[str]]] = {}
|
||||
for cls in pools:
|
||||
kld_files, cal_files = pools[cls]
|
||||
fr = {}
|
||||
src = {"kld": kld_files, "cal": cal_files}
|
||||
if cls == "tools":
|
||||
fr["kld"] = [tool_call(rng, cmds) for _ in range(600)]
|
||||
fr["cal"] = [tool_call(rng, cmds) for _ in range(40000)]
|
||||
else:
|
||||
fr["kld"] = [text_of(p) for p in src["kld"]]
|
||||
fr["cal"] = [text_of(p) for p in src["cal"]]
|
||||
frags[cls] = fr
|
||||
# per-class fragment caps: prose rows are short (~66 tok) so need many;
|
||||
# codeparrot files are long (~2.6k tok) so a small cap suffices
|
||||
hfcap = {"code": 40000, "prose": 60000, "tools": 0}
|
||||
for cls, spec in (("code", a.hf_code_parquet),
|
||||
("prose", a.hf_prose_parquet)):
|
||||
for path in [p for p in spec.split(",") if p]:
|
||||
frags[cls]["cal"].extend(
|
||||
load_parquet_texts(path, cls, rng, cap=hfcap[cls]))
|
||||
for cls in frags:
|
||||
rng.shuffle(frags[cls]["cal"])
|
||||
|
||||
def toks(s: str) -> int:
|
||||
return len(tokenizer.encode(s, add_special_tokens=False).ids)
|
||||
|
||||
def emit_calib() -> tuple[int, list[dict]]:
|
||||
targets = {"code": .45, "prose": .30, "tools": .25}
|
||||
rows: list[dict] = []
|
||||
for cls, frac in targets.items():
|
||||
want = int(a.calib_tokens * frac)
|
||||
pool = frags[cls]["cal"]
|
||||
if not pool:
|
||||
continue
|
||||
got, i, buf, nbuf = 0, 0, [], 0
|
||||
while got < want:
|
||||
frag = pool[i % len(pool)]
|
||||
i += 1
|
||||
n = toks(frag)
|
||||
if nbuf + n > a.chunk_tokens and buf:
|
||||
rows.append({"prompt": "\n".join(buf), "cls": cls})
|
||||
got += nbuf
|
||||
buf, nbuf = [], 0
|
||||
if got >= want:
|
||||
break
|
||||
buf.append(frag)
|
||||
nbuf += n
|
||||
if i > len(pool) * 40:
|
||||
break # pool exhausted with repeats; stop
|
||||
if buf and got < want:
|
||||
rows.append({"prompt": "\n".join(buf), "cls": cls})
|
||||
got += nbuf
|
||||
print(f"calib {cls}: {got} tokens, {i} fragments")
|
||||
rng.shuffle(rows)
|
||||
return len(rows), [{k: v for k, v in r.items() if k == "prompt"}
|
||||
for r in rows]
|
||||
|
||||
def emit_kld() -> tuple[int, list[dict]]:
|
||||
rows, total = [], 0
|
||||
classes = ["code", "prose", "tools"]
|
||||
i = {c: 0 for c in classes}
|
||||
guard = 0
|
||||
while total < a.kld_tokens and guard < 5000:
|
||||
guard += 1
|
||||
msgs, srcs = [], []
|
||||
for t in range(rng.choice([2, 3, 4])):
|
||||
cls = classes[(len(rows) + t) % 3]
|
||||
pool = frags[cls]["kld"]
|
||||
if not pool:
|
||||
continue
|
||||
frag = pool[i[cls] % len(pool)]
|
||||
i[cls] += 2
|
||||
msgs.append({"role": "user",
|
||||
"content": "Summarize and explain this "
|
||||
f"engineering material:\n{frag}"})
|
||||
msgs.append({"role": "assistant",
|
||||
"content": frag[:4000]})
|
||||
srcs.append(cls)
|
||||
rendered = env.get_template(tname).render(
|
||||
messages=msgs, add_generation_prompt=False)
|
||||
n = toks(rendered)
|
||||
if n < a.kld_min_prompt:
|
||||
continue
|
||||
rows.append({"prompt": rendered,
|
||||
"score_from": n - a.score_tail,
|
||||
"source": "+".join(sorted(set(srcs)))})
|
||||
total += n
|
||||
return len(rows), rows
|
||||
|
||||
os.makedirs(a.out_dir, exist_ok=True)
|
||||
nc, rows = emit_calib()
|
||||
with open(os.path.join(a.out_dir, "calib.jsonl"), "w") as f:
|
||||
for r in rows:
|
||||
f.write(json.dumps(r) + "\n")
|
||||
nk, krows = emit_kld()
|
||||
with open(os.path.join(a.out_dir, "kld.jsonl"), "w") as f:
|
||||
for r in krows:
|
||||
f.write(json.dumps(r) + "\n")
|
||||
scored = sum(len(tokenizer.encode(r["prompt"], add_special_tokens=False).ids)
|
||||
- r["score_from"] for r in krows)
|
||||
manifest = {
|
||||
"seed": a.seed, "template": tname,
|
||||
"tokenizer": os.path.basename(a.tokenizer_json),
|
||||
"calib_prompts": nc, "kld_prompts": nk,
|
||||
"kld_scored_positions": scored,
|
||||
"kld_score_tail": a.score_tail,
|
||||
"disjoint_file_pools": "local KLD material only; HF code feeds calib",
|
||||
"calib_mix": {"code": .45, "prose": .30, "tools": .25},
|
||||
"hf_code": a.hf_code_parquet, "hf_prose": a.hf_prose_parquet,
|
||||
}
|
||||
with open(os.path.join(a.out_dir, "manifest.json"), "w") as f:
|
||||
json.dump(manifest, f, indent=1)
|
||||
print(json.dumps(manifest, indent=1))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# STATUS: WIP — first of three patches in the "compile the GLM trunk" changelist. The
|
||||
# set does NOT yet boot the engine (see 03 for the next blocker), so it is NOT part of
|
||||
# the deployed frank-primary shape. Nothing in scripts/ mounts these files.
|
||||
# the deployed frank-archon shape. Nothing in scripts/ mounts these files.
|
||||
#
|
||||
# WHY THIS IS THE RIGHT LEVER (measured 2026-09-30, not inferred):
|
||||
#
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# gpu_worker-kv-cap.patch — fork-local fix for the expert-offload KV cap.
|
||||
#
|
||||
# STATUS: NOT part of the deployed frank-primary shape. It exists because the
|
||||
# STATUS: NOT part of the deployed frank-archon shape. It exists because the
|
||||
# fork's plan under-models this model's KV, and it is the enabler for a future
|
||||
# MTP attempt (the production GLM shape runs without MTP and boots fine on the
|
||||
# plan's own reserve).
|
||||
|
||||
Executable
+201
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env python3
|
||||
"""probe-frank-archon-kv.py — verify the seqs x max-model-len KV contract on a
|
||||
frank-archon (radiance) service. Run it ON dankserver: it talks to
|
||||
127.0.0.1:7770 and builds its prompts from the repo checkout there.
|
||||
No scratch container, no mask: the service under test is the service.
|
||||
|
||||
ssh dankserver 'python3 /home/dank/serverAppConfigs/scripts/probe-frank-archon-kv.py'
|
||||
|
||||
Four probes, in increasing cost:
|
||||
A decode at depth — one ~51k-token chat completion, 512 tokens, greedy.
|
||||
Reports the engine's own `timings` (prefill, decode, draft acceptance).
|
||||
The bench law applies: >=128 generated tokens, greedy, real text.
|
||||
B the capacity contract — as many CONCURRENT chats of ~197k prompt tokens as
|
||||
the deployed `FRANK_ARCHON_MAX_SEQS` (read from the repo env file on this
|
||||
host), each carrying a unique marker at its head. All must answer their own
|
||||
marker. This is what the endpoint owes: N full-height conversations
|
||||
resident at once, exact retrieval in every one of them.
|
||||
C prefix-cache restore — probe A's cold prompt again, so the disk tier has
|
||||
to give the blocks back (cache_n > 0, TTFT collapses).
|
||||
D concurrency decode — `seqs` CONCURRENT 512-token generations at ~25 % of
|
||||
the window. This is the cost a slot count imposes on DECODE, which no
|
||||
prefill number shows: the fused-step schedule is unfair, so every stream is
|
||||
printed plus the aggregate. Measured 2026-10-13 at 3 x 200,000 fp8:
|
||||
single-stream 99.0 t/s (43.5k) and 93.3 (197k) — decode is nearly
|
||||
window-free — against 3-stream 113.9/43.3/10.9 t/s = 18.4 t/s aggregate.
|
||||
Concurrency, not the slab, is what decode pays for slots. The 2026-10-13
|
||||
shape hunt missed this for a whole day because leg B generates 24 tokens.
|
||||
PROBE LAW (learned the hard way, twice): radiance routes the thinking block to
|
||||
`reasoning_content`, so a chat request whose budget is spent inside
|
||||
`<think>` returns EMPTY `content` with `finish_reason: stop`. A probe that
|
||||
reads `content` alone scores 0/2 on a healthy server. Every request here sets
|
||||
chat_template_kwargs.enable_thinking=false and reads both fields.
|
||||
|
||||
A/B method for a shape change (no scratch container): write a variant env file
|
||||
on the host, point a transient systemd drop-in at it
|
||||
(`Environment=FRANK_ARCHON_ENV_FILE=...` — drop-ins are read AFTER the unit's
|
||||
own Environment= lines, so they win), restart, probe, delete the drop-in,
|
||||
daemon-reload, restart. `systemctl mask` is not available: the unit file in
|
||||
/etc/systemd/system is a relink symlink into the checkout.
|
||||
"""
|
||||
import json
|
||||
import glob
|
||||
import time
|
||||
import urllib.request
|
||||
import concurrent.futures
|
||||
|
||||
BASE = "http://127.0.0.1:7770"
|
||||
MODEL = "frank-archon"
|
||||
REPO = "/home/dank/serverAppConfigs"
|
||||
GLOBS = [f"{REPO}/scripts/*.sh", f"{REPO}/serverAppConfigs-docs/**/*.md"]
|
||||
|
||||
|
||||
def post(payload, timeout=1800):
|
||||
req = urllib.request.Request(
|
||||
BASE + "/v1/chat/completions",
|
||||
data=json.dumps(payload).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
t0 = time.time()
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
body = json.loads(r.read().decode())
|
||||
hdr = dict(r.headers.items())
|
||||
return body, time.time() - t0, hdr
|
||||
|
||||
|
||||
def get(path, timeout=60):
|
||||
with urllib.request.urlopen(BASE + path, timeout=timeout) as r:
|
||||
return json.loads(r.read().decode())
|
||||
|
||||
|
||||
def corpus(min_chars):
|
||||
files = []
|
||||
for g in GLOBS:
|
||||
files.extend(sorted(glob.glob(g, recursive=True)))
|
||||
out, total = [], 0
|
||||
for f in files:
|
||||
try:
|
||||
t = open(f, encoding="utf-8", errors="replace").read()
|
||||
except OSError:
|
||||
continue
|
||||
out.append(t)
|
||||
total += len(t)
|
||||
if total > min_chars:
|
||||
break
|
||||
return "".join(text for text in out)
|
||||
|
||||
|
||||
def prompt(text, chars, salt, fact):
|
||||
"""Real repo text, salted so the prefix cache cannot answer, marker planted
|
||||
near the head, question at the tail."""
|
||||
head = f"\n[SALT {salt}] confidential marker: {fact}\n"
|
||||
body = text + text
|
||||
p = (head + body * max(1, chars // len(body)))[:chars]
|
||||
return p + "\n\nQuestion: what is the confidential marker in the " \
|
||||
"SALT block near the top of this document? Answer with the marker " \
|
||||
"alone, no other words.\n"
|
||||
|
||||
|
||||
def chat(text, chars, salt, fact, max_tokens):
|
||||
return {"model": MODEL,
|
||||
"messages": [{"role": "user",
|
||||
"content": prompt(text, chars, salt, fact)}],
|
||||
"max_tokens": max_tokens, "temperature": 0.0,
|
||||
"chat_template_kwargs": {"enable_thinking": False}}
|
||||
|
||||
|
||||
def show(tag, j, wall, expect=None):
|
||||
tim, u = j.get("timings") or {}, j.get("usage", {})
|
||||
ch = (j.get("choices") or [{}])[0]
|
||||
msg = ch.get("message") or {}
|
||||
txt = (msg.get("content") or msg.get("reasoning_content") or "").strip()
|
||||
acc = (tim["draft_n_accepted"] / tim["draft_n"]) if tim.get("draft_n") else None
|
||||
line = (f"[{tag}] prompt={u.get('prompt_tokens')} gen={u.get('completion_tokens')} "
|
||||
f"wall={wall:.1f}s prefill={tim.get('prompt_per_second', 0):.0f} t/s "
|
||||
f"decode={tim.get('predicted_per_second', 0):.1f} t/s "
|
||||
f"draft={tim.get('draft_n')}/{tim.get('draft_n_accepted')} "
|
||||
f"accept={acc if acc is None else round(acc, 4)} "
|
||||
f"cache_n={tim.get('cache_n')} finish={ch.get('finish_reason')}")
|
||||
if expect:
|
||||
hit = expect.split("-")[0] in txt.lower()
|
||||
line += f" expected={expect} answered={txt[:80]!r} HIT={hit}"
|
||||
print(line)
|
||||
|
||||
|
||||
def deployed_shape():
|
||||
"""The contract is the deployed env, not a constant: slot count and window
|
||||
come from the checkout on this host, so the probe cannot drift from the
|
||||
service it is testing (the 2026-10-13 shape is 4 x 200,000; before that
|
||||
2 x 200,000, and the fp8 era 3 x 200,000)."""
|
||||
shape = {}
|
||||
with open(f"{REPO}/scripts/frank-archon.env") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if line.startswith("FRANK_ARCHON_") and "=" in line:
|
||||
key, _, val = line.partition("=")
|
||||
shape[key] = val.strip().strip('"')
|
||||
return (int(shape["FRANK_ARCHON_MAX_SEQS"]),
|
||||
int(shape["FRANK_ARCHON_MAX_MODEL_LEN"]),
|
||||
shape["FRANK_ARCHON_KV_DTYPE"])
|
||||
|
||||
|
||||
def main():
|
||||
seqs, maxlen, dtype = deployed_shape()
|
||||
print(f"shape: {seqs} x {maxlen:,} tokens, kv {dtype}")
|
||||
print("health:", json.dumps(get("/health")))
|
||||
print("models:", [m.get("id") for m in get("/v1/models").get("data", [])])
|
||||
deep = corpus(160_000)
|
||||
wide = corpus(1_100_000)
|
||||
|
||||
# 3.28 chars/token is the measured ratio of this corpus through the
|
||||
# froggeric template (646,000 chars -> 196,919 tokens at the 2026-10-11
|
||||
# deploy); 98.5 % of the window leaves the response room under the ceiling.
|
||||
chars = int(0.985 * maxlen * 3.28)
|
||||
# Each marker's FIRST hyphen-separated word is unique, because show()
|
||||
# scores the hit on that word — a shared prefix would score every slot HIT
|
||||
# on any answer that says the word.
|
||||
words = ["cobalt", "quartz", "basalt", "ember", "willow", "cobweb", "flint"]
|
||||
facts = {f"B{i+1}": f"{words[i % len(words)]}-flag-{(i + 1) * 137}"
|
||||
for i in range(seqs)}
|
||||
bodies = {s: chat(wide, chars, s, f, 24) for s, f in facts.items()}
|
||||
print(f"[B] {len(bodies['B1']['messages'][0]['content']):,} chars per "
|
||||
f"request, x{seqs} concurrent")
|
||||
|
||||
# A + C first (cheap), B last (the expensive set).
|
||||
for tag, salt in (("A-cold", "A1-9137"), ):
|
||||
j, wall, _ = post(chat(deep, 140_000, salt, "zephyr-lantern-7", 512))
|
||||
show(tag, j, wall, "zephyr-lantern-7")
|
||||
t0 = time.time()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=seqs) as ex:
|
||||
futs = {s: ex.submit(post, b) for s, b in bodies.items()}
|
||||
for s, fut in futs.items():
|
||||
try:
|
||||
j, wall, _ = fut.result()
|
||||
show(f"B-{s}", j, wall, facts[s])
|
||||
except Exception as e: # a preemption/timeout lands here
|
||||
print(f"[B-{s}] FAILED after {time.time() - t0:.1f}s: {e!r}")
|
||||
print(f"[B] total {time.time() - t0:.1f}s")
|
||||
# D — concurrency decode: seqs streams, 512 tokens each, ~25 % depth.
|
||||
dchars = int(0.25 * maxlen * 3.28)
|
||||
dbodies = [chat(deep, dchars, f"D{i + 1}-9137", f"flint-marker-{i}", 512)
|
||||
for i in range(seqs)]
|
||||
print(f"[D] {seqs} concurrent streams x 512 tokens, "
|
||||
f"{dchars:,} chars each")
|
||||
t0 = time.time()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=seqs) as ex:
|
||||
futs = [ex.submit(post, b) for b in dbodies]
|
||||
for i, fut in enumerate(futs, 1):
|
||||
try:
|
||||
j, wall, _ = fut.result()
|
||||
show(f"D-s{i}", j, wall)
|
||||
except Exception as e: # a preemption/timeout lands here
|
||||
print(f"[D-s{i}] FAILED after {time.time() - t0:.1f}s: {e!r}")
|
||||
tot = time.time() - t0
|
||||
print(f"[D] {seqs} x 512 tokens in {tot:.1f}s = "
|
||||
f"{seqs * 512 / tot:.1f} t/s aggregate decode")
|
||||
j, wall, _ = post(chat(deep, 140_000, "A1-9137", "zephyr-lantern-7", 512))
|
||||
show("C-prefix-hit", j, wall, "zephyr-lantern-7")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,460 @@
|
||||
{%- set template_version = "qwen3.8-froggeric-v22.5" %}
|
||||
{%- set _tool_format = tool_call_format if tool_call_format is defined else 'xml' %}
|
||||
{%- set image_count = namespace(value=0) %}
|
||||
{%- set video_count = namespace(value=0) %}
|
||||
{%- set add_vision_id = add_vision_id if add_vision_id is defined else false %}
|
||||
{%- set enable_thinking = enable_thinking if enable_thinking is defined else true %}
|
||||
{%- set auto_disable_thinking_with_tools = auto_disable_thinking_with_tools if auto_disable_thinking_with_tools is defined else false %}
|
||||
{%- if preserve_reasoning is defined and preserve_reasoning is not none %}
|
||||
{%- set _preserve_thinking = preserve_reasoning %}
|
||||
{%- elif preserve_thinking is defined and preserve_thinking is not none %}
|
||||
{%- set _preserve_thinking = preserve_thinking %}
|
||||
{%- else %}
|
||||
{%- set _preserve_thinking = true %}
|
||||
{%- endif %}
|
||||
{%- set max_tool_arg_chars = max_tool_arg_chars if max_tool_arg_chars is defined else 0 %}
|
||||
{%- set max_tool_response_chars = max_tool_response_chars if max_tool_response_chars is defined else 0 %}
|
||||
{%- set _default_reasoning_effort = 'medium' %}
|
||||
{%- set _has_tools = (tools is defined and tools and tools is iterable and tools is not mapping) %}
|
||||
{%- set _effort_raw = (reasoning_effort | string | lower) if reasoning_effort is defined and reasoning_effort is not none else _default_reasoning_effort %}
|
||||
{%- set _initial_thinking = enable_thinking %}
|
||||
{%- if _effort_raw in ('none', 'off') %}
|
||||
{%- set _initial_thinking = false %}
|
||||
{%- set _initial_effort = 'medium' %}
|
||||
{%- elif _effort_raw in ('minimal', 'low') %}
|
||||
{%- set _initial_effort = 'low' %}
|
||||
{%- elif _effort_raw in ('high', 'xhigh', 'max', 'ultracode', 'extreme') %}
|
||||
{%- set _initial_effort = 'xhigh' %}
|
||||
{%- else %}
|
||||
{%- set _initial_effort = 'medium' %}
|
||||
{%- endif %}
|
||||
{%- set ns_state = namespace(thinking=_initial_thinking, effort=_initial_effort) %}
|
||||
{%- if auto_disable_thinking_with_tools and _has_tools %}
|
||||
{%- set ns_state.thinking = false %}
|
||||
{%- endif %}
|
||||
{%- for msg in messages %}
|
||||
{%- if msg.role == 'system' or msg.role == 'developer' or msg.role == 'user' %}
|
||||
{%- if msg.content is string %}
|
||||
{%- if '<|think_off|>' in msg.content %}
|
||||
{%- set ns_state.thinking = false %}
|
||||
{%- elif '<|think_on|>' in msg.content %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- elif '<|think_xhigh|>' in msg.content or '<|think_high|>' in msg.content or '<|think_ultracode|>' in msg.content or '<|think_extreme|>' in msg.content or '<|think_max|>' in msg.content %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'xhigh' %}
|
||||
{%- elif '<|think_low|>' in msg.content or '<|think_minimal|>' in msg.content %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'low' %}
|
||||
{%- elif '<|think_medium|>' in msg.content %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'medium' %}
|
||||
{%- endif %}
|
||||
{%- elif msg.content is iterable and msg.content is not mapping %}
|
||||
{%- for item in msg.content %}
|
||||
{%- if item is string %}
|
||||
{%- set _item_text = item %}
|
||||
{%- elif item is mapping and 'text' in item and item.text is string %}
|
||||
{%- set _item_text = item.text %}
|
||||
{%- else %}
|
||||
{%- set _item_text = '' %}
|
||||
{%- endif %}
|
||||
{%- if _item_text %}
|
||||
{%- if '<|think_off|>' in _item_text %}
|
||||
{%- set ns_state.thinking = false %}
|
||||
{%- elif '<|think_on|>' in _item_text %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- elif '<|think_xhigh|>' in _item_text or '<|think_high|>' in _item_text or '<|think_ultracode|>' in _item_text or '<|think_extreme|>' in _item_text or '<|think_max|>' in _item_text %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'xhigh' %}
|
||||
{%- elif '<|think_low|>' in _item_text or '<|think_minimal|>' in _item_text %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'low' %}
|
||||
{%- elif '<|think_medium|>' in _item_text %}
|
||||
{%- set ns_state.thinking = true %}
|
||||
{%- set ns_state.effort = 'medium' %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- set reasoning_instructions = '' %}
|
||||
{%- if ns_state.thinking %}
|
||||
{%- if ns_state.effort == 'xhigh' %}
|
||||
{%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}
|
||||
{%- elif ns_state.effort == 'low' %}
|
||||
{%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- macro render_content(content, do_vision_count, is_system_content=false) %}
|
||||
{%- if content is string %}
|
||||
{{- content }}
|
||||
{%- elif content is iterable and content is not mapping %}
|
||||
{%- for item in content %}
|
||||
{%- if item is mapping %}
|
||||
{%- if item.type == 'image' or 'image' in item or 'image_url' in item %}
|
||||
{%- if is_system_content %}
|
||||
{{- raise_exception('System message cannot contain images.') }}
|
||||
{%- endif %}
|
||||
{%- if do_vision_count %}
|
||||
{%- set image_count.value = image_count.value + 1 %}
|
||||
{%- endif %}
|
||||
{%- if add_vision_id %}
|
||||
{{- 'Picture ' ~ image_count.value ~ ': ' }}
|
||||
{%- endif %}
|
||||
{{- '<|vision_start|><|image_pad|><|vision_end|>' }}
|
||||
{%- elif item.type == 'video' or 'video' in item or 'video_url' in item %}
|
||||
{%- if is_system_content %}
|
||||
{{- raise_exception('System message cannot contain videos.') }}
|
||||
{%- endif %}
|
||||
{%- if do_vision_count %}
|
||||
{%- set video_count.value = video_count.value + 1 %}
|
||||
{%- endif %}
|
||||
{%- if add_vision_id %}
|
||||
{{- 'Video ' ~ video_count.value ~ ': ' }}
|
||||
{%- endif %}
|
||||
{{- '<|vision_start|><|video_pad|><|vision_end|>' }}
|
||||
{%- elif 'text' in item %}
|
||||
{{- item.text }}
|
||||
{%- else %}
|
||||
{{- raise_exception('Unexpected item type in content.') }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- item | string }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- elif content is none or content is undefined %}
|
||||
{{- '' }}
|
||||
{%- else %}
|
||||
{{- raise_exception('Unexpected content type.') }}
|
||||
{%- endif %}
|
||||
{%- endmacro %}
|
||||
{%- if not messages %}
|
||||
{{- raise_exception('No messages provided.') }}
|
||||
{%- endif %}
|
||||
{%- set head = namespace(count=0, seen_non_system=false) %}
|
||||
{%- for message in messages %}
|
||||
{%- set _is_sys = (message.role == 'system' or message.role == 'developer') %}
|
||||
{%- if _is_sys and not head.seen_non_system %}
|
||||
{%- set head.count = head.count + 1 %}
|
||||
{%- else %}
|
||||
{%- set head.seen_non_system = true %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- set sys_state = namespace(content='') %}
|
||||
{%- for message in messages[:head.count] %}
|
||||
{%- set _part = render_content(message.content, false, true) | trim %}
|
||||
{%- if '<|think_off|>' in _part %}{%- set _part = _part.split('<|think_off|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_on|>' in _part %}{%- set _part = _part.split('<|think_on|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_xhigh|>' in _part %}{%- set _part = _part.split('<|think_xhigh|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_high|>' in _part %}{%- set _part = _part.split('<|think_high|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_ultracode|>' in _part %}{%- set _part = _part.split('<|think_ultracode|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_extreme|>' in _part %}{%- set _part = _part.split('<|think_extreme|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_max|>' in _part %}{%- set _part = _part.split('<|think_max|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_medium|>' in _part %}{%- set _part = _part.split('<|think_medium|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_low|>' in _part %}{%- set _part = _part.split('<|think_low|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_minimal|>' in _part %}{%- set _part = _part.split('<|think_minimal|>') | join('') | trim %}{%- endif %}
|
||||
{%- if _part %}
|
||||
{%- if sys_state.content %}
|
||||
{%- set sys_state.content = sys_state.content ~ '\n\n' ~ _part %}
|
||||
{%- else %}
|
||||
{%- set sys_state.content = _part %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- set _sc = sys_state.content %}
|
||||
{%- set _msgs = messages[head.count:] %}
|
||||
{%- if _has_tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if reasoning_instructions %}
|
||||
{{- reasoning_instructions + '\n\n' }}
|
||||
{%- endif %}
|
||||
{{- '# Tools\n\nYou have access to the following functions:\n\n<tools>' }}
|
||||
{%- for tool in tools %}
|
||||
{{- '\n' }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- '\n</tools>' }}
|
||||
{%- if _tool_format == 'json' %}
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n' }}
|
||||
{%- if ns_state.thinking %}
|
||||
{{- '<think>\nBrief explanation of tool call\n</think>\n' }}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n{"name": "example_function_name", "arguments": {"example_parameter_1": "value_1", "example_parameter_2": "This is the value for the second parameter"}}\n</tool_call>\n\n<IMPORTANT>\nReminder:\n' }}
|
||||
{%- if ns_state.thinking %}
|
||||
{{- '- You can use the <think></think> block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the <think></think> block.\n' }}
|
||||
{%- endif %}
|
||||
{{- '- Function calls MUST follow the specified format: a single JSON object with "name" and "arguments" keys inside <tool_call></tool_call> XML tags.\n' }}
|
||||
{%- if ns_state.thinking %}
|
||||
{{- '- If you choose to call a tool, you MUST output the <tool_call> block IMMEDIATELY after thinking, with NO conversational text before it.\n' }}
|
||||
{%- else %}
|
||||
{{- '- If you choose to call a tool, you MUST output the <tool_call> block IMMEDIATELY, with NO conversational text before it.\n' }}
|
||||
{%- endif %}
|
||||
{{- '- The <tool_call> tag MUST be at the very beginning of a new line, with NO spaces or indentation before it.\n- To call multiple functions, output a separate, completely closed <tool_call></tool_call> block for EACH function. Do NOT nest <tool_call> blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n</IMPORTANT>' }}
|
||||
{%- else %}
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n' }}
|
||||
{%- if ns_state.thinking %}
|
||||
{{- '<think>\nBrief explanation of tool call\n</think>\n' }}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n' }}
|
||||
{%- if ns_state.thinking %}
|
||||
{{- '- You can use the <think></think> block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the <think></think> block.\n' }}
|
||||
{%- endif %}
|
||||
{{- '- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags.\n' }}
|
||||
{%- if ns_state.thinking %}
|
||||
{{- '- If you choose to call a tool, you MUST output the <tool_call> block IMMEDIATELY after thinking, with NO conversational text before it.\n' }}
|
||||
{%- else %}
|
||||
{{- '- If you choose to call a tool, you MUST output the <tool_call> block IMMEDIATELY, with NO conversational text before it.\n' }}
|
||||
{%- endif %}
|
||||
{{- '- The <tool_call> and <function> tags MUST be at the very beginning of a new line, with NO spaces or indentation before them.\n- To call multiple functions, output a separate, completely closed <tool_call></tool_call> block for EACH function. Do NOT nest <tool_call> blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n</IMPORTANT>' }}
|
||||
{%- endif %}
|
||||
{%- if _sc %}
|
||||
{{- '\n\n' + _sc }}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- if _sc %}
|
||||
{{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '') + _sc + '<|im_end|>\n' }}
|
||||
{%- elif reasoning_instructions %}
|
||||
{{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set _last_idx = _msgs | length - 1 %}
|
||||
{%- set ns = namespace(multi_step_tool=true, last_query_index=_last_idx) %}
|
||||
{%- for message in _msgs[::-1] %}
|
||||
{%- set index = (_msgs | length - 1) - loop.index0 %}
|
||||
{%- if ns.multi_step_tool and message.role == 'user' %}
|
||||
{%- set _rc = render_content(message.content, false) | trim %}
|
||||
{%- if not (_rc.startswith('<tool_response>') and _rc.endswith('</tool_response>')) %}
|
||||
{%- set ns.multi_step_tool = false %}
|
||||
{%- set ns.last_query_index = index %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if ns.multi_step_tool %}
|
||||
{%- if _last_idx > 50 %}
|
||||
{%- set ns.last_query_index = _last_idx %}
|
||||
{%- else %}
|
||||
{%- set ns.last_query_index = 0 %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set ns2 = namespace(prev_role='', consecutive_failures=0) %}
|
||||
{%- for message in _msgs %}
|
||||
{%- set is_system = (message.role == "system" or message.role == "developer") %}
|
||||
{%- set content = render_content(message.content, true, is_system) | trim %}
|
||||
{%- if is_system or message.role == 'user' %}
|
||||
{%- if '<|think_off|>' in content %}{%- set content = content.split('<|think_off|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_on|>' in content %}{%- set content = content.split('<|think_on|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_xhigh|>' in content %}{%- set content = content.split('<|think_xhigh|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_high|>' in content %}{%- set content = content.split('<|think_high|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_ultracode|>' in content %}{%- set content = content.split('<|think_ultracode|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_extreme|>' in content %}{%- set content = content.split('<|think_extreme|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_max|>' in content %}{%- set content = content.split('<|think_max|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_medium|>' in content %}{%- set content = content.split('<|think_medium|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_low|>' in content %}{%- set content = content.split('<|think_low|>') | join('') | trim %}{%- endif %}
|
||||
{%- if '<|think_minimal|>' in content %}{%- set content = content.split('<|think_minimal|>') | join('') | trim %}{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if is_system %}
|
||||
{{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
|
||||
{%- elif message.role == 'user' %}
|
||||
{%- set ns2.consecutive_failures = 0 %}
|
||||
{{- '<|im_start|>user\n' + content + '<|im_end|>\n' }}
|
||||
{%- elif message.role == 'assistant' %}
|
||||
{%- set reasoning_content = '' %}
|
||||
{%- set _explicit_reasoning = '' %}
|
||||
{%- if message.reasoning_content is defined and message.reasoning_content is not none %}
|
||||
{%- if message.reasoning_content is string %}
|
||||
{%- set _explicit_reasoning = message.reasoning_content %}
|
||||
{%- else %}
|
||||
{%- set _explicit_reasoning = message.reasoning_content | string %}
|
||||
{%- endif %}
|
||||
{%- elif message.thinking is defined and message.thinking is not none %}
|
||||
{%- if message.thinking is string %}
|
||||
{%- set _explicit_reasoning = message.thinking %}
|
||||
{%- else %}
|
||||
{%- set _explicit_reasoning = message.thinking | string %}
|
||||
{%- endif %}
|
||||
{%- elif message.reasoning is defined and message.reasoning is not none %}
|
||||
{%- if message.reasoning is string %}
|
||||
{%- set _explicit_reasoning = message.reasoning %}
|
||||
{%- else %}
|
||||
{%- set _explicit_reasoning = message.reasoning | string %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if _explicit_reasoning %}
|
||||
{%- set _lead_end = '' %}
|
||||
{%- if content.startswith('<think>') and '</think>' in content %}
|
||||
{%- set _lead_end = '</think>' %}
|
||||
{%- elif content.startswith('<thinking>') and '</thinking>' in content %}
|
||||
{%- set _lead_end = '</thinking>' %}
|
||||
{%- elif content.startswith('</think>') %}
|
||||
{%- set _lead_end = '</think>' %}
|
||||
{%- elif content.startswith('</thinking>') %}
|
||||
{%- set _lead_end = '</thinking>' %}
|
||||
{%- endif %}
|
||||
{%- if _lead_end %}
|
||||
{%- set content = content.split(_lead_end)[-1].lstrip('\n') %}
|
||||
{%- endif %}
|
||||
{%- set reasoning_content = _explicit_reasoning %}
|
||||
{%- else %}
|
||||
{%- set _think_end = '' %}
|
||||
{%- if content.startswith('</think>') %}
|
||||
{%- set _think_end = '</think>' %}
|
||||
{%- elif content.startswith('</thinking>') %}
|
||||
{%- set _think_end = '</thinking>' %}
|
||||
{%- elif '\n</think>' in content %}
|
||||
{%- set _think_end = '\n</think>' %}
|
||||
{%- elif '\n</thinking>' in content %}
|
||||
{%- set _think_end = '\n</thinking>' %}
|
||||
{%- elif '\n</ think>' in content %}
|
||||
{%- set _think_end = '\n</ think>' %}
|
||||
{%- elif '\n</think >' in content %}
|
||||
{%- set _think_end = '\n</think >' %}
|
||||
{%- elif content.startswith('<think>') and '</think>' in content %}
|
||||
{%- set _think_end = '</think>' %}
|
||||
{%- elif content.startswith('<thinking>') and '</thinking>' in content %}
|
||||
{%- set _think_end = '</thinking>' %}
|
||||
{%- endif %}
|
||||
{%- if _think_end %}
|
||||
{%- if 'thinking' in _think_end %}
|
||||
{%- set _think_start = '<thinking>' %}
|
||||
{%- else %}
|
||||
{%- set _think_start = '<think>' %}
|
||||
{%- endif %}
|
||||
{%- set reasoning_content = content.split(_think_end)[0].rstrip('\n') %}
|
||||
{%- if _think_start in reasoning_content %}
|
||||
{%- set reasoning_content = reasoning_content.split(_think_start)[-1].lstrip('\n') %}
|
||||
{%- endif %}
|
||||
{%- set content = content.split(_think_end)[-1].lstrip('\n') %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set reasoning_content = reasoning_content | trim %}
|
||||
{%- if (_preserve_thinking or loop.index0 > ns.last_query_index) %}
|
||||
{{- '<|im_start|>assistant\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n' + content }}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls is defined and message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined and tool_call.function is not none %}
|
||||
{%- set tc = tool_call.function %}
|
||||
{%- else %}
|
||||
{%- set tc = tool_call %}
|
||||
{%- endif %}
|
||||
{%- set tc_name = tc.name if (tc.name is defined and tc.name is not none) else '' %}
|
||||
{%- if _tool_format == 'json' %}
|
||||
{%- if loop.first %}
|
||||
{%- if content | trim %}
|
||||
{{- '\n\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '\n' }}
|
||||
{%- endif %}
|
||||
{%- set _args = '{}' %}
|
||||
{%- if tc.arguments is defined and tc.arguments is not none %}
|
||||
{%- if tc.arguments is mapping %}
|
||||
{%- set _args = tc.arguments | tojson %}
|
||||
{%- elif tc.arguments is string %}
|
||||
{%- if tc.arguments %}
|
||||
{%- set _args = tc.arguments %}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{%- set _args = tc.arguments | tojson %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n{"name": ' }}{{- tc_name | tojson }}{{- ', "arguments": ' }}{{- _args }}{{- '}\n</tool_call>' }}
|
||||
{%- else %}
|
||||
{%- if loop.first %}
|
||||
{%- if content | trim %}
|
||||
{{- '\n\n<tool_call>\n<function=' + tc_name + '>\n' }}
|
||||
{%- else %}
|
||||
{{- '<tool_call>\n<function=' + tc_name + '>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '\n<tool_call>\n<function=' + tc_name + '>\n' }}
|
||||
{%- endif %}
|
||||
{%- if tc.arguments is defined and tc.arguments is not none %}
|
||||
{%- if tc.arguments is mapping %}
|
||||
{%- for args_name, args_value in tc.arguments.items() %}
|
||||
{{- '<parameter=' + args_name + '>\n' }}
|
||||
{%- if args_value is string %}
|
||||
{%- set _av = args_value %}
|
||||
{%- else %}
|
||||
{%- set _av = args_value | tojson %}
|
||||
{%- endif %}
|
||||
{%- if max_tool_arg_chars > 0 and _av | length > max_tool_arg_chars %}
|
||||
{{- _av[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_av | length | string) ~ ' chars]' }}
|
||||
{%- else %}
|
||||
{{- _av }}
|
||||
{%- endif %}
|
||||
{{- '\n</parameter>\n' }}
|
||||
{%- endfor %}
|
||||
{%- else %}
|
||||
{%- if tc.arguments is string %}
|
||||
{%- set _raw_args = tc.arguments %}
|
||||
{%- else %}
|
||||
{%- set _raw_args = tc.arguments | tojson %}
|
||||
{%- endif %}
|
||||
{%- if _raw_args %}
|
||||
{%- if max_tool_arg_chars > 0 and _raw_args | length > max_tool_arg_chars %}
|
||||
{{- _raw_args[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_raw_args | length | string) ~ ' chars]' }}
|
||||
{%- else %}
|
||||
{{- _raw_args }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{{- '</function>\n</tool_call>' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == 'tool' %}
|
||||
{%- set _content_lower = content | lower %}
|
||||
{%- set _content_head = _content_lower[:120] %}
|
||||
{%- set _is_code_or_grep = ('throw new ' in _content_lower or 'throw error' in _content_lower or 'console.error' in _content_lower or 'logger.error' in _content_lower or 'logging.error' in _content_lower or 'import ' in _content_head or 'def ' in _content_head or 'function ' in _content_head) %}
|
||||
{%- set _exit_code_zero = ('exit code: 0' in _content_head or 'process exited with code 0' in _content_head) %}
|
||||
{%- set _error_field_ok = ('"error": null' in _content_head or '"error":null' in _content_head or '"error": false' in _content_head or '"error":false' in _content_head or '"error": ""' in _content_head or '"error":""' in _content_head) %}
|
||||
{%- set _strong_error = (('"error":' in _content_head and not _error_field_ok) or '"status": "error"' in _content_head or '"status":"error"' in _content_head or 'traceback (most recent call last):' in _content_head or 'command not found' in _content_head or 'invalid syntax' in _content_head or 'fatal:' in _content_head or (('exit code: ' in _content_head or 'process exited with code' in _content_head) and not _exit_code_zero) or _content_head.startswith('exception:') or _content_head.startswith('failed to ')) %}
|
||||
{%- set _weak_error = ('error:' in _content_head or 'err!' in _content_head) %}
|
||||
{%- set _weak_suppressed = ('$ ' in _content_head or 'took ' in _content_head or content | length >= 600) %}
|
||||
{%- if not _is_code_or_grep and (_strong_error or (_weak_error and not _weak_suppressed)) %}
|
||||
{%- set ns2.consecutive_failures = ns2.consecutive_failures + 1 %}
|
||||
{%- else %}
|
||||
{%- set ns2.consecutive_failures = 0 %}
|
||||
{%- endif %}
|
||||
{%- if ns2.prev_role != 'tool' %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{%- set _is_json_payload = (_tool_format == 'json' and content | trim | length > 0 and (content | trim)[:1] in ('{', '[')) %}
|
||||
{%- if not _is_json_payload and max_tool_response_chars > 0 and content | length > max_tool_response_chars %}
|
||||
{%- set content = content[:max_tool_response_chars] + '\n[TRUNCATED - original length ' ~ (content | length | string) ~ ' chars]' %}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' + content }}
|
||||
{%- if ns2.consecutive_failures >= 2 %}
|
||||
{{- '\n\n⚠️ SYSTEM WARNING: ' ~ ns2.consecutive_failures ~ ' consecutive tool errors detected. Your previous approach is incorrect. You MUST use a fundamentally different approach or corrected arguments.' }}
|
||||
{%- elif ns2.consecutive_failures == 1 %}
|
||||
{{- '\n\n⚠️ SYSTEM WARNING: The previous tool call returned an error. Diagnose the failure and retry with completely corrected arguments.' }}
|
||||
{%- endif %}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- set _next_role = _msgs[loop.index0 + 1].role %}
|
||||
{%- if _next_role != 'tool' %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>user\n[' + message.role + ']: ' + content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- set ns2.prev_role = message.role %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- if not ns_state.thinking %}
|
||||
{{- '<think>\n\n</think>\n\n' }}
|
||||
{%- else %}
|
||||
{{- '<think>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
@@ -0,0 +1,105 @@
|
||||
# qwen4exp-swift15-w4dn8-i8-hc8m.recipe -- the Swift 1.5 finetune of Qwen3.8-Flash-Next
|
||||
# (ukisai/Swift1.5-Qwen3.8-Flash-Next, BF16 source, 335.3 GiB, key set identical to the official
|
||||
# base) as a radiance container for frank-archon: EVERY routed expert -- gate_up AND down --
|
||||
# through GPTQ into the w4nl64a8h table. The plugin's one-table-one-encoding law
|
||||
# ("one table of experts takes one encoding, and only a whole expert kept plain bf16 is served
|
||||
# beside it", qwen4exp_fp8 rad_arch_declare) forbids the original split tier (down block-fp8 +
|
||||
# gate_up w4): the served kernel is w4nl64a8h for the whole expert set, bf16 experts beside it.
|
||||
# The rest is qwen4exp-w5dn64-i8-hc8m.recipe verbatim: int8 trunk linears, block-fp8 hc mixers
|
||||
# and MTP fc, the fixed-scale n-gram table, the 2-bit draft head.
|
||||
#
|
||||
# Why down is the calibrated half: the calibration tap rides the down projection, so its GPTQ
|
||||
# rounding propagates against real down-input Hessians; gate_up shares the same w4 table
|
||||
# uncalibrated. The published w5dn64 comparison (mean KL 0.0789 -> 0.0717, top-1 92.85% ->
|
||||
# 93.34%, decode +4%, prefill -19%) is the 5-bit upper bound; this tier trades part of that
|
||||
# quality back for prefill.
|
||||
#
|
||||
# CALIB=DIR rad-convert <Swift1.5-Qwen3.8-Flash-Next checkpoint> \
|
||||
# --recipe scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe -o qwen4exp-swift15-w4dn8-i8-hc8m.rad
|
||||
#
|
||||
# DIR is the calibration directory the bf16 container wrote (RADIANCE_CALIB_DIR, 10M tokens of
|
||||
# traffic-shaped corpus). Everything not named here is served as the checkpoint holds it (bf16:
|
||||
# embeddings, the vision tower, the QSA indexer); the first rule that matches a weight decides it.
|
||||
# The bf16 container is NOT passed with --reuse: the peak-disk arithmetic does not fit (checkpoint +
|
||||
# container + container + output > purge-freed space), and --reuse only saves a re-read.
|
||||
|
||||
# The protected experts, cast to bf16 ahead of both expert rules. THE NAMES BELOW ARE THE BASE
|
||||
# CHECKPOINT'S (from its 1M-token recording, qwen4exp-w5dn64-i8-hc8m.recipe) -- a warm start only.
|
||||
# The Swift 1.5 finetune's routing distribution is not assumed equal: the derivation step rewrites
|
||||
# this section from Swift's OWN calibration recording before this recipe is ever run, by the base's
|
||||
# law -- an expert whose share of its layer's down-input energy is at least 3% and at least three
|
||||
# times its share of the routing, an even count on each rank's half of the layer (the four-bit GEMM
|
||||
# takes the rest as two equal tables by parity) by adding that half's next most energetic expert.
|
||||
# PROTECTED-BEGIN (rewritten from Swift's OWN grams by derive-protected-experts.py)
|
||||
blk.1.ffn_*_exps.271.weight cast dtype=bf16
|
||||
blk.1.ffn_*_exps.355.weight cast dtype=bf16
|
||||
blk.2.ffn_*_exps.279.weight cast dtype=bf16
|
||||
blk.2.ffn_*_exps.413.weight cast dtype=bf16
|
||||
blk.34.ffn_*_exps.146.weight cast dtype=bf16
|
||||
blk.34.ffn_*_exps.153.weight cast dtype=bf16
|
||||
blk.34.ffn_*_exps.407.weight cast dtype=bf16
|
||||
blk.34.ffn_*_exps.453.weight cast dtype=bf16
|
||||
blk.37.ffn_*_exps.288.weight cast dtype=bf16
|
||||
blk.37.ffn_*_exps.377.weight cast dtype=bf16
|
||||
blk.38.ffn_*_exps.406.weight cast dtype=bf16
|
||||
blk.38.ffn_*_exps.420.weight cast dtype=bf16
|
||||
blk.42.ffn_*_exps.37.weight cast dtype=bf16
|
||||
blk.42.ffn_*_exps.90.weight cast dtype=bf16
|
||||
blk.42.ffn_*_exps.302.weight cast dtype=bf16
|
||||
blk.42.ffn_*_exps.388.weight cast dtype=bf16
|
||||
blk.44.ffn_*_exps.315.weight cast dtype=bf16
|
||||
blk.44.ffn_*_exps.350.weight cast dtype=bf16
|
||||
blk.46.ffn_*_exps.290.weight cast dtype=bf16
|
||||
blk.46.ffn_*_exps.419.weight cast dtype=bf16
|
||||
blk.47.ffn_*_exps.119.weight cast dtype=bf16
|
||||
blk.47.ffn_*_exps.143.weight cast dtype=bf16
|
||||
blk.47.ffn_*_exps.399.weight cast dtype=bf16
|
||||
blk.47.ffn_*_exps.445.weight cast dtype=bf16
|
||||
# PROTECTED-END
|
||||
# (derive-protected-experts.py rewrote this section from Swift's own grams)
|
||||
|
||||
# Every other routed expert, trunk and MTP layer alike: u4 codes into the w4nl table, rotated by
|
||||
# the 128-wide Hadamard, an E4M3 scale a group of 64 under the fixed 2^-13 searched for least
|
||||
# error as stored, chosen by GPTQ against $CALIB and swept three times by coordinate descent.
|
||||
# The calibration Grams are down-input Hessians, so down is the calibrated half of the table;
|
||||
# gate_up has no recording of its own and quantises uncalibrated (said on stderr, as the base
|
||||
# recipe says). The MTP layer has no recording either. Expert mass: gate_up 150.00 GiB -> ~37.6,
|
||||
# down 75.00 GiB -> ~18.8: ~56 GiB of experts, ~28 GiB a rank at TP=2 -- well inside the pool.
|
||||
blk.*.ffn_*_exps.*.weight gptq table=w4nl group=64 scale=fp8_e4m3 scale2=f32 block2=*x* scale2_value=0.0001220703125 transform=fwht128 rule=search cd=3 calib=$CALIB
|
||||
|
||||
# Every connection's two mixing matrices -- the 96 trunk connections, the final mixer and the MTP
|
||||
# head's three -- as E4M3 rows with an f32 scale a group of each row. A 128x128 block cannot cover
|
||||
# them: lowrank is 320, 2.5 blocks. Down is [320, 10240] and takes a scale every 128 columns; up is
|
||||
# [10240, 320] and takes four scales a row, 80 columns each.
|
||||
*_hc_down.weight rtn codes=fp8_e4m3 group=128 scale=f32
|
||||
*_hc_up.weight rtn codes=fp8_e4m3 group=80 scale=f32
|
||||
|
||||
# The MTP head's fc_hidden and fc_embedding, the same rows. They only ever change the drafts.
|
||||
mtp.fc_hidden.weight rtn codes=fp8_e4m3 group=128 scale=f32
|
||||
mtp.fc_embedding.weight rtn codes=fp8_e4m3 group=128 scale=f32
|
||||
|
||||
# The trunk's block linears as int8, a scale per 128 columns of a row searched for least error.
|
||||
# The QSA indexer's projection is not one of them: its top-k is a hard choice between blocks, and
|
||||
# keeping it as the checkpoint holds it costs 20 MB.
|
||||
blk.*.attn_qg.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
blk.*.attn_k.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
blk.*.attn_v.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
blk.*.attn_output.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
blk.*.ssm_inz.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
blk.*.ssm_out.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
blk.*.ffn_gate_up_shexp.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
blk.*.ffn_down_shexp.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
|
||||
# The lm_head, int8 with a bf16 scale a row per 128 columns (logits_gemm_i8): at E4M3 it costs
|
||||
# ~0.6 points of top-1 on its own.
|
||||
output.weight rtn codes=i8 group=128 scale=bf16 clamp=sym rule=search
|
||||
|
||||
# The n-gram table at one fixed E4M3 scale; qwen4exp-w4.recipe says where the value comes from.
|
||||
# The value sits on a three-octave plateau (2^-13..2^-10, rel_l2 0.02662) with 1.7x headroom over
|
||||
# the base table's absmax; the prep phase verifies Swift's PLE absmax lands inside the plateau
|
||||
# before this rule is trusted.
|
||||
blk.*.ple_ngram.weight rtn codes=fp8_e4m3 block=*x* scale=bf16 rule=fixed scale_value=1.99317932128906e-4
|
||||
|
||||
# The MTP head's 2-bit draft head. INERT for this checkpoint: the HF weight map has no
|
||||
# mtp.draft_head.weight (same as the base), kept for shape parity with the family.
|
||||
mtp.draft_head.weight rtn codes=u2 zero=u8 group=128 scale=f16
|
||||
Executable
+116
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env bash
|
||||
# run-swift15-bf16.sh — RUNS ON DANKSERVER. Boot the no-recipe bf16 Swift1.5
|
||||
# container for the two offline campaign phases. The engine owns the GPUs, so
|
||||
# the production unit must be stopped first (this script does it and restores
|
||||
# on exit, autoresearch pattern: stop, never mask — relinked unit).
|
||||
#
|
||||
# kld --kld-record the reference logits from the bf16 model
|
||||
# (runs the corpus and EXITS; engine writes DIR and leaves)
|
||||
# calib serve with the calibration tap; a separate driver
|
||||
# (scripts/calib-push.py) pushes the corpus; the engine writes the
|
||||
# Hessians at RADIANCE_CALIB_TOKENS and the tap switches itself off.
|
||||
# Stop this script after calib-push prints TAP_DONE.
|
||||
#
|
||||
# Shape = the vendor's bf16 calibration lines (GUIDE.md §4.3:528 + MOE-W4.md
|
||||
# "calibrate the checkpoint as it ships"): expert_tiered + weights-disk-tier,
|
||||
# no speculation, deterministic. Large prefill steps because a disk-tier step
|
||||
# costs the same at 1 token as at 32K. The container declared --max-ctx 4096;
|
||||
# --max-model-len 16384 exceeds it on purpose (the KLD prompts run to 12,007
|
||||
# tokens, calib to 8,242) — the boot ledger arbitrates the QSA buffers; if
|
||||
# declare refuses, re-convert with a wider --max-ctx.
|
||||
set -uo pipefail
|
||||
|
||||
MODE="${1:?usage: run-swift15-bf16.sh kld|calib}"
|
||||
PIN_1013=sha256:4c3f484564bb2872784e46b0a59cbd71becc9c1e746e301a3596ffae253c322e
|
||||
PIN_120=sha256:665775ed1d90de6dbb25558e07fc60d20328f1346f96f3c6e00845eb763f01fa
|
||||
RAD=/home/dank/models/rad/qwen4exp-swift15-bf16.rad
|
||||
C=swift15ref
|
||||
LOG=/home/dank/logs/swift15_bf16.log
|
||||
KLD_REF_DIR=/home/dank/corpora/kld-ref
|
||||
CALIB_DIR=/home/dank/corpora/calib
|
||||
KLD_CORPUS=/home/dank/corpora/out/kld.jsonl
|
||||
|
||||
restore() { sudo systemctl start frank-archon >/dev/null 2>&1 || true; }
|
||||
trap restore EXIT
|
||||
sudo systemctl stop frank-archon >/dev/null 2>&1 || true
|
||||
podman rm -f "$C" >/dev/null 2>&1 || true
|
||||
|
||||
ARGS=(--model /models/rad/qwen4exp-swift15-bf16.rad
|
||||
--tp 2 --placement expert_tiered --weights-disk-tier
|
||||
--host-pool-mib 12288 --deterministic --gpu-headroom-mib 768
|
||||
--num-speculative-tokens 0
|
||||
--kv-cache-dtype fp8 --max-model-len 16384
|
||||
--ngram-placement ram
|
||||
--override-chat-template /models/tpl.jinja)
|
||||
|
||||
case "$MODE" in
|
||||
kld)
|
||||
# 1.0.13: the engine that will serve the candidate — the reference is
|
||||
# recorded on the same engine that scores it. The weights slab DERIVES as
|
||||
# card - headroom - kv, and the disk tier refuses under 9.57 GiB (4.21 of
|
||||
# layer buffers + 5.36 fixed), so kv is explicit SMALL: recording is
|
||||
# prefill-only, self-batched at MNBT 32768 = two 16k prompts at a time;
|
||||
# 4 slots x 16k fp8 (29 KiB/pos, 1.7 GiB) + headroom fits 3072.
|
||||
IMAGE="stilldeadcode/radiance@$PIN_1013"
|
||||
mkdir -p "$KLD_REF_DIR"
|
||||
ARGS+=(--kld-record /data/kld-ref --kld-corpus /data/kld.jsonl
|
||||
--vram-kv-mib 3072
|
||||
--max-num-seqs 4 --max-num-batched-tokens 32768)
|
||||
;;
|
||||
calib)
|
||||
# 1.2.0: the calibration tap (RADIANCE_CALIB_DIR + RADIANCE_CALIB_TOKENS)
|
||||
# lives in the 1.2.0 qwen4exp plugin; 1.0.13 has only the DIR half with an
|
||||
# unparameterisable threshold. Grams are bf16 activations — engine-invariant.
|
||||
# Same KV ceiling as kld: the disk-tier weights floor (9.57 GiB) plus
|
||||
# ~18 GiB of fixed per-rank claims cap the pool near 3.5 GiB on a 32 GiB
|
||||
# card — pool minus the 2.17 GiB fixed holds ~237k fp8 tokens a rank, so
|
||||
# the vendor's big-step calibration shape is affordable: MOE-W4.md:244
|
||||
# measures 640 tok/s at a 32k step against 930 at 55k, and the doc's own
|
||||
# run drives MNBT 65536. 8 slots x 16k = 131k resident tokens fits the
|
||||
# pool; the step batches 65k. KLD's observed rate confirmed step size,
|
||||
# not slot count, is the throughput lever.
|
||||
IMAGE="stilldeadcode/radiance@$PIN_120"
|
||||
mkdir -p "$CALIB_DIR"
|
||||
ARGS+=(--port 7770 --vram-kv-mib 3072
|
||||
--max-num-seqs 8 --max-num-batched-tokens 65536
|
||||
--prefix-cache-dir /data/calib-kvcache)
|
||||
;;
|
||||
probe)
|
||||
IMAGE="stilldeadcode/radiance@$PIN_1013"
|
||||
ARGS+=(--port 7770 --max-num-seqs 8 --max-num-batched-tokens 4096)
|
||||
;;
|
||||
*) echo "unknown mode: $MODE" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
# The prod template copy, readable from the container: mount the models tree
|
||||
# ro plus the state dir rw (KLD record + prefix tier land there).
|
||||
cp -f /home/dank/serverAppConfigs/scripts/qwen38-froggeric-v22.5.jinja \
|
||||
/home/dank/models/tpl.jinja 2>/dev/null || true
|
||||
|
||||
KLD_MOUNT=()
|
||||
if [ "$MODE" = kld ]; then
|
||||
# Nested under the /data mount, so these must come after it (line order
|
||||
# in podman run below): the corpus read-only, the record dir writable.
|
||||
KLD_MOUNT=(-v "$KLD_CORPUS:/data/kld.jsonl:ro"
|
||||
-v "$KLD_REF_DIR:/data/kld-ref")
|
||||
fi
|
||||
EXTRA_ENV=()
|
||||
if [ "$MODE" = calib ]; then
|
||||
EXTRA_ENV=(-v "$CALIB_DIR:/data/calib"
|
||||
-e RADIANCE_CALIB_DIR=/data/calib
|
||||
-e RADIANCE_CALIB_TOKENS=10000000)
|
||||
fi
|
||||
|
||||
podman run --rm --name "$C" \
|
||||
--network host --init \
|
||||
--security-opt seccomp=unconfined \
|
||||
--ulimit memlock=-1:-1 \
|
||||
--device /dev/kfd --device /dev/dri \
|
||||
-v /home/dank/models:/models:ro \
|
||||
-v /home/dank/radiance-state:/data \
|
||||
"${KLD_MOUNT[@]}" \
|
||||
"${EXTRA_ENV[@]}" \
|
||||
"$IMAGE" "${ARGS[@]}" 2>&1 | tee "$LOG"
|
||||
rc=${PIPESTATUS[0]}
|
||||
echo "swift15-bf16 $MODE exited rc=$rc; log: $LOG"
|
||||
exit "$rc"
|
||||
Executable
+196
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env bash
|
||||
# serve-frank-archon.sh — frank-archon endpoint: radiance (StillDeadcode,
|
||||
# Apache-2.0) serving the OFFICIAL Qwen/Qwen3.8-Flash-Next-FP8 checkpoint at
|
||||
# NATIVE block-FP8 precision from a rad-convert container
|
||||
# (/home/dank/models/rad/qwen4exp-fp8-native.rad), on BOTH R9700s (TP=2,
|
||||
# wht6 wire), dankserver :7770. 2026-10-05 cutover from FreeToken (PR #587,
|
||||
# one card; git tag frank-archon-freetoken-pr587-20261005 is the
|
||||
# pre-cutover state).
|
||||
#
|
||||
# The run line below IS the engine argv: the image's entrypoint is the
|
||||
# engine and its tools sit beside it on PATH. Unlike every predecessor
|
||||
# there is no per-boot VRAM planner to second-guess: the boot ledger in the
|
||||
# log states every budget before anything is allocated (weights vs KV
|
||||
# split, the expert host pool's coverage of the worst case, the PLE pin,
|
||||
# the P2P answer per pair) — read it after every resize.
|
||||
# Deployed shape (2026-10-13): THREE decode slots x 200,000 tokens, fp8
|
||||
# (quantized) KV in a 5,120 MiB pool (contract 3 x 200,000 + 1.3 %) and a
|
||||
# 25,600 MiB resident expert slab — 30,720 MiB of the 30,969 MiB the engine's
|
||||
# own budget table makes claimable, spent on the slab because prefill is
|
||||
# slab-limited. 44 GiB/rank pinned expert pool, MTP depth 2, prefix cache host
|
||||
# 8 GiB + disk 64 GiB, PLE pinned in RAM.
|
||||
#
|
||||
# The root unit (scripts/frank-archon.service, User=dank,
|
||||
# LimitMEMLOCK=infinity) grants the pin budget: 2 x 44 GiB expert pools +
|
||||
# the 47.7 GiB PLE table + 8 GiB prefix host tier — ~147 GiB of pinned
|
||||
# pages, which is ALSO why this script refuses to start when ttm's
|
||||
# pages_limit/pages_max cap is under FRANK_ARCHON_TTM_MIN_MIB (see below).
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_ARCHON_ENV_FILE:-$SCRIPT_DIR/frank-archon.env}"
|
||||
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
echo "ERROR: frank-archon env file absent: $ENV_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
|
||||
: "${FRANK_ARCHON_IMAGE:?FRANK_ARCHON_IMAGE must name the radiance image}"
|
||||
: "${FRANK_ARCHON_MODEL_DIR:?FRANK_ARCHON_MODEL_DIR must name the .rad dir}"
|
||||
: "${FRANK_ARCHON_MODEL_FILE:?FRANK_ARCHON_MODEL_FILE must name the .rad file}"
|
||||
: "${FRANK_ARCHON_STATE_DIR:?FRANK_ARCHON_STATE_DIR must name the /data state dir}"
|
||||
: "${FRANK_ARCHON_CONTAINER:?FRANK_ARCHON_CONTAINER must name the container}"
|
||||
: "${FRANK_ARCHON_SERVED_NAME:?FRANK_ARCHON_SERVED_NAME must name the alias}"
|
||||
: "${FRANK_ARCHON_PORT:?FRANK_ARCHON_PORT must be the port}"
|
||||
: "${FRANK_ARCHON_TEMPLATE:?FRANK_ARCHON_TEMPLATE must name the chat template}"
|
||||
: "${FRANK_ARCHON_TP:?FRANK_ARCHON_TP must be set}"
|
||||
: "${FRANK_ARCHON_TP_WIRE:?FRANK_ARCHON_TP_WIRE must be set}"
|
||||
: "${FRANK_ARCHON_MAX_SEQS:?FRANK_ARCHON_MAX_SEQS must be set}"
|
||||
: "${FRANK_ARCHON_MAX_MODEL_LEN:?FRANK_ARCHON_MAX_MODEL_LEN must be set}"
|
||||
: "${FRANK_ARCHON_PLACEMENT:?FRANK_ARCHON_PLACEMENT must be set}"
|
||||
: "${FRANK_ARCHON_HOST_POOL_MIB:?FRANK_ARCHON_HOST_POOL_MIB must be set}"
|
||||
: "${FRANK_ARCHON_GPU_HEADROOM_MIB:?FRANK_ARCHON_GPU_HEADROOM_MIB must be set}"
|
||||
: "${FRANK_ARCHON_VRAM_WEIGHTS_MIB:?FRANK_ARCHON_VRAM_WEIGHTS_MIB must be set}"
|
||||
: "${FRANK_ARCHON_VRAM_KV_MIB:?FRANK_ARCHON_VRAM_KV_MIB must be set}"
|
||||
: "${FRANK_ARCHON_KV_DTYPE:?FRANK_ARCHON_KV_DTYPE must be set}"
|
||||
: "${FRANK_ARCHON_SPEC_TOKENS:?FRANK_ARCHON_SPEC_TOKENS must be set}"
|
||||
: "${FRANK_ARCHON_MAX_BATCHED_TOKENS:?FRANK_ARCHON_MAX_BATCHED_TOKENS must be set}"
|
||||
: "${FRANK_ARCHON_PREFIX_HOST_MIB:?FRANK_ARCHON_PREFIX_HOST_MIB must be set}"
|
||||
: "${FRANK_ARCHON_PREFIX_DISK_MIB:?FRANK_ARCHON_PREFIX_DISK_MIB must be set}"
|
||||
: "${FRANK_ARCHON_NGRAM_PLACEMENT:?FRANK_ARCHON_NGRAM_PLACEMENT must be set}"
|
||||
: "${FRANK_ARCHON_TTM_MIN_MIB:?FRANK_ARCHON_TTM_MIN_MIB must be set}"
|
||||
: "${FRANK_ARCHON_TTM_TARGET_MIB:?FRANK_ARCHON_TTM_TARGET_MIB must be set}"
|
||||
: "${FRANK_ARCHON_RAD_MIN_MIB:?FRANK_ARCHON_RAD_MIN_MIB must be set}"
|
||||
|
||||
# The image gate is the digest-pinned ref itself: a name@sha256 ref exists
|
||||
# in storage only if THAT manifest was pulled — a retagged look-alike or a
|
||||
# newer latest does not satisfy it. `podman image exists` takes a
|
||||
# name@digest ref in one command — no pipefail/grep race (the 2026-09-30
|
||||
# lesson). Runs as dank: the unit's process uses dank's ROOTLESS podman
|
||||
# storage (root unit, User=dank — the fleet's frank-archon shape).
|
||||
if ! podman image exists "$FRANK_ARCHON_IMAGE"; then
|
||||
echo "ERROR: radiance image absent: $FRANK_ARCHON_IMAGE" >&2
|
||||
echo "frank-archon: podman pull docker://stilldeadcode/radiance:latest (as dank)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The payload gate: the .rad container file, sized. A truncated convert
|
||||
# would otherwise buy a guaranteed long-timeout boot instead of this fast
|
||||
# failure. The staging dir / raw checkpoint are NOT this script's concern
|
||||
# (scripts/build-frank-archon-rad.sh built the file; its rad-info output
|
||||
# is the record).
|
||||
RAD="$FRANK_ARCHON_MODEL_DIR/$FRANK_ARCHON_MODEL_FILE"
|
||||
if [ ! -f "$RAD" ]; then
|
||||
echo "ERROR: .rad container missing: $RAD" >&2
|
||||
echo "frank-archon: run scripts/build-frank-archon-rad.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
RAD_MIB=$(( $(stat -c %s "$RAD") / 1048576 ))
|
||||
if [ "$RAD_MIB" -lt "$FRANK_ARCHON_RAD_MIN_MIB" ]; then
|
||||
echo "ERROR: .rad container too small: $RAD is ${RAD_MIB} MiB, want >= $FRANK_ARCHON_RAD_MIN_MIB" >&2
|
||||
echo "frank-archon: the convert was truncated — rebuild it" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$FRANK_ARCHON_TEMPLATE" ]; then
|
||||
echo "ERROR: chat template missing: $FRANK_ARCHON_TEMPLATE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$FRANK_ARCHON_STATE_DIR/kvcache/frank-archon"
|
||||
# The engine's create is NOT recursive: with only the state dir mounted, its
|
||||
# "cannot create /data/kvcache/frank-archon: No such file or directory"
|
||||
# left the SSD prefix tier off (measured 2026-10-06 boot ledger). Pre-create
|
||||
# the full path on the host so the tier opens on first boot.
|
||||
|
||||
mkdir -p "$FRANK_ARCHON_STATE_DIR/tune"
|
||||
# rad-tune's kernel-selection cache (see the runbook's "Tuning levers"):
|
||||
# the engine's selector reads $RADIANCE_HOME/tune/<machine>.tune at boot;
|
||||
# $RADIANCE_HOME defaults to /opt/radiance, which lives in the image, so
|
||||
# the cache must ride a host mount or it dies with the --rm container
|
||||
# that wrote it. The dir is pre-created for the same reason as kvcache
|
||||
# (the engine's create is not recursive).
|
||||
|
||||
if [ ! -e /dev/kfd ] || [ ! -d /dev/dri ]; then
|
||||
echo "ERROR: GPU device nodes absent (/dev/kfd, /dev/dri)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# TTM preflight: the pinned ledger (2 x FRANK_ARCHON_HOST_POOL_MIB + the
|
||||
# 47.7 GiB PLE table + the prefix host tier, ~147 GiB) rides TTM's shared
|
||||
# pages, capped by ttm's pages_limit (the current kernel exposes NO
|
||||
# pages_max; the cap function takes the min over the knobs that exist).
|
||||
# Below FRANK_ARCHON_TTM_MIN_MIB the load dies mid-pin with an allocator
|
||||
# error, so the gate refuses first — and RAISES the knob before refusing:
|
||||
# the cap is a runtime knob whose default is half of RAM, so a power failure
|
||||
# resets it (measured 2026-10-12 23:17 on this host: 200 GiB -> 125,988 MiB,
|
||||
# and Restart=always re-ran the refusal 6,919 times over ~20 h).
|
||||
# FRANK_ARCHON_TTM_TARGET_MIB is the recorded host ceiling, not the floor.
|
||||
# shellcheck source=/dev/null
|
||||
source "$SCRIPT_DIR/ttm-preflight.sh"
|
||||
ttm_ensure_cap || exit 1
|
||||
|
||||
# A crashed/killed unit can leave the container name registered (systemd
|
||||
# SIGKILLs conmon before cleanup runs). Clear OUR name plus the legacy
|
||||
# identities (the FreeToken-era frank-archon-qwen, the retired R9V
|
||||
# container and the ad hoc experiment containers all hold this GPU); all
|
||||
# idempotent.
|
||||
for CTR in $FRANK_ARCHON_LEGACY_CONTAINERS "$FRANK_ARCHON_CONTAINER"; do
|
||||
podman rm -f "$CTR" >/dev/null 2>&1 || true
|
||||
done
|
||||
|
||||
LOG=/home/dank/logs/frank_archon.log
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
|
||||
# Publisher-vetted container contract (docs/DOCKER.md, compose/common.yaml):
|
||||
# --network host (the server binds the host's addresses directly, no NAT),
|
||||
# --init (the engine releases the cards on SIGTERM), seccomp=unconfined
|
||||
# (Docker/podman's default profile refuses io_uring, which the n-gram
|
||||
# table's reads go through, and the NUMA binding calls the ROCm runtime
|
||||
# makes for host memory), memlock unbounded (the pinned ledger — legal
|
||||
# because the root unit grants LimitMEMLOCK=infinity to this dank process),
|
||||
# /dev/kfd + /dev/dri (render/video groups are moot for a root container).
|
||||
# The template rides a single-file mount; the .rad dir is ro; /data holds
|
||||
# the prefix-cache disk tier.
|
||||
podman run -d --name "$FRANK_ARCHON_CONTAINER" \
|
||||
--network host \
|
||||
--init \
|
||||
--security-opt seccomp=unconfined \
|
||||
--ulimit memlock=-1:-1 \
|
||||
--device /dev/kfd --device /dev/dri \
|
||||
-v "$FRANK_ARCHON_MODEL_DIR":/models:ro \
|
||||
-v "$FRANK_ARCHON_STATE_DIR":/data \
|
||||
-v "$FRANK_ARCHON_STATE_DIR/tune":/opt/radiance/share/radiance/tune \
|
||||
-v "$FRANK_ARCHON_TEMPLATE":/templates/template.jinja:ro \
|
||||
"$FRANK_ARCHON_IMAGE" \
|
||||
--model "/models/$FRANK_ARCHON_MODEL_FILE" \
|
||||
--host :: --port "$FRANK_ARCHON_PORT" \
|
||||
--served-model-name "$FRANK_ARCHON_SERVED_NAME" \
|
||||
--override-chat-template /templates/template.jinja \
|
||||
--tp "$FRANK_ARCHON_TP" --tp-wire "$FRANK_ARCHON_TP_WIRE" \
|
||||
--max-num-seqs "$FRANK_ARCHON_MAX_SEQS" \
|
||||
--max-model-len "$FRANK_ARCHON_MAX_MODEL_LEN" \
|
||||
--placement "$FRANK_ARCHON_PLACEMENT" \
|
||||
--host-pool-mib "$FRANK_ARCHON_HOST_POOL_MIB" \
|
||||
--gpu-headroom-mib "$FRANK_ARCHON_GPU_HEADROOM_MIB" \
|
||||
--vram-weights-mib "$FRANK_ARCHON_VRAM_WEIGHTS_MIB" \
|
||||
--vram-kv-mib "$FRANK_ARCHON_VRAM_KV_MIB" \
|
||||
--kv-cache-dtype "$FRANK_ARCHON_KV_DTYPE" \
|
||||
--prefix-cache-host-mib "$FRANK_ARCHON_PREFIX_HOST_MIB" \
|
||||
--prefix-cache-dir /data/kvcache/frank-archon \
|
||||
--prefix-cache-disk-mib "$FRANK_ARCHON_PREFIX_DISK_MIB" \
|
||||
--ngram-placement "$FRANK_ARCHON_NGRAM_PLACEMENT" \
|
||||
--num-speculative-tokens "$FRANK_ARCHON_SPEC_TOKENS" \
|
||||
--max-num-batched-tokens "$FRANK_ARCHON_MAX_BATCHED_TOKENS" \
|
||||
${FRANK_ARCHON_EXTRA_ARGS:+$FRANK_ARCHON_EXTRA_ARGS} \
|
||||
> "$LOG" 2>&1
|
||||
|
||||
# From here on, this script's lifetime IS the container's: wait returns the
|
||||
# container's exit code, so a crash is a nonzero ExecStart and the unit's
|
||||
# Restart=always is real. (systemd stop goes through ExecStop in the unit —
|
||||
# podman wait alone cannot stop the container child.)
|
||||
exec podman wait "$FRANK_ARCHON_CONTAINER"
|
||||
Executable
+200
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/env bash
|
||||
# serve-frank-dragoon.sh — frank-dragoon endpoint: halogen-flash-server 0.16.2
|
||||
# (closed-source engine, image ghcr.io/peonist-ai/halogen-flash-server,
|
||||
# digest-pinned in frank-dragoon.env) serving the Quat3rnion abliterated
|
||||
# Swift1.5 finetune of peonist's Qwen3.8-Flash-Next v2 checkpoint on
|
||||
# Dankclaw's Strix Halo iGPU, host :7770. 2026-10-06 cutover from the
|
||||
# halogen 27B-dense shape — revert the swap commit's trio to roll back
|
||||
# (the 27B payload stays on disk).
|
||||
#
|
||||
# The engine is driven by `podman run` DIRECTLY (the fleet's frank-archon
|
||||
# pattern): the image starts engine + OpenAI frontend in one container.
|
||||
# The frontend listens on 8731 inside (HALOGEN_API_PORT default) and is
|
||||
# published as host 7770; the engine's own token port (8730, no auth)
|
||||
# stays unpublished on the container's loopback. HALOGEN_CHECKPOINT
|
||||
# selects the file inside the /models mount; the n-gram lookup table
|
||||
# (HALOGEN_NGRAM_TABLE, the card sets it explicitly though the documented
|
||||
# fallback is "beside the checkpoint") and the vision tower
|
||||
# (HALOGEN_VISION_TOWER) ride in the same directory.
|
||||
#
|
||||
# Gates mirror the deployer EXACTLY (the 2026-10-01 crash-loop lesson —
|
||||
# serve gate and deploy gate disagreeing = green deploy, red unit):
|
||||
# checkpoint/n-gram/vision byte-exact, tokenizer directory present with
|
||||
# every named file a FLAT REGULAR FILE (a HF cache snapshot's symlinks
|
||||
# into blobs/ dangle inside the mount), GPU device nodes present.
|
||||
#
|
||||
# Why containers instead of a bare binary: the payload is .hgn and only
|
||||
# the flash-server line reads it; it ships as a container; the root unit
|
||||
# keeps LimitMEMLOCK=infinity for parity and to outlive the login session,
|
||||
# exactly like frank-archon's.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}"
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
echo "frank-dragoon: env file not found: $ENV_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
|
||||
: "${FRANK_DRAGOON_IMAGE:?FRANK_DRAGOON_IMAGE must name the digest-pinned image}"
|
||||
: "${FRANK_DRAGOON_CONTAINER:?FRANK_DRAGOON_CONTAINER must name the container}"
|
||||
: "${FRANK_DRAGOON_PUBLISH:?FRANK_DRAGOON_PUBLISH must be host:container}"
|
||||
: "${FRANK_DRAGOON_MODEL_DIR:?FRANK_DRAGOON_MODEL_DIR must name the payload dir}"
|
||||
: "${FRANK_DRAGOON_CHECKPOINT:?FRANK_DRAGOON_CHECKPOINT must name the .hgn inside it}"
|
||||
: "${FRANK_DRAGOON_CHECKPOINT_BYTES:?FRANK_DRAGOON_CHECKPOINT_BYTES must pin the size}"
|
||||
: "${FRANK_DRAGOON_NGRAM_TABLE:?FRANK_DRAGOON_NGRAM_TABLE must name the lookup table}"
|
||||
: "${FRANK_DRAGOON_NGRAM_BYTES:?FRANK_DRAGOON_NGRAM_BYTES must pin the table size}"
|
||||
: "${FRANK_DRAGOON_VISION_TOWER:?FRANK_DRAGOON_VISION_TOWER must name the tower}"
|
||||
: "${FRANK_DRAGOON_VISION_BYTES:?FRANK_DRAGOON_VISION_BYTES must pin the tower size}"
|
||||
: "${FRANK_DRAGOON_TOKENIZER_FILES:?FRANK_DRAGOON_TOKENIZER_FILES must list the tokenizer dir contents}"
|
||||
: "${HALOGEN_MODEL_ID:?HALOGEN_MODEL_ID must be the served alias (frank-dragoon)}"
|
||||
: "${HALOGEN_CTX:?HALOGEN_CTX must be set (per-slot window)}"
|
||||
: "${HALOGEN_KV_POOL_POSITIONS:?HALOGEN_KV_POOL_POSITIONS must be set}"
|
||||
: "${HALOGEN_KV_SLOTS:?HALOGEN_KV_SLOTS must be set}"
|
||||
: "${HALOGEN_SPEC_ADAPT:?HALOGEN_SPEC_ADAPT must be set (0 = head always on)}"
|
||||
|
||||
# Capture podman's output BEFORE grepping: under `set -o pipefail` the
|
||||
# `podman images | grep -q` pattern races (grep exits on first match,
|
||||
# podman dies SIGPIPE 141, pipefail turns the 141 into "no match" — the
|
||||
# 2026-09-30 deploy refusal).
|
||||
DIGESTS="$(podman images --format '{{.RepoDigests}}')"
|
||||
if ! grep -qF "${FRANK_DRAGOON_IMAGE#*@}" <<<"$DIGESTS"; then
|
||||
echo "frank-dragoon: image $FRANK_DRAGOON_IMAGE not in podman storage;" >&2
|
||||
echo " pull it: podman pull ${FRANK_DRAGOON_IMAGE%@*}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Size-exact gates on all three payload files: a truncated curl exits 0 on
|
||||
# some failures and a partial .hgn loads far enough to look alive, so the
|
||||
# byte counts from the Hub tree API are the contract (fleet fetch law).
|
||||
ck() {
|
||||
# ck <env-name> <path> <expected-bytes>
|
||||
if [ ! -f "$2" ]; then
|
||||
echo "frank-dragoon: $1 $2 missing;" >&2
|
||||
echo " fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
local sz
|
||||
sz="$(stat -c %s "$2")"
|
||||
if [ "$sz" != "$3" ]; then
|
||||
echo "frank-dragoon: $1 is $sz bytes, expected $3 (partial download?)" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
ck checkpoint "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_CHECKPOINT" "$FRANK_DRAGOON_CHECKPOINT_BYTES"
|
||||
ck n-gram-table "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_NGRAM_TABLE" "$FRANK_DRAGOON_NGRAM_BYTES"
|
||||
ck vision-tower "$FRANK_DRAGOON_MODEL_DIR/$FRANK_DRAGOON_VISION_TOWER" "$FRANK_DRAGOON_VISION_BYTES"
|
||||
|
||||
# The payload ships the tokenizer as a FLAT DIRECTORY. A HuggingFace cache
|
||||
# snapshot symlinks each file into a sibling blobs/, and those symlinks
|
||||
# dangle inside the container mount — so the directory must be real and
|
||||
# every named file must be a regular file, not a link.
|
||||
TOK_DIR="$FRANK_DRAGOON_MODEL_DIR/tokenizer"
|
||||
if [ -L "$TOK_DIR" ] || [ ! -d "$TOK_DIR" ]; then
|
||||
echo "frank-dragoon: tokenizer dir $TOK_DIR missing or a symlink (HF cache snapshot?);" >&2
|
||||
echo " fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
for TF in $FRANK_DRAGOON_TOKENIZER_FILES; do
|
||||
if [ ! -f "$TOK_DIR/$TF" ] || [ -L "$TOK_DIR/$TF" ]; then
|
||||
echo "frank-dragoon: tokenizer file $TOK_DIR/$TF missing or a symlink;" >&2
|
||||
echo " fetch it: bash scripts/download-frank-dragoon-flashnext.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# GPU access is mandatory but the node paths are variables so the seam can
|
||||
# drive the gate on a box without an AMD GPU.
|
||||
DEV_KFD="${FRANK_DRAGOON_DEV_KFD:-/dev/kfd}"
|
||||
DEV_DRI="${FRANK_DRAGOON_DEV_DRI:-/dev/dri}"
|
||||
if [ ! -e "$DEV_KFD" ] || [ ! -d "$DEV_DRI" ]; then
|
||||
echo "frank-dragoon: $DEV_KFD or $DEV_DRI missing — no gfx1151 to serve on" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Host-RAM preflight — the engine's pin gate, checked BEFORE podman run.
|
||||
# The engine refuses to pin when MemAvailable is below the checkpoint plus
|
||||
# a 16 GiB floor: pinned pages cannot be swapped, and under the floor the
|
||||
# n-gram lookup pages in from disk on every long prompt. A RAM-short host
|
||||
# (a co-tenant spike, a sibling engine dying) crash-loops the unit — each
|
||||
# boot dies in ~5 s at the pin gate — until the RAM clears. The 2026-10-06
|
||||
# cutover hit this: 8 boots, MemAvailable 74.77 GiB vs the 78.11 GiB needed,
|
||||
# self-resolved in ~2 min when the dying 27B engine freed RAM. Check here and
|
||||
# WAIT for the RAM to clear, so a transient dip boots once instead of
|
||||
# crash-looping; if it never clears, refuse loudly.
|
||||
MEMINFO="${FRANK_DRAGOON_MEMINFO:-/proc/meminfo}"
|
||||
PIN_KB=$(( FRANK_DRAGOON_CHECKPOINT_BYTES / 1024 ))
|
||||
NEED_KB=$(( PIN_KB + 16 * 1048576 ))
|
||||
ram_avail_kb() { awk '/^MemAvailable:/{print $2; exit}' "$MEMINFO"; }
|
||||
MEMAVAIL_KB="$(ram_avail_kb)" || MEMAVAIL_KB=""
|
||||
if [ -n "$MEMAVAIL_KB" ] && [ "$MEMAVAIL_KB" -lt "$NEED_KB" ]; then
|
||||
WAIT_S="${FRANK_DRAGOON_RAM_WAIT:-600}"
|
||||
GIB() { awk -v kb="$1" 'BEGIN{printf "%.2f", kb/1048576}'; }
|
||||
echo "frank-dragoon: host RAM below the pin gate — MemAvailable $(GIB "$MEMAVAIL_KB") GiB < needed $(GIB "$NEED_KB") GiB ($(GIB "$PIN_KB") GiB checkpoint + 16.00 GiB floor); waiting up to ${WAIT_S}s for host RAM to clear" >&2
|
||||
deadline=$(( $(date +%s) + WAIT_S ))
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
sleep 5
|
||||
MEMAVAIL_KB="$(ram_avail_kb)" || MEMAVAIL_KB=""
|
||||
if [ -n "$MEMAVAIL_KB" ] && [ "$MEMAVAIL_KB" -ge "$NEED_KB" ]; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ -z "$MEMAVAIL_KB" ] || [ "$MEMAVAIL_KB" -lt "$NEED_KB" ]; then
|
||||
echo "frank-dragoon: MemAvailable $(GIB "${MEMAVAIL_KB:-0}") GiB after ${WAIT_S}s; refusing to start (the engine would refuse the pin)" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "frank-dragoon: MemAvailable cleared to $(GIB "$MEMAVAIL_KB") GiB; starting" >&2
|
||||
fi
|
||||
for CTR in ${FRANK_DRAGOON_LEGACY_CONTAINERS:-} "$FRANK_DRAGOON_CONTAINER"; do
|
||||
podman rm -f "$CTR" >/dev/null 2>&1 || true
|
||||
done
|
||||
|
||||
LOG="${FRANK_DRAGOON_LOG:-/home/dank/logs/frank_dragoon.log}"
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
# --ipc=host for the udev-granted supplementary group, seccomp unconfined
|
||||
# for the ROCm ioctl surface, and unbounded memlock (the unit's
|
||||
# LimitMEMLOCK=infinity is inherited, so runc never has to RAISE it). NO
|
||||
# --ulimit nofile: Dankclaw's hard limit is 524288, rootless runc cannot
|
||||
# raise a hard limit and dies with "error setting rlimit type 7:
|
||||
# operation not permitted" (measured 2026-10-05 cutover); the model card
|
||||
# and both sibling serve scripts pass no nofile either. No comment lines
|
||||
# inside the backslash chain — a mid-chain # terminates the continuation
|
||||
# (the 2026-10-05 exit-127 crash loop on the sibling serve script).
|
||||
podman run --name "$FRANK_DRAGOON_CONTAINER" \
|
||||
--publish "$FRANK_DRAGOON_PUBLISH" \
|
||||
--ipc=host \
|
||||
--ulimit memlock=-1:-1 \
|
||||
--device "$DEV_KFD" --device "$DEV_DRI" \
|
||||
--group-add keep-groups \
|
||||
--security-opt seccomp=unconfined \
|
||||
-e HALOGEN_MODEL_ID="$HALOGEN_MODEL_ID" \
|
||||
-e HALOGEN_CHECKPOINT="/models/$FRANK_DRAGOON_CHECKPOINT" \
|
||||
-e HALOGEN_NGRAM_TABLE="/models/$FRANK_DRAGOON_NGRAM_TABLE" \
|
||||
-e HALOGEN_VISION_TOWER="/models/$FRANK_DRAGOON_VISION_TOWER" \
|
||||
-e HALOGEN_TOKENIZER=/tokenizer \
|
||||
-e HALOGEN_CTX="$HALOGEN_CTX" \
|
||||
-e HALOGEN_KV_POOL_POSITIONS="$HALOGEN_KV_POOL_POSITIONS" \
|
||||
-e HALOGEN_KV_SLOTS="$HALOGEN_KV_SLOTS" \
|
||||
-e HALOGEN_HOST_RESERVE_GIB="${HALOGEN_HOST_RESERVE_GIB:-20}" \
|
||||
-e HALOGEN_SPEC_ADAPT="$HALOGEN_SPEC_ADAPT" \
|
||||
-e HALOGEN_MAX_TOK="${HALOGEN_MAX_TOK:-8192}" \
|
||||
-e HALOGEN_MAX_TOKENS_CAP="${HALOGEN_MAX_TOKENS_CAP:-65536}" \
|
||||
-e HALOGEN_QUEUE_TIMEOUT="${HALOGEN_QUEUE_TIMEOUT:-7200}" \
|
||||
-e HALOGEN_TEMPERATURE="${HALOGEN_TEMPERATURE:-1.0}" \
|
||||
-e HALOGEN_TOP_P="${HALOGEN_TOP_P:-0.95}" \
|
||||
-e HALOGEN_TOP_K="${HALOGEN_TOP_K:-20}" \
|
||||
${HALOGEN_WEIGHTS_LOCK:+-e HALOGEN_WEIGHTS_LOCK="$HALOGEN_WEIGHTS_LOCK"} \
|
||||
-v "$FRANK_DRAGOON_MODEL_DIR:/models:ro" \
|
||||
-v "$FRANK_DRAGOON_MODEL_DIR/tokenizer:/tokenizer:ro" \
|
||||
"$FRANK_DRAGOON_IMAGE" \
|
||||
> "$LOG" 2>&1
|
||||
|
||||
# FOREGROUND `podman run`: the engine's stdout/stderr are this script's
|
||||
# output, so the boot ledger (kv pool, prompt cache, MTP head, any clamp)
|
||||
# lands in $LOG as documented. podman run returns the container's exit
|
||||
# code, so a crash is a nonzero ExecStart and the unit's Restart=always
|
||||
# stays real; systemd stop goes through the unit's ExecStop and the run
|
||||
# client forwards SIGTERM to the engine, which drains on it.
|
||||
@@ -1,146 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# serve-frank-large.sh — frank-large endpoint: halogen (closed-source
|
||||
# C++/HIP engine, image ghcr.io/peonist-ai/halogen, digest-pinned in
|
||||
# frank-large.env) serving peonist-ai/halogen-qwen3.8-27b on Dankclaw's
|
||||
# Strix Halo iGPU, host :7770. 2026-10-05 cutover from the ROCmFPX
|
||||
# llama.cpp shape — revert the swap commit's trio to roll back (payload
|
||||
# and fork build stay on disk).
|
||||
#
|
||||
# The engine is driven by `podman run` DIRECTLY (the fleet's frank-primary
|
||||
# pattern): the image entrypoint `halogen-docker` in default "all" mode
|
||||
# starts engine + OpenAI frontend in one container. The frontend listens on
|
||||
# 8731 inside (HALOGEN_API_PORT default) and is published as host 7770; the
|
||||
# engine's own token port (8730, no auth) stays unpublished on the
|
||||
# container's loopback. The image ships NO default checkpoint:
|
||||
# HALOGEN_CHECKPOINT selects the file inside the /models mount.
|
||||
#
|
||||
# Gates mirror the deployer EXACTLY (the 2026-10-01 crash-loop lesson —
|
||||
# serve gate and deploy gate disagreeing = green deploy, red unit):
|
||||
# tokenizer directory present with every named file a FLAT REGULAR FILE —
|
||||
# the model card ships `tokenizer/` as a flat directory and the engine
|
||||
# defaults HALOGEN_TOKENIZER to /tokenizer, so a HF cache snapshot's
|
||||
# symlinks into blobs/ dangle inside the mount; GPU device nodes present.
|
||||
#
|
||||
# Why containers instead of the bare binary (the old frank-large shape):
|
||||
# the payload is .hgn and only halogen reads it; halogen ships as a
|
||||
# container; the root unit keeps LimitMEMLOCK=infinity for parity and to
|
||||
# outlive the login session, exactly like frank-primary's.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
echo "frank-large: env file not found: $ENV_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
|
||||
: "${FRANK_LARGE_IMAGE:?FRANK_LARGE_IMAGE must name the digest-pinned image}"
|
||||
: "${FRANK_LARGE_CONTAINER:?FRANK_LARGE_CONTAINER must name the container}"
|
||||
: "${FRANK_LARGE_PUBLISH:?FRANK_LARGE_PUBLISH must be host:container}"
|
||||
: "${FRANK_LARGE_MODEL_DIR:?FRANK_LARGE_MODEL_DIR must name the payload dir}"
|
||||
: "${FRANK_LARGE_CHECKPOINT:?FRANK_LARGE_CHECKPOINT must name the .hgn inside it}"
|
||||
: "${FRANK_LARGE_CHECKPOINT_BYTES:?FRANK_LARGE_CHECKPOINT_BYTES must pin the size}"
|
||||
: "${FRANK_LARGE_TOKENIZER_FILES:?FRANK_LARGE_TOKENIZER_FILES must list the tokenizer dir contents}"
|
||||
: "${HALOGEN_MODEL_ID:?HALOGEN_MODEL_ID must be the served alias (frank-large)}"
|
||||
: "${HALOGEN_SLOT_CTX:?HALOGEN_SLOT_CTX must be set}"
|
||||
: "${HALOGEN_KV_SLOTS:?HALOGEN_KV_SLOTS must be set (1 keeps the drafter on)}"
|
||||
: "${HALOGEN_DRAFTER:?HALOGEN_DRAFTER must be set (2 = DFlash2)}"
|
||||
|
||||
# Capture podman's output BEFORE grepping: under `set -o pipefail` the
|
||||
# `podman images | grep -q` pattern races (grep exits on first match,
|
||||
# podman dies SIGPIPE 141, pipefail turns the 141 into "no match" — the
|
||||
# 2026-09-30 deploy refusal).
|
||||
DIGESTS="$(podman images --format '{{.RepoDigests}}')"
|
||||
if ! grep -qF "${FRANK_LARGE_IMAGE#*@}" <<<"$DIGESTS"; then
|
||||
echo "frank-large: image $FRANK_LARGE_IMAGE not in podman storage;" >&2
|
||||
echo " pull it: podman pull ${FRANK_LARGE_IMAGE%@*}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The checkpoint gate is size-exact: a truncated curl exits 0 on some
|
||||
# failures and a partial .hgn loads far enough to look alive, so the byte
|
||||
# count from the model card is the contract (the fleet fetch law).
|
||||
CKPT="$FRANK_LARGE_MODEL_DIR/$FRANK_LARGE_CHECKPOINT"
|
||||
if [ ! -f "$CKPT" ]; then
|
||||
echo "frank-large: checkpoint $CKPT missing;" >&2
|
||||
echo " fetch it: bash scripts/download-frank-large-halogen.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
CKPT_SIZE="$(stat -c %s "$CKPT")"
|
||||
if [ "$CKPT_SIZE" != "$FRANK_LARGE_CHECKPOINT_BYTES" ]; then
|
||||
echo "frank-large: checkpoint is $CKPT_SIZE bytes, expected $FRANK_LARGE_CHECKPOINT_BYTES (partial download?)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The payload ships the tokenizer as a FLAT DIRECTORY (model card:
|
||||
# "tokenizer/ flat tokenizer directory"). A HuggingFace cache snapshot
|
||||
# symlinks each file into a sibling blobs/, and those symlinks dangle
|
||||
# inside the container mount — so the directory must be real and every
|
||||
# named file must be a regular file, not a link.
|
||||
TOK_DIR="$FRANK_LARGE_MODEL_DIR/tokenizer"
|
||||
if [ -L "$TOK_DIR" ] || [ ! -d "$TOK_DIR" ]; then
|
||||
echo "frank-large: tokenizer dir $TOK_DIR missing or a symlink (HF cache snapshot?);" >&2
|
||||
echo " fetch it: bash scripts/download-frank-large-halogen.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
for TF in $FRANK_LARGE_TOKENIZER_FILES; do
|
||||
if [ ! -f "$TOK_DIR/$TF" ] || [ -L "$TOK_DIR/$TF" ]; then
|
||||
echo "frank-large: tokenizer file $TOK_DIR/$TF missing or a symlink;" >&2
|
||||
echo " fetch it: bash scripts/download-frank-large-halogen.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# GPU access is mandatory but the node paths are variables so the seam can
|
||||
# drive the gate on a box without an AMD GPU.
|
||||
DEV_KFD="${FRANK_LARGE_DEV_KFD:-/dev/kfd}"
|
||||
DEV_DRI="${FRANK_LARGE_DEV_DRI:-/dev/dri}"
|
||||
if [ ! -e "$DEV_KFD" ] || [ ! -d "$DEV_DRI" ]; then
|
||||
echo "frank-large: $DEV_KFD or $DEV_DRI missing — no gfx1151 to serve on" >&2
|
||||
exit 1
|
||||
fi
|
||||
for CTR in ${FRANK_LARGE_LEGACY_CONTAINERS:-} "$FRANK_LARGE_CONTAINER"; do
|
||||
podman rm -f "$CTR" >/dev/null 2>&1 || true
|
||||
done
|
||||
|
||||
LOG="${FRANK_LARGE_LOG:-/home/dank/logs/frank_large.log}"
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
# for the udev-granted supplementary group, seccomp unconfined for the ROCm
|
||||
# ioctl surface, and unbounded memlock (the unit's LimitMEMLOCK=infinity is
|
||||
# inherited, so runc never has to RAISE it). NO --ulimit nofile: Dankclaw's
|
||||
# hard limit is 524288, rootless runc cannot raise a hard limit and dies
|
||||
# with "error setting rlimit type 7: operation not permitted" (measured
|
||||
# 2026-10-05 cutover), and 524288 fds is already ~100x what the engine
|
||||
# wants; the model card and both sibling serve scripts pass no nofile
|
||||
# either. No comment lines inside the backslash
|
||||
# chain — a mid-chain # terminates the continuation (the 2026-10-05
|
||||
# exit-127 crash loop on the sibling serve script).
|
||||
podman run -d --name "$FRANK_LARGE_CONTAINER" \
|
||||
--publish "$FRANK_LARGE_PUBLISH" \
|
||||
--ipc=host \
|
||||
--ulimit memlock=-1:-1 \
|
||||
--device "$DEV_KFD" --device "$DEV_DRI" \
|
||||
--group-add keep-groups \
|
||||
--security-opt seccomp=unconfined \
|
||||
-e HALOGEN_MODEL_ID="$HALOGEN_MODEL_ID" \
|
||||
-e HALOGEN_CHECKPOINT="/models/$FRANK_LARGE_CHECKPOINT" \
|
||||
-e HALOGEN_SLOT_CTX="$HALOGEN_SLOT_CTX" \
|
||||
-e HALOGEN_KV_SLOTS="${HALOGEN_KV_SLOTS:-1}" \
|
||||
-e HALOGEN_DRAFTER="$HALOGEN_DRAFTER" \
|
||||
-e HALOGEN_MAX_TOKENS_CAP="${HALOGEN_MAX_TOKENS_CAP:-65536}" \
|
||||
-e HALOGEN_QUEUE_TIMEOUT="${HALOGEN_QUEUE_TIMEOUT:-7200}" \
|
||||
${HALOGEN_CACHE_MB:+-e HALOGEN_CACHE_MB="$HALOGEN_CACHE_MB"} \
|
||||
-v "$FRANK_LARGE_MODEL_DIR:/models:ro" \
|
||||
-v "$FRANK_LARGE_MODEL_DIR/tokenizer:/tokenizer:ro" \
|
||||
"$FRANK_LARGE_IMAGE" \
|
||||
> "$LOG" 2>&1
|
||||
|
||||
# This script's lifetime IS the container's: podman wait returns the
|
||||
# container's exit code, so a crash is a nonzero ExecStart and the unit's
|
||||
# Restart=always is real. systemd stop goes through the unit's ExecStop
|
||||
# (podman wait alone cannot stop the child).
|
||||
exec podman wait "$FRANK_LARGE_CONTAINER"
|
||||
@@ -1,132 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# serve-frank-primary.sh — frank-primary endpoint (FreeToken PR #587 ROCm
|
||||
# build, orcarouter Qwen3.8-Flash-Next-Uncensored-FP8, ONE R9700 gfx1201,
|
||||
# dankserver :7770; 2026-10-05 cutover from the tcclaviger vLLM fork — git
|
||||
# tag frank-primary-vllm-shapeG-20261003 is the pre-cutover state).
|
||||
#
|
||||
# The engine is driven by `podman run` DIRECTLY and the run line below IS
|
||||
# the engine argv: the image carries FreeToken installed at pinned PR head
|
||||
# 77a7201 (scripts/docker/frank-primary-freetoken.Dockerfile builds it from
|
||||
# the PR's own docs/install_amd.md stack). Unlike the vLLM fork there is no
|
||||
# per-boot planner: experts live in pinned host RAM with an LRU GPU cache,
|
||||
# KV is an explicit token pool, and the boot ledger in the log names
|
||||
# Shape: 1 GPU (first-boot conservative — the second R9700 sits idle;
|
||||
# --tensor-parallel-size EXISTS in FreeToken, so a 2-card TP shape is future
|
||||
# tuning once this single-card shape has a decode baseline), KV pool 300,000
|
||||
# tokens (24 KiB/token f16 across the 12 full-attention layers = 7.2 GiB;
|
||||
# clears the harness worst case of 177,356 tokens and admits one NATIVE
|
||||
# 262,144-token conversation), MoE LRU at 10% of all experts (~12 GiB),
|
||||
# 2 decode slots. First-boot conservative; `ft ctl cache --kv/--moe` grows
|
||||
# pools live, and the env comment block carries the resize law. Flag
|
||||
# surface verified against docs/cli.md of the pinned head AND against
|
||||
# `ft serve --help` inside the built image; every value here is env-driven
|
||||
# (frank-primary.env).
|
||||
|
||||
# The root unit (scripts/frank-primary.service, User=dank,
|
||||
# LimitMEMLOCK=infinity) grants the pin budget: ~120 GiB of expert weights
|
||||
# plus the ~47.7 GiB PLE table pin in host RAM. A plain shell session also
|
||||
# works, but the unit is the fleet-parity boot-persistent shape.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_PRIMARY_ENV_FILE:-$SCRIPT_DIR/frank-primary.env}"
|
||||
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
echo "serve-frank-primary: env file missing: $ENV_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
|
||||
: "${FRANK_PRIMARY_IMAGE:?FRANK_PRIMARY_IMAGE must name the FreeToken image}"
|
||||
: "${FRANK_PRIMARY_MODEL_DIR:?FRANK_PRIMARY_MODEL_DIR must name the FP8 payload}"
|
||||
: "${FRANK_PRIMARY_MODEL_SHARDS:?FRANK_PRIMARY_MODEL_SHARDS must give the shard count}"
|
||||
: "${FRANK_PRIMARY_CONTAINER:?FRANK_PRIMARY_CONTAINER must name the container}"
|
||||
: "${FRANK_PRIMARY_SERVED_NAME:?FRANK_PRIMARY_SERVED_NAME must name the alias}"
|
||||
: "${FRANK_PRIMARY_PUBLISH:?FRANK_PRIMARY_PUBLISH must be host:container}"
|
||||
: "${FRANK_PRIMARY_FT_GPU:?FRANK_PRIMARY_FT_GPU must name the GPU index}"
|
||||
: "${FRANK_PRIMARY_FT_NUM_TOKENS:?FRANK_PRIMARY_FT_NUM_TOKENS must size the KV pool}"
|
||||
: "${FRANK_PRIMARY_FT_MAX_RUNNING:?FRANK_PRIMARY_FT_MAX_RUNNING must be set}"
|
||||
: "${FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS:?FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS must be set}"
|
||||
: "${FRANK_PRIMARY_FT_MOE_RATE:?FRANK_PRIMARY_FT_MOE_RATE must be set}"
|
||||
|
||||
# The image gate is EXISTENCE IN STORAGE, not a registry digest: the tag is
|
||||
# built ON THE HOST (podman build from
|
||||
# scripts/docker/frank-primary-freetoken.Dockerfile) and local builds carry
|
||||
# no RepoDigests entry — the 362-line vLLM-era RepoDigests grep refused
|
||||
# exactly this shape. Pinning lives in the Dockerfile instead: the FreeToken
|
||||
# source is checked out at a hardcoded PR-head SHA and the base image is
|
||||
# digest-pinned there. `podman image exists` accepts both a local tag and a
|
||||
# name@digest ref, and it is one command — no pipefail/grep race (the
|
||||
# 2026-09-30 lesson) for either shape.
|
||||
if ! podman image exists "$FRANK_PRIMARY_IMAGE"; then
|
||||
echo "serve-frank-primary: image absent from rootless storage: $FRANK_PRIMARY_IMAGE" >&2
|
||||
echo " build it: podman build -t $FRANK_PRIMARY_IMAGE -f scripts/docker/frank-primary-freetoken.Dockerfile scripts/docker/" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$FRANK_PRIMARY_MODEL_DIR/config.json" ]; then
|
||||
echo "serve-frank-primary: payload config missing: $FRANK_PRIMARY_MODEL_DIR/config.json" >&2
|
||||
exit 1
|
||||
fi
|
||||
SHARDS="$(find "$FRANK_PRIMARY_MODEL_DIR" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l)"
|
||||
if [ "$SHARDS" != "$FRANK_PRIMARY_MODEL_SHARDS" ]; then
|
||||
echo "serve-frank-primary: payload shard count wrong: $FRANK_PRIMARY_MODEL_DIR" >&2
|
||||
echo " found $SHARDS safetensors, want $FRANK_PRIMARY_MODEL_SHARDS" >&2
|
||||
echo " re-fetch: scripts/download-frank-primary.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -f "$FRANK_PRIMARY_MODEL_DIR/model.safetensors.index.json" ]; then
|
||||
echo "serve-frank-primary: weight-map index missing: $FRANK_PRIMARY_MODEL_DIR/model.safetensors.index.json" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -e /dev/kfd ] || [ ! -d /dev/dri ]; then
|
||||
echo "serve-frank-primary: GPU device nodes absent (/dev/kfd, /dev/dri)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# A crashed/killed unit can leave the container name registered (systemd
|
||||
# SIGKILLs conmon before cleanup runs). Clear OUR name plus the legacy
|
||||
# identities (the vLLM-era frank-primary-qwen, the retired R9V container and
|
||||
# the ad hoc experiment containers all hold this GPU); all idempotent.
|
||||
for CTR in $FRANK_PRIMARY_LEGACY_CONTAINERS "$FRANK_PRIMARY_CONTAINER"; do
|
||||
podman rm -f "$CTR" >/dev/null 2>&1 || true
|
||||
done
|
||||
|
||||
LOG=/home/dank/logs/frank_primary.log
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
|
||||
# --device /dev/dri is the WHOLE DIRECTORY (individual renderD nodes enumerate
|
||||
# without the card nodes and HIP refuses contexts). --ipc=host for the
|
||||
# engine's shared memory; memlock unbounded for the ~120 GiB pinned expert
|
||||
# rows + the 47.7 GiB PLE pin (the unit's LimitMEMLOCK=infinity is the other
|
||||
# half of that law).
|
||||
# HIP_VISIBLE_DEVICES pins visibility to ONE card so in-container index 0 is
|
||||
# the right R9700; --gpu 0 selects it inside the visible set.
|
||||
podman run -d --name "$FRANK_PRIMARY_CONTAINER" \
|
||||
--publish "$FRANK_PRIMARY_PUBLISH" \
|
||||
--ipc=host --ulimit memlock=-1:-1 \
|
||||
--cap-add SYS_PTRACE \
|
||||
--device /dev/kfd --device /dev/dri --device /dev/mem \
|
||||
-e HIP_VISIBLE_DEVICES="$FRANK_PRIMARY_FT_GPU" \
|
||||
-v "$FRANK_PRIMARY_MODEL_DIR":/models:ro \
|
||||
"$FRANK_PRIMARY_IMAGE" \
|
||||
ft serve --model /models \
|
||||
--served-model-name "$FRANK_PRIMARY_SERVED_NAME" \
|
||||
--host 0.0.0.0 --port 7770 \
|
||||
--gpu "$FRANK_PRIMARY_FT_GPU" \
|
||||
--num-tokens "$FRANK_PRIMARY_FT_NUM_TOKENS" \
|
||||
--max-running-requests "$FRANK_PRIMARY_FT_MAX_RUNNING" \
|
||||
--max-output-tokens "$FRANK_PRIMARY_FT_MAX_OUTPUT_TOKENS" \
|
||||
--moe-cache-rate "$FRANK_PRIMARY_FT_MOE_RATE" \
|
||||
${FRANK_PRIMARY_FT_EXTRA_ARGS:+$FRANK_PRIMARY_FT_EXTRA_ARGS} \
|
||||
> "$LOG" 2>&1
|
||||
|
||||
# From here on, this script's lifetime IS the container's: wait returns the
|
||||
# container's exit code, so a crash is a nonzero ExecStart and the unit's
|
||||
# Restart=always is real. (systemd stop goes through ExecStop in the unit —
|
||||
# podman wait alone cannot stop the container child.)
|
||||
exec podman wait "$FRANK_PRIMARY_CONTAINER"
|
||||
@@ -1,99 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve ornith-ai/Ornith-1.5-9B-GGUF (Q8_0) as frank-small on
|
||||
# Dankclaw's RTX 3090 via llama.cpp (master build, CUDA sm_86).
|
||||
#
|
||||
# 2026-10-01 night (Frank): swapped from SGLang + INT8-W8A16-AutoRound
|
||||
# safetensors to the llama.cpp GGUF path, because llama.cpp is the only
|
||||
# runtime on this host that moves K and V independently — the asymmetric
|
||||
# f16-K + q8_0-V contract that endpoint was built for. 2026-10-03 (Frank):
|
||||
# the pair is now f16/f16 — two wide slots freed the VRAM, so the values are
|
||||
# unquantized too. The per-tensor flags stay explicit either way.
|
||||
#
|
||||
# Architecture: qwen3_5 hybrid — 32 layers, 8 full-attention + 24 Gated
|
||||
# DeltaNet, native 262144 window. KV lives only in the 8 attention layers;
|
||||
# the DeltaNet state is per-slot fixed size. Vision comes from the repo's
|
||||
# separate projector (the Q6_K GGUF carries no `v.*` tensors at all), so
|
||||
# --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is the documented
|
||||
# text-only opt-out.
|
||||
#
|
||||
# Sampling: llama.cpp takes it per request (the fleet harness sends its
|
||||
# recipe from models.yml); greedy is the harness default.
|
||||
#
|
||||
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -------------------
|
||||
: "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
||||
: "${MODEL_FILE:=/home/dank/models/Ornith-Q8/Ornith-1.5-9B-Q8_0.gguf}"
|
||||
: "${ALIAS:=frank-small}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7771}"
|
||||
# ctx-size is the POOL llama.cpp divides across slots: 401408 / 2 gives
|
||||
# 200,704 per slot — two independent conversations, each one wide enough to
|
||||
# hold a whole frank-primary-sized prompt in a single shot (the native 262,144
|
||||
# no longer fits twice on a 24 GiB card once the vision tower is loaded;
|
||||
# frank-small.env carries the three measurements that set this number).
|
||||
: "${CTX_SIZE:=401408}"
|
||||
: "${PARALLEL:=2}"
|
||||
# Compute-buffer geometry. Before 2026-10-03 this endpoint never passed either
|
||||
# flag, so it ran on llama.cpp's own defaults (2048/512). The pair is 4096/2048
|
||||
# since 2026-10-05: on this CUDA card the prefill curve climbs all the way to
|
||||
# ubatch 2048 (unlike the R9700 layer-split sibling, where 512 is the peak),
|
||||
# and the logical batch is VRAM- and throughput-neutral. frank-small.env
|
||||
# carries the sweep tables and the VRAM risk note.
|
||||
: "${BATCH_SIZE:=4096}"
|
||||
: "${UBATCH_SIZE:=2048}"
|
||||
: "${CACHE_TYPE_K:=f16}"
|
||||
: "${CACHE_TYPE_V:=f16}"
|
||||
: "${NGL:=99}"
|
||||
# Vision projector. No colon on purpose: an explicitly empty value means
|
||||
# "text-only, on purpose" and must NOT be replaced by the default path.
|
||||
: "${MMPROJ_FILE=/home/dank/models/Ornith-Q8/mmproj-Ornith-1.5-9B-BF16.gguf}"
|
||||
|
||||
# Fail loudly rather than half-loading: a missing file is a download that
|
||||
# has not finished, and Restart=always would turn it into a crash loop that
|
||||
# reads like a runtime fault.
|
||||
for path in "$LLAMA_BIN" "$MODEL_FILE"; do
|
||||
[[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
|
||||
done
|
||||
EXTRA_ARGS=()
|
||||
if [[ -n "$MMPROJ_FILE" ]]; then
|
||||
[[ -e "$MMPROJ_FILE" ]] || { echo "frank-small: missing $MMPROJ_FILE" >&2; exit 1; }
|
||||
EXTRA_ARGS+=(--mmproj "$MMPROJ_FILE")
|
||||
fi
|
||||
|
||||
# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
|
||||
# open after boot; make sure it exists before the server needs it.
|
||||
nvidia-modprobe -u -c=0 2>/dev/null || true
|
||||
|
||||
# Flash attention stays pinned on: it is the fast path for f16 KV, and it is
|
||||
# the only path that could serve quantized V at all — so if CACHE_TYPE_V is
|
||||
# ever set back to q8_0 this flag must not be lost, or the build falls back
|
||||
# silently and the KV contract in frank-small.env becomes a lie.
|
||||
ARGS=(
|
||||
--model "$MODEL_FILE"
|
||||
--alias "$ALIAS"
|
||||
--host "$BIND_ADDR"
|
||||
--port "$LISTEN_PORT"
|
||||
--ctx-size "$CTX_SIZE"
|
||||
--parallel "$PARALLEL"
|
||||
--batch-size "$BATCH_SIZE"
|
||||
--ubatch-size "$UBATCH_SIZE"
|
||||
--n-gpu-layers "$NGL"
|
||||
--flash-attn on
|
||||
--cache-type-k "$CACHE_TYPE_K"
|
||||
--cache-type-v "$CACHE_TYPE_V"
|
||||
--jinja
|
||||
--reasoning auto
|
||||
--reasoning-format deepseek
|
||||
)
|
||||
exec "$LLAMA_BIN" "${ARGS[@]}" ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}
|
||||
Executable
+132
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive (Q6_K) as
|
||||
# frank-zealot on Dankclaw's RTX 3090 via llama.cpp (master build, CUDA sm_86).
|
||||
#
|
||||
# 2026-10-07 (Frank): Qwen3.5-9B swap at TWO 200,704-token conversations
|
||||
# (--ctx-size 401408 --parallel 2) with EXACT f16 KV — the 9B rung frees the
|
||||
# room the 26B MoE Gemma4 shape lacked, so quantized V retires. MTP stays off:
|
||||
# this GGUF carries no MTP tensors at all (verified from its tensor-name
|
||||
# list), so there is no in-file head to draft with. Verbosity 4 ships so the
|
||||
# VRAM-fit forensics (KV layout, buffer sizes, graph capture) land in the journal.
|
||||
#
|
||||
# Architecture: qwen35 — dense 9B, 32 layers with full_attention_interval 4
|
||||
# (8 full-attention layers at 3,7,...,31 + 24 Gated DeltaNet linear layers),
|
||||
# 16 heads / 4 KV heads × 256 dim, native 262,144, vocab 248,320. The KV pool
|
||||
# scales with the 8 full layers: 32 KiB/token at f16 K + f16 V (12.25 GiB at
|
||||
# the 401,408 pool); the DeltaNet layers cost a fixed recurrent state per
|
||||
# slot. Feasibility is priced from the MiMo-V2.6-Distill-Qwen-9B boots (same
|
||||
# geometry, same card, same shape): 21,276/2,849 at ubatch 1024, 21,846/2,279
|
||||
# at ubatch 2048 — the boot ledger, not arithmetic, is the referee.
|
||||
# Vision comes from the publisher's own BF16 projector (the model is natively
|
||||
# multimodal), so --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is
|
||||
# the documented text-only opt-out.
|
||||
#
|
||||
# Sampling: llama.cpp takes it per request (the fleet harness sends the
|
||||
# publisher recipe from models.yml: thinking temp 0.6 / top_p 0.95 / top_k 20).
|
||||
#
|
||||
# Tunables come from frank-zealot.env unless FRANK_ZEALOT_ENV_FILE overrides.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_ZEALOT_ENV_FILE:-$SCRIPT_DIR/frank-zealot.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -------------------
|
||||
: "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
||||
: "${MODEL_FILE:=/home/dank/models/Qwen35-9B-HauhauCS-Q6K/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K.gguf}"
|
||||
: "${ALIAS:=frank-zealot}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7771}"
|
||||
# ctx-size is the POOL llama.cpp divides across slots: 401,408 / 2 = 200,704
|
||||
# per conversation, inside the model's trained 262,144. The harness
|
||||
# contextWindow MUST quote the PER-SLOT number — a longer prompt is answered
|
||||
# with a 400, never truncated.
|
||||
: "${CTX_SIZE:=401408}"
|
||||
: "${PARALLEL:=2}"
|
||||
# Compute-buffer geometry. Ubatch is a prefill knob (decode never moves).
|
||||
# 2048 is the measured CUDA optimum (MiMo era, same arch/shape: +4.6 % prefill
|
||||
# at 8.9k depth, decode flat, 616 MiB). The logical batch is VRAM- and
|
||||
# throughput-neutral at a fixed ubatch.
|
||||
: "${BATCH_SIZE:=4096}"
|
||||
: "${UBATCH_SIZE:=2048}"
|
||||
# Exact (unquantized) KV: f16 K + f16 V, restoring the MiMo-era contract that
|
||||
# the 26B Gemma4 shape could not afford. 32 KiB/token at this geometry
|
||||
# (8 full-attention layers × 4 KV heads × 256 dim); 12.25 GiB at the 401,408
|
||||
# pool. Explicit either way so the contract is visible in the argv, and
|
||||
# --flash-attn stays required for any quantized-V fallback (q8_0 = llama.cpp's
|
||||
# FP8 class; no true fp8 KV dtype exists in this build).
|
||||
: "${CACHE_TYPE_K:=f16}"
|
||||
: "${CACHE_TYPE_V:=f16}"
|
||||
# Speculative decoding is OFF: this Q6_K file carries NO nextn/mtp tensors, so
|
||||
# there is no in-file head (the law: MTP is a property of the file, not of the
|
||||
# flags). SPEC_TYPE=none strips BOTH speculative flags (flags move as a set);
|
||||
# naming a drafter requires SPEC_DRAFT_FILE alongside SPEC_TYPE — the Gemma4
|
||||
# assistant on disk is arch gemma4-assistant and cannot drafter-match qwen35.
|
||||
: "${SPEC_TYPE:=none}"
|
||||
: "${SPEC_DRAFT_N_MAX:=3}"
|
||||
: "${SPEC_DRAFT_FILE:=}"
|
||||
: "${NGL:=99}"
|
||||
# Log verbosity threshold (llama.cpp -lv). 3 is the engine default (info);
|
||||
# 4 adds debug detail (KV layout, buffer sizes, graph capture) which is what
|
||||
# the VRAM-fit forensics need. Levels above 4 flood the journal.
|
||||
: "${VERBOSITY:=3}"
|
||||
# Vision projector. No colon on purpose: an explicitly empty value means
|
||||
# "text-only, on purpose" and must NOT be replaced by the default path.
|
||||
: "${MMPROJ_FILE=/home/dank/models/Qwen35-9B-HauhauCS-Q6K/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf}"
|
||||
|
||||
# Fail loudly rather than half-loading: a missing file is a download that
|
||||
# has not finished, and Restart=always would turn it into a crash loop that
|
||||
# reads like a runtime fault.
|
||||
for path in "$LLAMA_BIN" "$MODEL_FILE"; do
|
||||
[[ -e "$path" ]] || { echo "frank-zealot: missing $path" >&2; exit 1; }
|
||||
done
|
||||
EXTRA_ARGS=()
|
||||
if [[ -n "$MMPROJ_FILE" ]]; then
|
||||
[[ -e "$MMPROJ_FILE" ]] || { echo "frank-zealot: missing $MMPROJ_FILE" >&2; exit 1; }
|
||||
EXTRA_ARGS+=(--mmproj "$MMPROJ_FILE")
|
||||
fi
|
||||
|
||||
# Speculative flags move as a set: they reach the argv ONLY when a drafter
|
||||
# is named. SPEC_TYPE=none (the shipped contract) strips both — a no-spec
|
||||
# baseline must not carry half a spec pair.
|
||||
if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then
|
||||
EXTRA_ARGS+=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
|
||||
if [[ -n "$SPEC_DRAFT_FILE" ]]; then
|
||||
[[ -e "$SPEC_DRAFT_FILE" ]] || { echo "frank-zealot: missing $SPEC_DRAFT_FILE" >&2; exit 1; }
|
||||
EXTRA_ARGS+=(--spec-draft-model "$SPEC_DRAFT_FILE")
|
||||
fi
|
||||
fi
|
||||
|
||||
# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
|
||||
# open after boot; make sure it exists before the server needs it.
|
||||
nvidia-modprobe -u -c=0 2>/dev/null || true
|
||||
|
||||
# Flash attention stays pinned on: it is REQUIRED for quantized V (q8_0),
|
||||
# and the build carries GGML_CUDA_FA_ALL_QUANTS=ON (since 2026-09-17), so
|
||||
# the q8_0 KV contract in frank-zealot.env is a real path — without this
|
||||
# flag or that build option the engine silently falls back and the
|
||||
# contract becomes a lie.
|
||||
ARGS=(
|
||||
--model "$MODEL_FILE"
|
||||
--alias "$ALIAS"
|
||||
--host "$BIND_ADDR"
|
||||
--port "$LISTEN_PORT"
|
||||
--ctx-size "$CTX_SIZE"
|
||||
--parallel "$PARALLEL"
|
||||
--batch-size "$BATCH_SIZE"
|
||||
--ubatch-size "$UBATCH_SIZE"
|
||||
--n-gpu-layers "$NGL"
|
||||
--flash-attn on
|
||||
--cache-type-k "$CACHE_TYPE_K"
|
||||
--cache-type-v "$CACHE_TYPE_V"
|
||||
--log-verbosity "$VERBOSITY"
|
||||
--jinja
|
||||
--reasoning auto
|
||||
--reasoning-format deepseek
|
||||
)
|
||||
exec "$LLAMA_BIN" "${ARGS[@]}" ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}
|
||||
Executable
+556
@@ -0,0 +1,556 @@
|
||||
#!/usr/bin/env python3
|
||||
"""stack-qwen38-official-fp8.py — reshape the official Qwen/Qwen3.8-Flash-Next-FP8
|
||||
checkpoint into the form the shipped radiance qwen4exp plugin maps.
|
||||
|
||||
Run INSIDE the staging dir's tree (python3 + numpy on the host; data never
|
||||
leaves the box). Two transforms, both VALUE-PRESERVING:
|
||||
|
||||
1. EXPERTS: the shipped plugin's stacked layout names ONE tensor per family
|
||||
(`mlp.experts.gate_up_proj` [E, 2F, H], `mlp.experts.down_proj` [E, H, F])
|
||||
plus a `<family>_scale_inv` plane at the same index (NO `.weight` in the
|
||||
stacked name — scale_name() appends _scale_inv to the tensor name verbatim),
|
||||
while the official FP8 repo ships per-expert unfused tensors
|
||||
(`experts.<e>.{gate,up,down}_proj.weight` + scale). Stacking is pure byte
|
||||
concatenation along dim 0 — gate rows
|
||||
first inside each expert's gate_up slab, which is the order the block
|
||||
wants (rad_block_moe_fp8.h: "a stacked gate_up is ALREADY FUSED in the
|
||||
order this block wants"). No requantisation: bytes are moved, not decoded.
|
||||
|
||||
2. PLE TABLE: the plugin's fp8 branch wants the table's codes beside a
|
||||
ONE-ELEMENT bf16 scale plane (`blk.%d.ple_ngram.scale`, view {1}); a
|
||||
per-shard-blocked scale can never fit that view, and `rtn` re-encoding the
|
||||
RAW scaleless codes would overflow (v / 1.99e-4 exceeds e4m3's 448 max).
|
||||
So staging decodes the scaleless shards with the checkpoint's OWN bf16
|
||||
scale into a bf16 table (exact: every e4m3 code value and the scale are
|
||||
exactly representable in bf16 to within the code's own 2^-3 grid), and the
|
||||
convert's rtn rule re-encodes at that same scale — the nearest-code
|
||||
distance is 2^-3 while the bf16 rounding perturbs by 2^-9, so every
|
||||
original code is restored exactly and the container gets the checkpoint's
|
||||
own codes plus the one-element scale plane.
|
||||
|
||||
The convert then runs with NO recipe except:
|
||||
blk.*.ple_ngram.weight=rtn:codes=fp8_e4m3,block=*x*,scale=bf16,rule=fixed,scale_value=<s>
|
||||
with <s> read from the checkpoint and written to ple_scale.txt.
|
||||
|
||||
Idempotent: re-runs skip when the staged index already carries the stacked
|
||||
names, the merged bf16 shard_0, and split_ngram_parts == 1. --force redoes.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import mmap
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
import numpy as np
|
||||
F8, BF16 = "F8_E4M3", "BF16"
|
||||
|
||||
|
||||
def log(msg):
|
||||
print(msg, flush=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- safetensors
|
||||
|
||||
def parse_header(path):
|
||||
"""Return (header_len, {name: (dtype, shape, data_offsets)})."""
|
||||
with open(path, "rb") as fh:
|
||||
(hlen,) = struct.unpack("<Q", fh.read(8))
|
||||
hdr = json.loads(fh.read(hlen))
|
||||
return hlen, hdr
|
||||
|
||||
|
||||
def write_safetensors(path, tensors, data_iter):
|
||||
"""tensors: [(name, dtype, shape, nbytes)]; data_iter yields bytes per tensor."""
|
||||
hdr = {}
|
||||
off = 0
|
||||
for name, dtype, shape, nbytes in tensors:
|
||||
hdr[name] = {"dtype": dtype, "shape": list(shape),
|
||||
"data_offsets": [off, off + nbytes]}
|
||||
off += nbytes
|
||||
blob = json.dumps(hdr, separators=(",", ":")).encode("utf-8")
|
||||
pad = (-len(blob)) % 8 # keep the data start 8-byte aligned
|
||||
blob += b" " * pad
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(struct.pack("<Q", len(blob)))
|
||||
fh.write(blob)
|
||||
for _, _, _, nbytes in tensors:
|
||||
fh.write(next(data_iter).tobytes())
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
return 8 + len(blob) + sum(t[3] for t in tensors)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- preflight
|
||||
|
||||
def build_meta(stage, index):
|
||||
"""name -> (file, global_byte_offset, nbytes, dtype, shape) for every tensor."""
|
||||
files = sorted(set(index["weight_map"].values()))
|
||||
meta = {}
|
||||
for f in files:
|
||||
path = os.path.join(stage, f)
|
||||
hlen, hdr = parse_header(path)
|
||||
for name, e in hdr.items():
|
||||
if name == "__metadata__":
|
||||
continue
|
||||
b0, b1 = e["data_offsets"]
|
||||
meta[name] = (f, 8 + hlen + b0, b1 - b0, e["dtype"], e["shape"])
|
||||
return meta
|
||||
|
||||
|
||||
def e4m3_lut():
|
||||
lut = np.zeros(256, np.float32)
|
||||
for b in range(256):
|
||||
sign = -1.0 if b & 0x80 else 1.0
|
||||
e = (b >> 3) & 0xF
|
||||
m = b & 0x7
|
||||
if e == 15 and m == 7:
|
||||
v = np.float32(np.nan) # OCP e4m3fn: only 0x7F/0xFF are NaN
|
||||
elif e == 0:
|
||||
v = sign * (m / 8.0) * 2.0 ** -6
|
||||
else:
|
||||
v = sign * (1.0 + m / 8.0) * 2.0 ** (e - 7)
|
||||
lut[b] = v
|
||||
assert lut[0x38] == 1.0 and lut[0x7E] == 448.0
|
||||
assert np.isnan(lut[0x7F]) and lut[0x00] == 0.0
|
||||
assert lut[0x08] == 2.0 ** -6 and lut[0x80] == 0.0
|
||||
return lut
|
||||
|
||||
|
||||
def f32_to_bf16_rne(v):
|
||||
"""float32 array -> uint16 bf16 bits, round-to-nearest-even."""
|
||||
q = np.ascontiguousarray(v, dtype=np.float32)
|
||||
q[~np.isfinite(q)] = 0.0
|
||||
bits = q.view(np.uint32)
|
||||
r = bits + 0x7FFF + ((bits >> 16) & 1)
|
||||
return (r >> 16).astype(np.uint16)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- workers
|
||||
|
||||
def stack_prefix(job):
|
||||
"""Byte-stack one layer's 512 experts into a 4-tensor safetensors file."""
|
||||
(stage, pre, idx, n_e, shapes, meta) = job
|
||||
(f2, h) = shapes["ffn2_h"] # 2*ffn, n_embd
|
||||
(f,) = shapes["ffn"] # ffn
|
||||
gs, ds = shapes["gs"], shapes["ds"] # scale shapes [5,20] and [20,5]
|
||||
|
||||
# byte widths: fp8 is 1 B/element, bf16 2 B; the fused gate_up scale is
|
||||
# the gate and up scale planes concatenated (gate rows first)
|
||||
g_bytes = (f2 // 2) * h
|
||||
gs_bytes = gs[0] * gs[1] * 2 # ONE projection's scale plane
|
||||
gus_bytes = 2 * gs_bytes # fused scale per expert
|
||||
dn_bytes = h * f
|
||||
ds_bytes = ds[0] * ds[1] * 2
|
||||
|
||||
gu = np.empty(n_e * f2 * h, np.uint8)
|
||||
gus = np.empty(n_e * gus_bytes, np.uint8)
|
||||
dn = np.empty(n_e * dn_bytes, np.uint8)
|
||||
dns = np.empty(n_e * ds_bytes, np.uint8)
|
||||
|
||||
mmaps = {}
|
||||
|
||||
def view(name):
|
||||
f_, off, ln, dt, shp = meta[name]
|
||||
if f_ not in mmaps:
|
||||
fh = open(os.path.join(stage, f_), "rb")
|
||||
mmaps[f_] = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)
|
||||
return memoryview(mmaps[f_])[off:off + ln], (dt, shp)
|
||||
|
||||
def need(name, dt, shp):
|
||||
mv, (got_dt, got_shp) = view(name)
|
||||
assert got_dt == dt and list(got_shp) == list(shp), f"{name}: {got_dt}{got_shp}"
|
||||
return mv
|
||||
|
||||
for e in range(n_e):
|
||||
base = f"{pre}.mlp.experts.{e}"
|
||||
slab = gu[e * f2 * h:(e + 1) * f2 * h]
|
||||
slab[:g_bytes] = need(f"{base}.gate_proj.weight", F8, [f2 // 2, h])
|
||||
slab[g_bytes:] = need(f"{base}.up_proj.weight", F8, [f2 // 2, h])
|
||||
sslab = gus[e * gus_bytes:(e + 1) * gus_bytes]
|
||||
sslab[:gs_bytes] = need(f"{base}.gate_proj.weight_scale_inv", BF16, gs)
|
||||
sslab[gs_bytes:] = need(f"{base}.up_proj.weight_scale_inv", BF16, gs)
|
||||
dn[e * dn_bytes:(e + 1) * dn_bytes] = need(f"{base}.down_proj.weight", F8, [h, f])
|
||||
dns[e * ds_bytes:(e + 1) * ds_bytes] = need(f"{base}.down_proj.weight_scale_inv", BF16, ds)
|
||||
|
||||
for mm in mmaps.values():
|
||||
mm.close()
|
||||
|
||||
# byte-identity check on two sampled experts (first and last), all four
|
||||
# tensors — the stack IS the checkpoint bytes, so they must compare equal
|
||||
for e in (0, n_e - 1):
|
||||
base = f"{pre}.mlp.experts.{e}"
|
||||
s = e * f2 * h
|
||||
assert bytes(gu[s:s + g_bytes]) == bytes(stack_src(f"{base}.gate_proj.weight", meta, stage))
|
||||
assert bytes(gu[s + g_bytes:s + f2 * h]) == bytes(stack_src(f"{base}.up_proj.weight", meta, stage))
|
||||
t = e * gus_bytes
|
||||
assert bytes(gus[t:t + gs_bytes]) == bytes(stack_src(f"{base}.gate_proj.weight_scale_inv", meta, stage))
|
||||
assert bytes(gus[t + gs_bytes:t + gus_bytes]) == bytes(stack_src(f"{base}.up_proj.weight_scale_inv", meta, stage))
|
||||
d = e * dn_bytes
|
||||
assert bytes(dn[d:d + dn_bytes]) == bytes(stack_src(f"{base}.down_proj.weight", meta, stage))
|
||||
u = e * ds_bytes
|
||||
assert bytes(dns[u:u + ds_bytes]) == bytes(stack_src(f"{base}.down_proj.weight_scale_inv", meta, stage))
|
||||
|
||||
out = os.path.join(stage, f"stacked.{idx:03d}.safetensors")
|
||||
tensors = [
|
||||
(f"{pre}.mlp.experts.gate_up_proj", F8, [n_e, f2, h], n_e * f2 * h),
|
||||
(f"{pre}.mlp.experts.gate_up_proj_scale_inv", BF16,
|
||||
[n_e, gs[0] * 2, gs[1]], n_e * gus_bytes),
|
||||
(f"{pre}.mlp.experts.down_proj", F8, [n_e, h, f], n_e * h * f),
|
||||
(f"{pre}.mlp.experts.down_proj_scale_inv", BF16,
|
||||
[n_e, ds[0], ds[1]], n_e * ds_bytes),
|
||||
]
|
||||
bufs = iter([gu, gus, dn, dns])
|
||||
written = write_safetensors(out, tensors, bufs)
|
||||
return (pre, written)
|
||||
|
||||
|
||||
def stack_src(name, meta, stage):
|
||||
"""Re-open a source tensor for the sampled-expert check."""
|
||||
f_, off, ln, dt, shp = meta[name]
|
||||
fh = open(os.path.join(stage, f_), "rb")
|
||||
mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)
|
||||
fh.close() # mmap keeps the mapping alive
|
||||
return mm[off:off + ln]
|
||||
|
||||
|
||||
def fill_ple_shard(job):
|
||||
"""Decode one fp8 shard with the table scale, write bf16 rows into the mmap."""
|
||||
(src_path, src_off, src_rows, dst_path, dst_off, scale_f32,
|
||||
cols, chunk_rows) = job
|
||||
lut = e4m3_lut()
|
||||
with open(src_path, "rb") as fh:
|
||||
sm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)
|
||||
with open(dst_path, "r+b") as dfh:
|
||||
dm = mmap.mmap(dfh.fileno(), 0, access=mmap.ACCESS_WRITE)
|
||||
for r0 in range(0, src_rows, chunk_rows):
|
||||
r1 = min(r0 + chunk_rows, src_rows)
|
||||
codes = np.frombuffer(sm, dtype=np.uint8,
|
||||
count=(r1 - r0) * cols,
|
||||
offset=src_off + r0 * cols)
|
||||
q = lut.take(codes)
|
||||
del codes # release the mmap export before sm.close()
|
||||
q[~np.isfinite(q)] = 0.0
|
||||
q *= np.float32(scale_f32)
|
||||
packed = f32_to_bf16_rne(q)
|
||||
dm[dst_off + r0 * cols * 2: dst_off + r1 * cols * 2] = packed.tobytes()
|
||||
dm.flush()
|
||||
os.fsync(dfh.fileno())
|
||||
dm.close()
|
||||
sm.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------- main
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("--src", default=None,
|
||||
help="raw checkpoint dir (documentation only; unused)")
|
||||
ap.add_argument("--stage", required=True, help="staging dir to reshape in place")
|
||||
ap.add_argument("--force", action="store_true", help="redo even if already staged")
|
||||
ap.add_argument("--workers", type=int, default=min(8, os.cpu_count() or 1))
|
||||
args = ap.parse_args()
|
||||
stage = args.stage
|
||||
t0 = time.time()
|
||||
|
||||
index_path = os.path.join(stage, "model.safetensors.index.json")
|
||||
cfg_path = os.path.join(stage, "config.json")
|
||||
if not (os.path.isfile(index_path) and os.path.isfile(cfg_path)):
|
||||
log(f"ERROR: {stage} is not a staged checkpoint (missing index/config)")
|
||||
return 2
|
||||
with open(index_path) as fh:
|
||||
index = json.load(fh)
|
||||
with open(cfg_path) as fh:
|
||||
cfg = json.load(fh)
|
||||
wm = index["weight_map"]
|
||||
|
||||
meta = build_meta(stage, index)
|
||||
|
||||
# ---- collect prefixes -------------------------------------------------
|
||||
pre_re = re.compile(r"^(?P<pre>.+)\.mlp\.experts\.(?P<e>\d+)\.gate_proj\.weight$")
|
||||
pres, n_e = {}, 0
|
||||
for k in wm:
|
||||
m = pre_re.match(k)
|
||||
if m:
|
||||
pres.setdefault(m.group("pre"), set()).add(int(m.group("e")))
|
||||
n_e = max(n_e, int(m.group("e")) + 1)
|
||||
prefixes = sorted(pres)
|
||||
if not prefixes:
|
||||
log("ERROR: no per-expert tensors found in the staged index")
|
||||
return 2
|
||||
mtp = [p for p in prefixes if p.startswith("mtp.")]
|
||||
trunk = [p for p in prefixes if p.startswith("model.language_model.layers.")]
|
||||
log(f"expert prefixes: {len(prefixes)} (trunk {len(trunk)}, mtp {len(mtp)}), "
|
||||
f"{n_e} experts each")
|
||||
|
||||
# shapes from expert 0 of the first prefix
|
||||
pre0 = prefixes[0]
|
||||
shapes = {}
|
||||
gdt, gsh = meta[f"{pre0}.mlp.experts.0.gate_proj.weight"][3:5]
|
||||
ddt, dsh = meta[f"{pre0}.mlp.experts.0.down_proj.weight"][3:5]
|
||||
gsd, gssh = meta[f"{pre0}.mlp.experts.0.gate_proj.weight_scale_inv"][3:5]
|
||||
dsd, dssh = meta[f"{pre0}.mlp.experts.0.down_proj.weight_scale_inv"][3:5]
|
||||
scale_law = (gdt == F8 and ddt == F8 and gsd == BF16 and dsd == BF16
|
||||
and list(gssh) == [(gsh[0] + 127) // 128, (gsh[1] + 127) // 128]
|
||||
and list(dssh) == [(dsh[0] + 127) // 128, (dsh[1] + 127) // 128])
|
||||
if not scale_law:
|
||||
log(f"ERROR: unexpected expert shapes: gate {gdt}{gsh} scale {gsd}{gssh} "
|
||||
f"down {ddt}{dsh} scale {dsd}{dssh} (scale law 128x128)")
|
||||
return 2
|
||||
shapes = {"ffn2_h": (gsh[0] * 2, gsh[1]), "ffn": (dsh[1],),
|
||||
"gs": tuple(gssh), "ds": tuple(dssh)}
|
||||
f2, h = shapes["ffn2_h"]
|
||||
f = shapes["ffn"][0]
|
||||
log(f"shapes: gate_up [{n_e},{f2},{h}] F8, down [{n_e},{h},{f}] F8, "
|
||||
f"scales {gssh}x2 / {dssh} bf16")
|
||||
|
||||
# ---- PLE table --------------------------------------------------------
|
||||
ple_re = re.compile(r"^(?P<pre>.+)\.ple\.ple_embedding\.ngram_embedding\.shard_(?P<i>\d+)\.weight$")
|
||||
ple_pre, shard_ids = None, {}
|
||||
for k in wm:
|
||||
m = ple_re.match(k)
|
||||
if m:
|
||||
ple_pre = m.group("pre")
|
||||
shard_ids[int(m.group("i"))] = k
|
||||
if ple_pre is None:
|
||||
log("ERROR: no PLE table shards found")
|
||||
return 2
|
||||
n_shards = len(shard_ids)
|
||||
if sorted(shard_ids) != list(range(n_shards)):
|
||||
log(f"ERROR: PLE shard ids not contiguous 0..{n_shards - 1}")
|
||||
return 2
|
||||
sc_name = f"{ple_pre}.ple.ple_embedding.ngram_embedding.weight_scale"
|
||||
sdt, ssh, sln = meta[sc_name][3], meta[sc_name][4], meta[sc_name][2]
|
||||
if sdt != BF16 or list(ssh) != [1]:
|
||||
log(f"ERROR: PLE scale is {sdt}{ssh}, expected BF16 [1]")
|
||||
return 2
|
||||
s_bits = struct.unpack("<H", bytes(view_src_tuple(meta, sc_name, stage)))[0]
|
||||
scale_f32 = struct.unpack("<f", struct.pack("<I", s_bits << 16))[0]
|
||||
log(f"PLE: {ple_pre}, {n_shards} shards, scale bf16 = {scale_f32!r}")
|
||||
|
||||
rows = [meta[shard_ids[i]][4][0] for i in range(n_shards)]
|
||||
total_rows = sum(rows)
|
||||
hd = meta[shard_ids[0]][4][1]
|
||||
if any(meta[shard_ids[i]][4][1] != hd for i in range(n_shards)):
|
||||
log("ERROR: PLE shards disagree on HD")
|
||||
return 2
|
||||
if hd % 128 != 0 and hd != 160:
|
||||
log(f"WARNING: PLE HD={hd} is neither 160 nor a multiple of 128")
|
||||
|
||||
# ---- idempotence gate -------------------------------------------------
|
||||
stacked_names = [f"{p}.mlp.experts.gate_up_proj" for p in prefixes]
|
||||
merged_name = f"{ple_pre}.ple.ple_embedding.ngram_embedding.shard_0.weight"
|
||||
already = (all(n in wm for n in stacked_names)
|
||||
and merged_name in wm
|
||||
and find_split(cfg) is not None and read_split(cfg) == 1)
|
||||
if already and not args.force:
|
||||
log("already staged (stacked names, merged shard_0 and split_ngram_parts=1 "
|
||||
"all present) — nothing to do")
|
||||
return 0
|
||||
|
||||
# ---- disk gate --------------------------------------------------------
|
||||
stacked_bytes = len(prefixes) * n_e * (f2 * h + h * f)
|
||||
ple_bytes = total_rows * hd * 2
|
||||
need = stacked_bytes + ple_bytes
|
||||
st = os.statvfs(stage)
|
||||
avail = st.f_bavail * st.f_frsize
|
||||
if avail < need * 1.05:
|
||||
log(f"ERROR: {avail / 2**30:.1f} GiB free < {need / 2**30:.1f} GiB needed (+5%)")
|
||||
return 2
|
||||
log(f"disk: {avail / 2**30:.1f} GiB free, need {need / 2**30:.1f} GiB "
|
||||
f"(experts {stacked_bytes / 2**30:.1f}, ple {ple_bytes / 2**30:.1f})")
|
||||
|
||||
# ---- stage experts ----------------------------------------------------
|
||||
jobs = []
|
||||
for idx, pre in enumerate(prefixes):
|
||||
jmeta = {n: meta[n] for n in meta if n.startswith(f"{pre}.mlp.experts.")}
|
||||
jobs.append((stage, pre, idx, n_e, shapes, jmeta))
|
||||
ctx = mp.get_context("fork")
|
||||
written_total = 0
|
||||
with ctx.Pool(args.workers) as pool:
|
||||
for pre, nbytes in pool.imap_unordered(stack_prefix, jobs):
|
||||
written_total += nbytes
|
||||
log(f" stacked {pre} ({nbytes / 2**30:.2f} GiB)")
|
||||
log(f"experts stacked: {written_total / 2**30:.2f} GiB in {len(prefixes)} files")
|
||||
|
||||
# ---- stage PLE --------------------------------------------------------
|
||||
# The file name sorts BEFORE the model-000NN shards ON PURPOSE: the reader
|
||||
# opens every index-named file in sorted order and emplaces EVERY tensor
|
||||
# it physically holds, first-wins (checkpoint.cpp add_shard). The raw
|
||||
# shards carry tensors of the SAME name as the merged one
|
||||
# (...ple.ple_embedding.ngram_embedding.shard_0.weight, [2500012,160]
|
||||
# fp8), so ple.merged.safetensors lost the race and the plan read one raw
|
||||
# shard as the codes. "00-" wins the sort and the name.
|
||||
out_name = "00-ple.merged.safetensors"
|
||||
out_path = os.path.join(stage, out_name)
|
||||
data_bytes = total_rows * hd * 2
|
||||
hdr = {merged_name: {"dtype": BF16, "shape": [total_rows, hd],
|
||||
"data_offsets": [0, data_bytes]}}
|
||||
blob = json.dumps(hdr, separators=(",", ":")).encode("utf-8")
|
||||
pad = (-len(blob)) % 8
|
||||
blob += b" " * pad
|
||||
hlen = len(blob)
|
||||
with open(out_path, "wb") as fh:
|
||||
fh.write(struct.pack("<Q", hlen))
|
||||
fh.write(blob)
|
||||
fh.truncate(8 + hlen + data_bytes)
|
||||
jobs = []
|
||||
row0 = 0
|
||||
for i in range(n_shards):
|
||||
f_, off, ln, _, shp = meta[shard_ids[i]]
|
||||
jobs.append((os.path.join(stage, f_), off, shp[0], out_path,
|
||||
8 + hlen + row0 * hd * 2, scale_f32, hd, 131072))
|
||||
row0 += shp[0]
|
||||
assert row0 == total_rows
|
||||
with ctx.Pool(args.workers) as pool:
|
||||
pool.map(fill_ple_shard, jobs)
|
||||
log(f"PLE merged: {data_bytes / 2**30:.2f} GiB bf16 "
|
||||
f"({total_rows} rows x {hd} cols)")
|
||||
|
||||
# ---- post-verify ------------------------------------------------------
|
||||
verify(stage, prefixes, n_e, shapes, meta, shard_ids, rows, hd,
|
||||
scale_f32, out_path, hlen)
|
||||
|
||||
# ---- rewrite index + config -------------------------------------------
|
||||
new_wm = {k: v for k, v in wm.items() if k not in set(shard_ids.values())}
|
||||
for idx, pre in enumerate(prefixes):
|
||||
fname = f"stacked.{idx:03d}.safetensors"
|
||||
for t in (f"{pre}.mlp.experts.gate_up_proj",
|
||||
f"{pre}.mlp.experts.gate_up_proj_scale_inv",
|
||||
f"{pre}.mlp.experts.down_proj",
|
||||
f"{pre}.mlp.experts.down_proj_scale_inv"):
|
||||
new_wm[t] = fname
|
||||
new_wm[merged_name] = out_name
|
||||
total = sum(meta_size(meta, k) for k in new_wm
|
||||
if k in meta) # originals
|
||||
# new tensors' sizes
|
||||
total += written_total + data_bytes
|
||||
index["weight_map"] = new_wm
|
||||
if "metadata" in index and isinstance(index["metadata"], dict):
|
||||
index["metadata"]["total_size"] = total
|
||||
# ATOMIC REPLACE, never open(path, "w"): the stage's index is a symlink
|
||||
# to the RAW checkpoint (staging symlinks everything but config.json) —
|
||||
# open("w") would rewrite the raw file through the link. os.replace
|
||||
# swaps the link itself for a real file; the raw tree stays pristine.
|
||||
for path, payload in ((index_path, json.dumps(index, indent=2)),
|
||||
(cfg_path, json.dumps(cfg, indent=2))):
|
||||
tmp = path + ".tmp"
|
||||
with open(tmp, "w") as fh:
|
||||
fh.write(payload)
|
||||
os.replace(tmp, path)
|
||||
n = set_split(cfg, 1)
|
||||
log(f"config: split_ngram_parts=1 written at {n} location(s)")
|
||||
|
||||
with open(os.path.join(stage, "ple_scale.txt"), "w") as fh:
|
||||
fh.write(repr(scale_f32) + "\n")
|
||||
|
||||
dt = time.time() - t0
|
||||
log(f"staged in {dt / 60:.1f} min: {written_total / 2**30:.1f} GiB experts + "
|
||||
f"{data_bytes / 2**30:.1f} GiB PLE; scale {scale_f32!r} -> ple_scale.txt")
|
||||
return 0
|
||||
|
||||
|
||||
def meta_size(meta, name):
|
||||
_, _, ln, _, _ = meta[name]
|
||||
return ln
|
||||
|
||||
|
||||
def view_src_tuple(meta, name, stage):
|
||||
f_, off, ln, _, _ = meta[name]
|
||||
with open(os.path.join(stage, f_), "rb") as fh:
|
||||
fh.seek(off)
|
||||
return fh.read(ln)
|
||||
|
||||
|
||||
def find_split(cfg):
|
||||
"""Locate split_ngram_parts keys (recursive); return list of paths."""
|
||||
hits = []
|
||||
|
||||
def walk(d, path):
|
||||
if isinstance(d, dict):
|
||||
for k, v in d.items():
|
||||
if k == "split_ngram_parts":
|
||||
hits.append(path + [k])
|
||||
walk(v, path + [k])
|
||||
walk(cfg, [])
|
||||
return hits
|
||||
|
||||
|
||||
def read_split(cfg):
|
||||
hits = find_split(cfg)
|
||||
if not hits:
|
||||
return None
|
||||
d = cfg
|
||||
for k in hits[0][:-1]:
|
||||
d = d[k]
|
||||
return d[hits[0][-1]]
|
||||
|
||||
|
||||
def set_split(cfg, value):
|
||||
hits = find_split(cfg)
|
||||
for path in hits:
|
||||
d = cfg
|
||||
for k in path[:-1]:
|
||||
d = d[k]
|
||||
d[path[-1]] = value
|
||||
if not hits: # plugin's meta_int2 falls back to the top-level key
|
||||
cfg["split_ngram_parts"] = value
|
||||
hits.append(["split_ngram_parts"])
|
||||
return hits
|
||||
|
||||
|
||||
def verify(stage, prefixes, n_e, shapes, meta, shard_ids, rows, hd,
|
||||
scale_f32, out_path, hlen):
|
||||
"""Cheap strong checks: headers parse, sizes match, sampled bytes identical."""
|
||||
f2, h = shapes["ffn2_h"]
|
||||
f = shapes["ffn"][0]
|
||||
gs, ds = shapes["gs"], shapes["ds"]
|
||||
# headers of every new file
|
||||
for idx, pre in enumerate(prefixes):
|
||||
path = os.path.join(stage, f"stacked.{idx:03d}.safetensors")
|
||||
hlen2, hdr = parse_header(path)
|
||||
want = {
|
||||
f"{pre}.mlp.experts.gate_up_proj": (F8, [n_e, f2, h]),
|
||||
f"{pre}.mlp.experts.gate_up_proj_scale_inv":
|
||||
(BF16, [n_e, gs[0] * 2, gs[1]]),
|
||||
f"{pre}.mlp.experts.down_proj": (F8, [n_e, h, f]),
|
||||
f"{pre}.mlp.experts.down_proj_scale_inv": (BF16, [n_e, ds[0], ds[1]]),
|
||||
}
|
||||
for name, (dt, shp) in want.items():
|
||||
e = hdr[name]
|
||||
assert e["dtype"] == dt and list(e["shape"]) == shp, name
|
||||
assert e["data_offsets"][1] - e["data_offsets"][0] == \
|
||||
int(np.prod(shp)) * (1 if dt == F8 else 2), name
|
||||
_, phdr = parse_header(out_path)
|
||||
e = phdr[next(iter(phdr))]
|
||||
assert e["dtype"] == BF16 and list(e["shape"]) == [sum(rows), hd]
|
||||
|
||||
# sampled PLE rows: recompute and compare bytes exactly
|
||||
lut = e4m3_lut()
|
||||
out_off = 8 + hlen
|
||||
for i in (0, len(shard_ids) // 2, len(shard_ids) - 1):
|
||||
name = shard_ids[i]
|
||||
f_, off, ln, _, shp = meta[name]
|
||||
with open(os.path.join(stage, f_), "rb") as fh:
|
||||
mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)
|
||||
for r in (0, shp[0] - 1):
|
||||
codes = np.frombuffer(mm, dtype=np.uint8, count=hd,
|
||||
offset=off + r * hd)
|
||||
q = lut.take(codes) * np.float32(scale_f32)
|
||||
del codes # release the mmap export before mm.close()
|
||||
want = f32_to_bf16_rne(q).tobytes()
|
||||
got_row = sum(rows[:i]) + r
|
||||
with open(out_path, "rb") as ofh:
|
||||
ofh.seek(out_off + got_row * hd * 2)
|
||||
got = ofh.read(hd * 2)
|
||||
assert got == want, f"PLE row {got_row} mismatch"
|
||||
mm.close()
|
||||
log("verify: new-file headers, sizes and sampled PLE rows all exact; "
|
||||
"expert slabs were byte-checked in-stack")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user