From 9f142b7eafd945668a02fe90dab368b7c33e2771 Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 30 Oct 2026 15:20:04 -0700 Subject: [PATCH] live: heartbeat sync 2026-10-30T15:20:04-07:00 (Refs #11) --- .../agenttrash-purger-invocation/SKILL.md | 31 +++---- .../SKILL.md | 49 +++++++++++ .../radiance-tap-arm-and-seal-verify/SKILL.md | 81 +++++++++++++++++++ .../radiance-w4-campaign/SKILL.md | 40 +++++---- .../SKILL.md | 34 ++++++++ .../serverAppConfig-eu5lc7uqff33/state.json | 2 +- .../architecture.md | 3 +- .../product.md | 2 +- .../state.json | 4 +- .../state.json | 4 + .../architecture.md | 3 + .../product.md | 3 + .../state.json | 10 +++ .../style.md | 0 14 files changed, 233 insertions(+), 33 deletions(-) create mode 100644 agent/managed-skills/radiance-calib-tap-produce-latest/SKILL.md create mode 100644 agent/managed-skills/radiance-tap-arm-and-seal-verify/SKILL.md create mode 100644 agent/managed-skills/upstream-engine-reference-into-vault/SKILL.md create mode 100644 agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/state.json create mode 100644 agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/architecture.md create mode 100644 agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/product.md create mode 100644 agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/state.json create mode 100644 agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/style.md diff --git a/agent/managed-skills/agenttrash-purger-invocation/SKILL.md b/agent/managed-skills/agenttrash-purger-invocation/SKILL.md index 6d5c90b..2164e93 100644 --- a/agent/managed-skills/agenttrash-purger-invocation/SKILL.md +++ b/agent/managed-skills/agenttrash-purger-invocation/SKILL.md @@ -1,9 +1,9 @@ --- name: agenttrash-purger-invocation -description: "Invoke the .AGENTTRASH purger (scripts/purge-agenttrash.sh) to clear staged deletions, and diagnose its two refusal traps: a false stale-lock error on a full filesystem, and $HOME refused on dankserver by podman-storage permission denials." +description: "Invoke the .AGENTTRASH purger (scripts/purge-agenttrash.sh) — plus its two refusal traps: ENOSPC masquerading as a stale lock, and an unreadable container-storage directory poisoning the whole root scan." --- -# Purging .AGENTTRASH — invocation and two refusal traps +# Purging .AGENTTRASH — invocation, and the two traps that make it refuse `scripts/purge-agenttrash.sh` (serverAppConfigs, merged in PR #79 `fad87aa`) is the tool form of the fleet's deletion policy: files land in `.AGENTTRASH`, and an approved purge runs this instead of being improvised per session. @@ -15,25 +15,28 @@ sh scripts/purge-agenttrash.sh --execute # perform the plan sh scripts/purge-agenttrash.sh --execute /some/root # scope to one root (default: $HOME) ``` -- `--quiet` silences per-line output; errors and the summary still print. `--log PATH` overrides the default `~/.AGENTTRASH-cleanup.log` (one dated line per run — written even for a dry run, so pass a scratch `--log` when you only want to look). +- `--quiet` silences per-line output; errors and the summary still print. `--log PATH` overrides the default `~/.AGENTTRASH-cleanup.log` (one dated line per run — written even for a dry run). - Every printable line is fixed at plan time in `$HOME/.agenttrash-purge.lock/plan`, so dry-run and `--execute` output cannot drift. Exit `0` ok · `1` any refusal or failure · `2` usage. -- Rule: a root's own `.AGENTTRASH` is *emptied* (the directory survives); deeper `.AGENTTRASH` trees are *removed*; symlinked or foreign-owned candidates are refused, never followed; `/` is refused outright. +- Deleting a big staged item by hand (e.g. `find -depth -delete`) is the fallback **only** when the tool cannot run at all — see trap 1 — and the operator's purge approval covers exactly that. -## Trap 1: a FULL filesystem makes it lie about a lock +## Trap 1: a full filesystem makes it lie about a lock (measured 2026-10-30) -The script acquires `~/.agenttrash-purge.lock` and writes a plan file **before** it scans. On a filesystem with 0 bytes free both writes fail `ENOSPC`, and the run dies with: +With **0 bytes free**, three invocations refused with ``` -purge-agenttrash: /home/…/.agenttrash-purge.lock exists and is not ours to reclaim +purge-agenttrash: refused /home/: exists and is not ours to reclaim ``` -while **no lock exists** (`find -maxdepth 2 -name '.agenttrash-purge.lock'` comes back empty, `pgrep -af purge-agenttrash` shows nothing). Do not hunt the phantom lock and do not delete anything to "reclaim" it: check `df -h` first. Free space by the purge's own intended action — delete the staged item the operator approved (a same-filesystem `mv` into `.AGENTTRASH` frees nothing; only deletion does) — then re-run the tool, which will clean the rest and log its summary. +and no lock directory existed anywhere. The script writes its lock *and* its plan file before scanning; both hit `ENOSPC`, and the failure surfaced as the lock-reclaim message. There is nothing to reclaim — **free space first** (delete the staged item, move it to another volume, `journalctl --vacuum-size=`), then run plan + `--execute`. Never hunt a phantom lock because the message says so; check `df` first. -## Trap 2: `$HOME` is always refused on dankserver +## Trap 2: an unreadable directory spoils the whole root -`sh scripts/purge-agenttrash.sh` on dankserver reports `refused /home/dank (find failed)` and exits 1 even though the plan is otherwise right, and `~/.AGENTTRASH-cleanup.log` records `refused=1` beside whatever it did manage. +`collect_root` turns any nonzero `find` status into a root refusal, so the plan is built from a partial scan and the root reports `refused=1` even though `find` printed every `.AGENTTRASH` it found. -- Cause: `find /home/dank -xdev -name .AGENTTRASH` exits 1 there because rootless podman storage contains UID-mismatched directories — `~/.local/share/containers/storage/overlay/*/diff/var/lib/mysql`, `…/etc/valkey`, `…/.config/calibre/plugins`, `…/var/cache/apt/archives/partial` — plus `~/Containers/romm/mariadb/*`. `collect_root` turns any nonzero find status into a root refusal, so a *partial* scan becomes no plan. -- Find still prints every real tree (14 on dankserver: `vLLM/`, `Containers/`, four under `serverAppConfigs`, the swift15 checkout), so the refusal is about the scan status, not the candidates. -- This is the same class as the uutils-find trap on the dev box, with a different cause (GNU find + foreign-owned overlay internals). Minimal fix, safe because a `.AGENTTRASH` never lives inside container storage: prune `-path "$HOME/.local/share/containers"` from the scan, or accept find's partial output and reserve the refusal for a hard find failure. Verify with `sh -n`/shellcheck, not `bash -n`. -- Do not run the purger with `sudo` to dodge this: candidates are judged by ownership, and a root-owned scan changes the refusal outcome rather than fixing it. Retire the blocker, then re-run as the deploy user. +- On the dev box the cause is uutils find (`find --version` → `find 0.8.0`), which exits 1 on `~/.local/share/containers/storage/**`. Proof: `find /home/blank -xdev -name .AGENTTRASH >/dev/null 2>&1; echo $?` → `1`, while a narrower root → `0`. +- On **dankserver** GNU find does the same, because rootless podman storage holds foreign-UID directories (`overlay/*/diff/var/lib/mysql`, `valkey`, `calibre/plugins`, `apt/.../partial`): `find /home/dank -xdev -name .AGENTTRASH 2>&1 >/dev/null | head` shows the denials. +- Fix to land in the script (not to improvise around): prune `~/.local/share/containers` from the scan — a `.AGENTTRASH` never lives inside container storage — or accept partial output and reserve refusal for a hard find failure. `sh -n` / shellcheck is the gate; `bash -n` is not. + +## Verified good path (dankserver, 2026-10-30) + +After freeing space by hand: `/tmp`-scoped plan → `--execute` deleted the staged item and reported `removed=1 deleted=1`; a second `--execute /mnt/entropy` cleaned that volume's root (`refused=0`). The cleanup log carries one dated line per run; read it instead of re-deriving what happened. diff --git a/agent/managed-skills/radiance-calib-tap-produce-latest/SKILL.md b/agent/managed-skills/radiance-calib-tap-produce-latest/SKILL.md new file mode 100644 index 0000000..acc2521 --- /dev/null +++ b/agent/managed-skills/radiance-calib-tap-produce-latest/SKILL.md @@ -0,0 +1,49 @@ +--- +name: radiance-calib-tap-produce-latest +description: "Produce the radiance calibration tap (layer Hessians for GPTQ calib=) on released radiance: which image actually arms it, the only shape that fits 2x R9700, the floor, the seal contract, and the two silent traps. Use when a w4-MoE quant needs a tap, not when merely verifying a build writes." +--- + +Friction-earned 2026-10-30 on dankserver (2x R9700 32 GiB, qwen3.8-flash-next bf16 container, campaign `/home/dank/campaign/qwen38-w5nl64`). + +## The image decides whether the tap exists + +`RADIANCE_CALIB_DIR` + `RADIANCE_CALIB_TOKENS` arm the tap in the **source** (`mc.calib = calib_dir() != nullptr`, identical in v1.8.1 and v1.14.0 trees), but a boot either prints + +``` +I[0] recorded passes: off, the program accumulates calibration Grams (gram_accum) +``` + +or it does not — and if it does not, the tap writes **zero bytes for the whole run** while `/health`, `/stats` and the pusher all look healthy. Measured: production pin 1.8.1 = 17 hours, 0 bytes; `:latest` (v1.14.0, revision `4a61d7296bc5`, digest `69add58f7de1…`) on the same argv/env = sealed in minutes. + +**Procedure:** boot the candidate image with the calib env and grep its log for `gram_accum` **before** pushing anything. Absence = wrong build; do not spend a window discovering it. + +## The vendor calibration shape does not fit two 32 GiB cards + +`docs/MOE-W4.md` prescribes `--max-num-batched-tokens 65536 --max-num-seqs 64` (~930 tok/s). On 2×32 GiB R9700s (27.95 GiB claimable) it dies in the VRAM resolver — arena alone 12.83 GiB; variants came up short by 3.19 GiB, 279 MiB, 59 MiB, 86 MiB, then a starved cache. Wider steps are refused by the arch before memory even matters: + +``` +radiance: qwen4exp: the QSA score plane would be 3125 MiB -- 16384 rows (--max-num-batched-tokens 16384) + by --max-model-len 200000 / compress ratio 4. Lower either. +``` + +What boots and seals: **4 slots, `--max-num-batched-tokens 2048`** (the quantised file tier's ceiling), `--kv-cache-dtype fp8`, `--weights-disk-tier`, `--placement expert_tiered`, `--ngram-placement disk`, `--max-model-len 200000` (pool 199,948 tokens: `kv cache 1.50 GiB … 1.0 full-length`). Prefill lands at 50–70 tok/s; the tap's cost is expert re-streaming per step, not the engine. + +## Concurrency: 4 stalls, 2 survives + +With `--parallel 4` the pusher sends four ~7.8 K-token prefills; **two complete (7,335 and 8,054 tokens), then a step never returns**: log frozen, `W step: nothing has been logged for 21 s` repeating, all threads in `futex`/`kfd_wait_on_events`, 0 MiB/s of reads, `/health` still 200 and `/stats` still `running: 4` with the pool 98 % idle. Reproduced on both a 32,768 and a 200,000 window, so it is not the pool. Production restarts fine afterwards (`chat 200` in 0.49 s) — the card is not wedged. Treat a frozen engine log with live requests as a hang, not a slow rig, and drive the pusher at `--parallel 2`. + +## Floor and seal contract + +- `RADIANCE_CALIB_TOKENS=1500000` is enough: the vendor's own measurement is that a pooled Gram from 10 M tokens beats one from 1 M by under 1 %; 6 M cost 29 h on this rig. +- One seal writes **~40 GiB** — 24,768 files `gram.r.blk..ffn_down_exps..bin`, 42.9 GB. Give the tap dir ≥60 GiB; a 37 GiB filesystem filled to zero and every later write came back short. +- The engine **exits by itself when the tap seals**, so a mid-flight client gets `429 rate_limit_error: the server is shutting down`. That is success. +- Grams are written only at the seal: a restart before it loses the accumulation but no artifact, so a self-healing driver (kill rig, reboot, re-push) is safe. + +## Two traps that read as successes + +- **A port that answers is not your rig.** After the runner stops production, the dying process still answers `:7770` for seconds; a health-only gate "passes" at 10 s on a container that is not up, and the pusher then refuses on container name (or worse, pushes into production). Gate on `podman ps` **naming the container**, then health. +- **A container-based GPU tool must own the cards.** An orphaned engine keeps `/dev/kfd` and the next boot dies with `E startup stopped: device error` (also produced by `kill -9`-ing an engine: let the container teardown run, or kill by exact pid from `sudo fuser /dev/kfd` — never `pkill -f`, which also matches your own command line). + +## Downstream + +The tap is consumed by `rad-convert` via `gptq:calib=`; the file contract (`RADGRAM2`, `gram.r..bin`) is version-stable, so a latest-written tap converts on latest. `scripts/grams-to-imatrix.py -o imatrix.gguf` turns it into an imatrix. Arm output size is ~the served container's size (114 G here) — check free space before converting, not after. diff --git a/agent/managed-skills/radiance-tap-arm-and-seal-verify/SKILL.md b/agent/managed-skills/radiance-tap-arm-and-seal-verify/SKILL.md new file mode 100644 index 0000000..ae0474b --- /dev/null +++ b/agent/managed-skills/radiance-tap-arm-and-seal-verify/SKILL.md @@ -0,0 +1,81 @@ +--- +name: radiance-tap-arm-and-seal-verify +description: "Prove quickly whether a radiance build writes calibration Grams (arming line, 8k-floor seal, engine exits on seal, 43 GB seal, GPU-exclusivity preflight, pgrep/health gate traps)." +--- + +# Verify a radiance calibration tap in 15 minutes + +Prove whether a radiance build writes calibration Grams (`RADIANCE_CALIB_DIR`) before +committing hours of drained serving to it. The whole answer is two log lines and a +directory size. + +## The two lines that decide it + +1. **Arming** — on the boot ledger, rank 0: + `recorded passes: off, the program accumulates calibration Grams (gram_accum)` + Absent = the tap is not in the program: nothing will ever be written, and + `/health`, `/stats` and a healthy push prove nothing. A 17-hour run on the + production pin (radiance 1.8.1) wrote **zero bytes** with no arming line. +2. **Seal** — when the pooled token floor is reached: + `radiance: calibration wrote layer Hessians after N tokens to /data/calib` + printed once per rank. Then the directory holds `gram.r...bin` + files (~24,768 files, ~43 GB). Check `du -sb`; a plausible-looking empty dir is + the failure mode. + +## The seal ends the server + +The engine **shuts itself down** once it seals. In-flight requests then receive +`429 rate_limit_error: the server is shutting down`. That is SUCCESS, not an error: +wait for the seal line, and treat a post-seal 429 as done. Do not retry into it. + +## Minimum proof run + +- Boot the candidate image with `RADIANCE_CALIB_DIR=/data/calib` and + `RADIANCE_CALIB_TOKENS=8192` (a small floor, not the campaign floor). +- Push three or four ~7k-token documents concurrently; at ~64 tok/s aggregate the + seal lands in ~2 minutes. +- One seal costs ~43 GB: point the tap at a filesystem with ≥60 GB free and check + before booting. +- Consumer side is version-stable: `rad-convert` takes `gptq:calib=` and reads + `RADGRAM2` grams written by any pin, so the producer and the converter may be + different builds. + +## Shape: the vendor recipe does not fit two 32 GiB cards + +Vendor `docs/MOE-W4.md` prescribes `--max-num-batched-tokens 65536 --max-num-seqs 64` +(~930 tok/s). On two 32 GiB R9700s the activation arena alone is 12.83 GiB of the +27.95 GiB claimable and every variant stops in the VRAM resolver. +The qwen4exp arch prices a **QSA score plane** of `rows × window / compress`: +`the QSA score plane would be 3125 MiB -- 16384 rows by --max-model-len 200000 / 4. +Lower either.` → 16,384 rows is refused outright; 8,192 rows costs ~1.56 GiB. +The shape that fits and seals: production's own — 4 slots, `--max-num-batched-tokens +2048`, fp8 KV, `expert_tiered` + weights disk tier, n-gram table on disk. + +## Exclusivity is not optional + +A leftover calibration container silently owns the cards and poisons every later +boot with `E startup stopped: device error`. Before any boot: + +``` +sudo fuser /dev/kfd # must print nothing +podman ps --format '{{.Names}}' +``` + +After stopping a rig, `podman rm -f ` **and** kill any `radiance` pid still +holding `/dev/kfd` (`kill -9` by the pid from `fuser`, never `pkill -f`). Then verify +`fuser` is empty before booting the next shape — a 21-second `device error` at boot +is a dirty device, not a shape refusal. + +Two gate traps, both measured: +- `curl /health` right after stopping the service can be answered by the **dying** + production process, so "up at 10 s" is a lie. Gate on the container name in + `podman ps` **and** the marker file the rig writes, not on the port alone. +- `pgrep -f "script.sh"` matches the polling shell's own command line; a + `while pgrep -f` wait loop then never exits. Gate on a file, a container name, or + the pid you captured at launch. + +## Report + +Write the outcome into the page/README that owns the campaign, not a diary: which +build arms the tap, the fitting shape, the floor and its warrant, and the seal size. +Numbers from a shape that is not running now are dead weight — keep the law. diff --git a/agent/managed-skills/radiance-w4-campaign/SKILL.md b/agent/managed-skills/radiance-w4-campaign/SKILL.md index 93011b4..3bed75e 100644 --- a/agent/managed-skills/radiance-w4-campaign/SKILL.md +++ b/agent/managed-skills/radiance-w4-campaign/SKILL.md @@ -1,27 +1,39 @@ --- name: radiance-w4-campaign -description: "Run a radiance GPTQ-w4 quantization campaign on dankserver: calibration → protected experts → convert → KLD gate → deploy, with the engine-pin, tap-engagement and MNBT laws." +description: "Run a radiance GPTQ-w4 quantization campaign on dankserver: calibration tap pin and engagement proof, protected experts, convert, KLD gate, deploy — plus the tap-disk and orphan-engine laws." --- -Pipeline for quantizing a BF16 Qwen3.8-Flash-Next checkpoint into a servable `qwen4exp` `.rad` on dankserver (frank-primary). Reference run: `serverAppConfigs-docs/LLM/frank-primary-radiance-swift15-w4dn8.md`; scripts in `scripts/` (`run-swift15-bf16.sh`, `calib-push.py`, `derive-protected-experts.py`, `apply-protected-cast.sh`, `convert-swift15-w4dn8.sh`, `kld-gate-swift15.sh`, `fleet-probes-swift15.sh`). +Pipeline for quantizing a BF16 MoE checkpoint into a servable `.rad` on dankserver (frank-primary). Reference run: `serverAppConfigs-docs/LLM/frank-primary-radiance-swift15-w4dn8.md`; scripts in `scripts/` (`run-swift15-bf16.sh`, `run--bf16.sh`, `calib-push.py`, `derive-protected-experts.py`, `apply-protected-cast.sh`, `convert-.sh`, `kld-gate-.sh`, `fleet-probes-.sh`). Sequence: 1. **Fetch** checkpoint with per-shard LFS sha256 verification (`-f`, byte counts, no token on argv). 2. **BF16 container**: `rad-convert` no-recipe (pure copy) — serves as the KLD reference engine. -3. **KLD reference**: record logits on the SERVE pin (1.0.13) at MNBT 32768, `--max-model-len 16384`, corpus ≤16k prompts. Writes atomically at completion (`rows:` line = receipt). Never scp a running bash script (lazy-read law). -4. **Calibration**: the tap arms on the **1.2.0** pin — see the section below before any hour is spent. +3. **KLD reference**: record logits on the SERVE pin at the production geometry, `--max-model-len` under the pool, corpus ≤ that window. Writes atomically at completion (`rows:` line = receipt). +4. **Calibration tap** — see the pin law and the engagement test below. 5. **Protected experts**: derive from gram energy (share ≥3 % AND ≥3× routing share), splice between `# PROTECTED-BEGIN/END` markers of the recipe. Plugin law: ONE encoding per expert table; only whole bf16 experts legal beside it — no gate_up/down split. 6. **Final convert** on the SERVE pin (CPU GPTQ, hours; disk gate ≥146 GiB free first). -7. **KLD gate** on the serve pin: **MNBT ≤ 2048** (w4 streams in place only there; 32768 → `DID NOT FIT ... stored form is not the container's`), pool 40960, fp8 KV matching the production KV dtype. Gate: mean KL ≤ 0.03, top-1 ≥ 95 %. -8. **Deploy** `./scripts/deploy.sh dankserver --service frank-archon`; verify `/v1/models` id = served-model-name (alias independent of file — harness models.yml untouched), boot ledger, streamed ≥128-token decode probe, fleet probes (vision needs ≥256-token budget + `reasoning_content` fallback). +7. **KLD gate** on the serve pin at the production MNBT (a w4 does not stream in place outside the shape it was measured at), pool and KV dtype matching production. Gate: mean KL ≤ 0.03, top-1 ≥ 95 %. +8. **Deploy** `./scripts/deploy.sh dankserver --service frank-archon`; verify `/v1/models` id = served-model-name, boot ledger, streamed ≥128-token decode probe, fleet probes (vision needs ≥512-token budget + `reasoning_content` fallback). -## The calibration tap: pin, arming, engagement gate, output size +## The tap pin law (measured 2026-10-30, cost: one 17-hour night) -- **Pin: 1.2.0 arms the tap, the production pin does not.** `RADIANCE_CALIB_DIR` + `RADIANCE_CALIB_TOKENS` are the whole interface (there is no `--calib*` flag on any pin). Running a tap on the 1.8.1 serve image produced **zero bytes over 17 h / 7.3 M prompt tokens** at a healthy 118 tok/s — no error, no ledger line, no `/stats` counter. Swift15's runner pins `PIN_120` for the `calib` mode only; a runner that uses `FRANK_ARCHON_IMAGE` for every mode silently calibrates into a void. -- **Strings prove code, not arming.** `CalibExperts`, `RADGRAM2` and the seal line `calibration wrote layer Hessians after %lld tokens` exist in `architectures/qwen4exp_fp8.so` from 1.2.4 through 1.8.1, and the boot ledger lists RADIANCE_* env **names** (a dump, not support). Neither is evidence the tap is engaged. -- **Engagement gate, before any long drain.** Boot with a tiny floor (`RADIANCE_CALIB_TOKENS=8192`), push 2–3 corpus documents, and require the seal line plus bytes in the tap dir. The seal fires after ~10 k tokens regardless of prompt size, so the test costs minutes — the absence of this check cost 17 h. -- **One seal writes ~40 GB** (per-expert gram + Hessian set, sized by expert width and layer count, essentially independent of how few tokens triggered it). Point `RADIANCE_CALIB_DIR` at a volume with ≥60 GiB free (`/mnt/entropy` on dankserver) and gate on free space before boot; landing it on a 37 GiB root filled the filesystem, and every subsequent write came back `was short` / `cannot write`. -- The tap needs the bf16 container (`calibrate the bf16 container`), tp=2 with `--placement expert_tiered --weights-disk-tier` is fine (swift15 did it), and a sliced-expert layout is refused by name (`calibration runs at one rank; this rank holds a slice of every expert's width`). 1.2.0 lacks `--flash-attn`, `--metrics`, `--expert-nl`, `--busy-prefill-chunk`; it does have every measuring-shape flag (`--kv-cache-dtype`, `--max-model-len`, `--ngram-placement`, `--num-speculative-tokens`, `--override-chat-template`). -- Engine rates: `prefill_tps` in `/stats` is a **rolling gauge**, never a cumulative counter — derive throughput from `total_requests × corpus mean tokens/doc`, or from `steps_prefill`. A wide calib step (MNBT 8192) turns ~2.5 steps/doc into 1.0 and buys ~1.2× on a disk-tiered bf16 rig, not 4.5×. +`RADIANCE_CALIB_DIR` + `RADIANCE_CALIB_TOKENS` arms the per-expert tap on the **1.2.x** line only. -Laws: stop frank-archon for every GPU phase (relinked unit: stop, mask fails; plain `systemctl` for a system unit is denied by polkit — use `sudo systemctl`); verify docs against `podman inspect --format {{join .Config.Cmd " "}}` (rootless, as dank) before writing shapes; ff-pull fails on untracked files the merge tracks — trash first. +- Every plugin from 1.2.4 through 1.8.1 still **contains** the machinery (`CalibExperts`, `RADGRAM2`, `calib_write_host`, the seal line `calibration wrote layer Hessians after %lld tokens`), so `strings` proves nothing about the pin. +- A runner that boots `FRANK_ARCHON_IMAGE` (1.8.1) for `calib` serves perfectly — `/health` 200, 1,700 documents at 118 tok/s, `.rig-boot` written, no error anywhere — and writes **zero bytes**: the tap never arms. +- The engine exposes **no tap counter** in `/stats` and prints **no arming line** in the boot ledger, so a dead tap is indistinguishable from a slow one for hours. +- Default `calib` to the 1.2.0 digest (`stilldeadcode/radiance@sha256:665775ed1d90de6dbb25558e07fc60d20328f1346f96f3c6e00845eb763f01fa` on dankserver as of 2026-10-30); override only by naming `--image` on purpose. + +**Engagement test — 3 minutes, run it before any long tap:** boot with `RADIANCE_CALIB_TOKENS=8192`, push 2–3 real corpus documents, then require the seal line *and* bytes in the tap dir. On 1.2.0 that reads `calibration wrote layer Hessians after 10207 tokens to /data/calib`. No seal line in ten minutes = the pin or the arming is wrong; stop. + +## The tap's disk and process laws + +- **One seal writes ~40 GB** (39.4 GiB measured for an 8,192-token floor): the output is expert-shaped, not token-shaped. Put the tap dir on a volume with ≥60 GiB free and gate it before boot (`Q38_CALIB_MIN_FREE_MIB`). On a 37 GiB root fs it filled `/` to **0 bytes**; the engine's later writes came back `cannot write` / `write was short`, and *other* tooling on the host then failed too. +- **Removing the container does not kill its engine.** `podman rm -f ` leaves `podman-init -- radiance …` alive holding `/dev/kfd`; production then fails `ar_oneshot_2rank_wht6 … device error` at kernel init, exits **0**, and `Restart=always` loops it (counter reached 100) while `/health` never answers. Diagnose with `sudo fuser -v /dev/kfd`; kill **by pid** from `fuser /dev/kfd` + `/proc//comm == radiance`, never `pkill -f` (the pattern matches the caller's own argv). Put that gate in the runner's restore trap. +- The service stop/restore path needs `sudo systemctl` — polkit denies the unprivileged call intermittently (`Access denied … interactive authentication`), and a failed stop means the rig boots *beside* production. + +## Rate measurement + +`/stats` `prefill_tps` is a **rolling gauge**, never a cumulative counter: `gauge × uptime` invents rates (it once produced a bogus 4.5×). Derive tok/s from `total_requests` deltas × the corpus mean tokens/document, or from `steps_prefill`. + +Laws: stop production for every GPU phase; verify shapes against `podman inspect --format {{join .Config.Cmd " "}}` (rootless, as dank) before writing them; ff-pull fails on untracked files the merge tracks — trash first. diff --git a/agent/managed-skills/upstream-engine-reference-into-vault/SKILL.md b/agent/managed-skills/upstream-engine-reference-into-vault/SKILL.md new file mode 100644 index 0000000..013e430 --- /dev/null +++ b/agent/managed-skills/upstream-engine-reference-into-vault/SKILL.md @@ -0,0 +1,34 @@ +--- +name: upstream-engine-reference-into-vault +description: "Author a multi-page reference for an upstream engine into the serverAppConfigs OKF vault, passing the docs seams." +--- + +# Upstream engine → vault reference set + +Use when asked to "deeply read and document" an upstream inference engine (radiance, llama.cpp, halogen) into `serverAppConfigs-docs`. Distill a repo's docs + ABI + option registry into progressive-disclosure pages, not a transcript. + +## Read order (source beats docs) + +1. `README.md`, then `docs/` TOC; read the serving guide, tools reference, plugin guide, architecture guide, op vocabulary. +2. Find the **authoritative option table** in source (`core/config.cpp` for serve flags, `tools/.cpp` `usage()` for tools, `abi/*.h` for plugin fields). Docs lag: radiance's `TOOLS.md` omitted `--chat-template`, `--lora`, `--lora-map`. +3. Version/ABI: `CMakeLists.txt` project VERSION and `#define RAD_ABI_VERSION`; these are independent. +4. Read one complete real plugin/example end to end (e.g. `plugin/spechub/exports.cpp`, `arch/llama_fp8/llama_fp8.cpp`) before writing "how to write a plugin". + +## Page set (one owner per fact) + +Hub + Options + HTTP API + Conversion + Model Families + Plugins + Architecture Plugins + Tools + Operations. The engine pages own engine defaults/flags/ABI; the fleet endpoint page (e.g. `LLM/frank-archon.md`) keeps live shape, Eras, measured laws, rollback. Link, never restate. + +State the upstream commit + release + ABI, and separately the fleet pin; a flag newer than the pin is a boot refusal, not a no-op. + +## Vault mechanics that fail the seams + +- Frontmatter: `type` ∈ reference/howto/tutorial/explanation, `description` non-blank, ≤120 chars, not the title, no fleet quantity; ISO `timestamp` not future. +- Every page ≤500 lines. Split rather than exempt. +- Tables: escape `|` inside cells as `\|` (pipes inside backticks still split GFM tables). +- Register pages: edit the directory `index.md` rows with the **exact** description, then `python3 scripts/gen-vault-index.py serverAppConfigs-docs` (must be a no-op on a second run). +- Run: `check-okf.py`, `check-vault-links.py`, `bats tests/docs_completeness.bats`; then `./tests/run.sh`. +- Do not add a second log/status file; the vault's `log.md` policy is owned by the repo. + +## Commit + +Branch `docs/` off `origin/main`; commit docs + generated indices together. If main moved, merge it (never rebase) and re-run the full suite. Report commit hash, page list, test evidence; ask before opening a PR. diff --git a/agent/memories/sharpshooter/serverAppConfig-eu5lc7uqff33/state.json b/agent/memories/sharpshooter/serverAppConfig-eu5lc7uqff33/state.json index 878baf7..5fe9e25 100644 --- a/agent/memories/sharpshooter/serverAppConfig-eu5lc7uqff33/state.json +++ b/agent/memories/sharpshooter/serverAppConfig-eu5lc7uqff33/state.json @@ -1,4 +1,4 @@ { "v": 1, - "lastConsolidatedAt": 1793322539823 + "lastConsolidatedAt": 1793396927634 } diff --git a/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/architecture.md b/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/architecture.md index 2bb11a1..7a574f1 100644 --- a/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/architecture.md +++ b/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/architecture.md @@ -1,3 +1,4 @@ ## Structural Standards -Quant quality must hold regardless of implementation choice. \ No newline at end of file +Quant quality must hold regardless of implementation choice. +Container versions must be digest-pinned to latest snapshot. diff --git a/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/product.md b/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/product.md index f47d1bb..9654ad2 100644 --- a/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/product.md +++ b/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/product.md @@ -2,4 +2,4 @@ ## Behavioral Updates -Merge any fixes immediately. \ No newline at end of file +Merge any fixes immediately. diff --git a/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/state.json b/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/state.json index 23db0db..d95ce62 100644 --- a/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/state.json +++ b/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/state.json @@ -1,8 +1,8 @@ { "v": 1, - "lastConsolidatedAt": 1793395014767, + "lastConsolidatedAt": 1793398614775, "lastResult": { - "at": 1793315963987, + "at": 1793397014344, "sessions": 1, "deltas": 1, "model": "frank-zealot" diff --git a/agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/state.json b/agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/state.json new file mode 100644 index 0000000..a9efb0b --- /dev/null +++ b/agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/state.json @@ -0,0 +1,4 @@ +{ + "v": 1, + "lastConsolidatedAt": 1793398691951 +} diff --git a/agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/architecture.md b/agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/architecture.md new file mode 100644 index 0000000..65a6bef --- /dev/null +++ b/agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/architecture.md @@ -0,0 +1,3 @@ +## Runtime Decisions + +Avoid subagent spawns when GPU saturation occurs on Dankserver. \ No newline at end of file diff --git a/agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/product.md b/agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/product.md new file mode 100644 index 0000000..8f6e41a --- /dev/null +++ b/agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/product.md @@ -0,0 +1,3 @@ +## Reliability + +No reboots on deployed containers. \ No newline at end of file diff --git a/agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/state.json b/agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/state.json new file mode 100644 index 0000000..0bd20e4 --- /dev/null +++ b/agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/state.json @@ -0,0 +1,10 @@ +{ + "v": 1, + "lastConsolidatedAt": 1793398788553, + "lastResult": { + "at": 1793397235249, + "sessions": 1, + "deltas": 14, + "model": "frank-zealot" + } +} diff --git a/agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/style.md b/agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/style.md new file mode 100644 index 0000000..e69de29