From 92c127ca08e2693519e6276b204594007b742a41 Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 30 Oct 2026 16:20:49 -0700 Subject: [PATCH] live: heartbeat sync 2026-10-30T16:20:49-07:00 (Refs #11) --- .../dankserver-disk-census/SKILL.md | 82 ++++++++++++++++ .../dankserver-space-reclaim/SKILL.md | 94 +++++++++++++++++++ .../fleet-maintenance-window-refresh/SKILL.md | 85 +++++++++++++++++ .../llamacpp-prism-fork-rebuild-swap/SKILL.md | 57 +++++++++++ .../niri-config-edit-validate/SKILL.md | 4 +- .../radiance-calib-tap-sizing/SKILL.md | 58 ++++++++++++ .../SKILL.md | 68 ++++++++++++++ .../radiance-calib-window-serve-gate/SKILL.md | 85 +++++++++++++++++ .../radiance-calibration-tap-latest/SKILL.md | 85 +++++++++++++++++ .../radiance-tap-run-observability/SKILL.md | 63 +++++++++++++ .../sharpshooter/omp-32c11n2wnglul/state.json | 2 +- .../architecture.md | 9 ++ .../product.md | 2 +- .../state.json | 4 +- .../style.md | 1 + .../architecture.md | 0 .../product.md | 2 + .../state.json | 8 +- .../style.md | 0 .../state.json | 2 +- .../architecture.md | 0 .../product.md | 2 + .../state.json | 10 ++ .../style.md | 0 dotfiles/gtk-3.0/.config/gtk-3.0/noctalia.css | 3 + dotfiles/gtk-4.0/.config/gtk-4.0/noctalia.css | 3 + .../.local/state/noctalia/settings.toml | 16 ++-- 27 files changed, 730 insertions(+), 15 deletions(-) create mode 100644 agent/managed-skills/dankserver-disk-census/SKILL.md create mode 100644 agent/managed-skills/dankserver-space-reclaim/SKILL.md create mode 100644 agent/managed-skills/fleet-maintenance-window-refresh/SKILL.md create mode 100644 agent/managed-skills/llamacpp-prism-fork-rebuild-swap/SKILL.md create mode 100644 agent/managed-skills/radiance-calib-tap-sizing/SKILL.md create mode 100644 agent/managed-skills/radiance-calib-window-orchestrator/SKILL.md create mode 100644 agent/managed-skills/radiance-calib-window-serve-gate/SKILL.md create mode 100644 agent/managed-skills/radiance-calibration-tap-latest/SKILL.md create mode 100644 agent/managed-skills/radiance-tap-run-observability/SKILL.md create mode 100644 agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/architecture.md create mode 100644 agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/product.md create mode 100644 agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/style.md create mode 100644 agent/memories/sharpshooter/wt-20261030-153725-7c8a99d-2jvaw6dcu0qzt/architecture.md create mode 100644 agent/memories/sharpshooter/wt-20261030-153725-7c8a99d-2jvaw6dcu0qzt/product.md create mode 100644 agent/memories/sharpshooter/wt-20261030-153725-7c8a99d-2jvaw6dcu0qzt/state.json create mode 100644 agent/memories/sharpshooter/wt-20261030-153725-7c8a99d-2jvaw6dcu0qzt/style.md diff --git a/agent/managed-skills/dankserver-disk-census/SKILL.md b/agent/managed-skills/dankserver-disk-census/SKILL.md new file mode 100644 index 0000000..d6c82e1 --- /dev/null +++ b/agent/managed-skills/dankserver-disk-census/SKILL.md @@ -0,0 +1,82 @@ +--- +name: dankserver-disk-census +description: "Analyze what fills dankserver's root filesystem (excluding /mnt/entropy) and price reclaim safely: du/findmnt method, podman+docker accounting, Plex DB backups, orphan KV spill, ext4 reserve and the calibration space gate." +--- + +# Dankserver root-filesystem census + +Use when asked "what is using disk on dankserver", why `/` is full, or what must be freed before a +calibration/convert window. Read-only first; recommend, never delete without Frank's explicit OK. + +## Shape of the box (verify, do not assume) + +`/` is `/dev/nvme0n1p2` ext4 (~1.8T). `/mnt/entropy` is a separate ~91T LUKS/ext4 volume; when the +task says exclude it, `du -x` already stays on the root filesystem. `/var/lib/docker` and +`/var/lib/plexmediaserver` are on `/`, not entropy. + +```bash +ssh dankserver 'df -hT -x tmpfs -x devtmpfs -x squashfs; findmnt -t ext4,xfs,btrfs,zfs' +ssh dankserver 'sudo du -x -d1 -h / 2>/dev/null | sort -h | tail -20' +ssh dankserver 'sudo du -x -d2 -h /home 2>/dev/null | sort -h | tail -30' +ssh dankserver 'sudo du -x -d2 -h /var 2>/dev/null | sort -h | tail -20' +``` + +`du` is device usage (st_blocks), so sparse files read small; `df` is the truth for free space. +A full top-level scan of `/` takes ~2 min — background it and keep working. + +## Where the bytes actually live + +- `/home/dank/models`: the vendor HF checkpoint **and** its `.rad` conversion are two representations + of the same weights (each ~336G for Qwen3.8-Flash-Next) plus the live serving `.rad`. All can be + legitimately present; check `scripts/frank-archon.env` for the live payload before calling any of + them dead. +- `/home/dank/.local/share/containers`: rootless podman storage is usually the largest non-model + consumer. Do not estimate it with `du` alone. +- `/home/dank/radiance-state/kvcache`: disk KV spill (`blocks.bin`, `ckpts.bin`). It is orphaned + whenever the running container does not mount it — check + `podman inspect frank-archon --format '{{range .Mounts}}{{.Source}} -> {{.Destination}}{{"\n"}}{{end}}'` + (rootless, `XDG_RUNTIME_DIR=/run/user/$(id -u)`). The unit is a root unit but the container is + rootless; `sudo podman ps` looks empty even while it serves. +- `/home/dank/corpora` vs `/home/dank/campaign/`: there may be an OLD campaign's grams/KLD + reference beside the current one. Identify a KLD reference's base by its header, not its date: + `head -c 600 corpora/kld-ref/kld.json` → `"model": "/models/rad/....rad"`. +- `/var/lib/plexmediaserver/.../Plug-in Support/Databases`: Plex keeps dated DB backups; four stale + sets can be tens of GiB. `Metadata/` and `Media/` are thumbnails and chapter art. +- `/opt/vllm`, `/opt/llama.cpp`, `~/.local/bin/llama*`: pre-radiance build leftovers; confirm no unit + references them (`grep -RIl /opt/vllm /etc/systemd/system /home/dank/.config/systemd`). + +## Container accounting + +```bash +XDG_RUNTIME_DIR=/run/user/$(id -u) podman system df +XDG_RUNTIME_DIR=/run/user/$(id -u) podman images --sort size --format '{{.Size}}\t{{.Repository}}:{{.Tag}}\t{{.ID}}' +XDG_RUNTIME_DIR=/run/user/$(id -u) podman ps --format '{{.Image}}' | sort -u # in use +sudo docker system df; sudo docker ps -aq | wc -l; systemctl is-enabled docker +``` + +- `podman system df`'s **reclaimable** is the authoritative figure. Summing per-image `.Size` over + dangling images double-counts shared layers and can read several times larger than the disk. +- Docker may be `disabled` with zero containers yet hold ~120G of images from the vLLM era: full + reclaim candidate. +- Keep the radiance pins and the media-stack images; the big unused ones are vllm/rocm/fp030/harbor. + +## Free-space levers and their traps + +- **Trash is space-neutral on the same filesystem.** Moving a 336G payload to `~/.AGENTTRASH` frees + nothing until the trash is purged; say so before recommending it. +- **ext4 reserves 5 %** (`tune2fs -l /dev/nvme0n1p2 | grep -i 'Reserved block count'`, × block size). + `df` Avail excludes it, and the campaign runs as `dank`, so the reserve silently blocks non-root + writers while root still has room. Lowering to 1 % releases the reserve with no deletion, but it is + the fragmentation cushion on a full disk — last resort, never the first move. +- **The calibration window has a 60 GiB gate** (`campaign/*/window-when-space.sh`, + `Q38_CALIB_MIN_FREE_MIB=61440`). A failed tap keeps the BF16 container on disk; banked-but-not-purged + is not free. +- `uv`'s archive cache (`~/.cache/uv/archive-v0`) is routinely tens of GiB and safe to `uv cache prune`. + +## Reporting + +Rank by reclaim size, then risk: orphan KV spill and Docker images first; stale Plex DB backups and +old-campaign corpora next; `/opt`/binaries/journal/snaps last. Give exact path, size, evidence that +nothing uses it, and what breaks if it is wrong. Never execute a prune or delete in the census turn — +Frank approves each removal, and the repo deletion law (trash, never `rm`) still applies to anything +that is a record rather than a cache. diff --git a/agent/managed-skills/dankserver-space-reclaim/SKILL.md b/agent/managed-skills/dankserver-space-reclaim/SKILL.md new file mode 100644 index 0000000..ca30329 --- /dev/null +++ b/agent/managed-skills/dankserver-space-reclaim/SKILL.md @@ -0,0 +1,94 @@ +--- +name: dankserver-space-reclaim +description: "Execute approved low-risk disk reclaim on dankserver's root fs: prove targets unreferenced, orphan KV first, dangling images, Docker prune, Plex DB backups; the campaign space watcher fires the moment the gate opens." +--- + +# Dankserver root-fs reclaim (approved low-risk sets) + +Run only with the user's explicit approval for the named set. Deletion is the point: moving a cache +to `.AGENTTRASH` on the same filesystem frees nothing, so say that before proposing it. Never touch +`/mnt/entropy` (separate 90T volume) and never delete `/home/dank/models/rad/qwen38-bf16.rad` or +`campaign/**` while the qwen38 campaign is unfinished. + +## Census (live, never from docs) + +```sh +ssh dankserver 'df -hT -x tmpfs -x devtmpfs; findmnt -o TARGET,SOURCE,FSTYPE,SIZE,USED,AVAIL -t ext4,xfs,btrfs' +ssh dankserver 'sudo du -x -d1 -h / 2>/dev/null | sort -h | tail -15' +``` + +`-x` keeps `/mnt/entropy` out automatically (separate mount). Then drill `/home` and `/var` with +`sudo du -x -d2`. Typical 2026-10 shape: `/home` ≈ 1.4T (models 785G, `.local/share/containers` 334G, +`radiance-state` 69G, corpora 64G, campaign 45G), `/var` ≈ 215G (docker 119G, plex 80G), `/opt` 17G. +Verify live; sizes are disposable. + +Do NOT `grep -R` across `/home/dank/Containers` or `~/Projects` — jellyfin cache JSON makes it run +for minutes. Scope `find -maxdepth 3 -iname '*harbor*'` and check unit dirs only. + +## Prove each target unreferenced before deleting + +- `sudo lsof ` (fuse warnings are noise) and, for podman image IDs, cross-check + `podman ps -a --format '{{.ImageID}}'`; `podman image prune` keeps an in-use untagged image, but + confirm the container survives. +- Rootless vs rootful matters: `frank-archon` runs as rootless podman under `dank` even though its + unit is a root unit. Check both `sudo podman ps -a` and + `XDG_RUNTIME_DIR=/run/user/$(id -u) podman ps -a`, including stopped containers, and inspect their + mounts (`podman inspect -f '{{range .Mounts}}{{.Source}} {{end}}'`). +- `grep -RIl '' /etc/systemd/system /home/dank/.config/systemd` must be empty. +- A running archon mounts only `/models`, `radiance-state/tune` and the template — **not** + `radiance-state/kvcache`, so stale `blocks.bin`/`ckpts.bin` are genuinely orphaned. + +## Reclaim ladder (safest first) + +1. **Orphan radiance KV spill** — `radiance-state/kvcache/frank-archon/{blocks.bin,ckpts.bin}` + (measured 69G, last write weeks old, no handle). `sudo rm -f` both. This alone can clear the + campaign gate. +2. **Docker** — only if `systemctl is-enabled docker` is `disabled` and `docker ps -aq` is empty: + `sudo docker system prune -a -f` (measured ~104GB; volumes are NOT removed, keep it that way). + Verify `docker volume ls` still lists the volumes. +3. **Podman dangling only** — `podman image prune -f` (default dangling, no `-a` without approval). + Measured 33 removed / ~62G freed; one dangling ID is the running calibration image and is skipped. + The remote shell is **zsh**: unquoted `$ids` does not word-split, so `podman rmi $ids` receives a + newline-joined string and fails with `invalid reference format`. Pipe IDs through xargs instead: + `podman images --format '{{.ID}} {{.Repository}}' | grep goharbor | cut -d' ' -f1 | sort -u | xargs -r podman rmi`. +4. **Harbor registry images** — all `goharbor/*` v2.13 images are ~2.1G total and safe once no + harbor container/unit/compose exists (`~/.harbor` is a cmdh manifest, not the goharbor stack). The + big `harbor-hf`/`harbor-vllm` build images live under Docker and disappear with step 2. +5. **Plex DB backups** — in `Plug-in Support/Databases`, keep the newest `*.db-YYYY-MM-DD` set and + delete older sets (measured 3 × 4.9G blobs + 3 × 1.4G library ≈ 19G). Plex stays up; never touch + the live `library.db`/`blobs.db`. +6. **Later, with separate approval** — Swift-era `corpora/calib` + `corpora/kld-ref` (KLD header + names the base: `model` field), `uv cache prune` (~25G), `/opt/vllm`, `/opt/llama.cpp`, + `~/.local/bin` stale llama binaries, journal vacuum, old snaps. Tagged-but-unused podman builds + (vllm/rocm/fp030) remain ~225G; removing them loses build caches, so scope them explicitly. + +## The campaign watcher is the side effect + +`campaign/qwen38-w5nl64/window-when-space.sh` polls +`df --output=avail -B 1M /calib` every 60 s for `Q38_CALIB_MIN_FREE_MIB` (default 61440). +The moment reclaim opens the gate it starts the documented calibration window, which **stops +frank-archon** and boots `qwen38-bf16` on port 7770. Expect production downtime and a `health=000` +during load; the engine is healthy once `/health` returns 200. Do not restart frank-archon by hand +during the window — it would fight the rig for both cards. + +If the window fails, read `logs/window-start-*.log` and `/home/dank/logs/qwen38_bf16_calib.log`: +`I stopping: draining in-flight requests` + 429 `the server is shutting down` means the orchestrator +SIGTERMed a healthy engine (tap marker state), not a crash; a retry may then fail with +`E startup stopped: device error` while the prior process's `VM memory stats ... non-zero when fini` +clears. Restore is automatic: the window script restores frank-archon and reports the banked/kept +`.rad`. + +## Verify + +```sh +df -h /; df --output=avail -B 1M / +XDG_RUNTIME_DIR=/run/user/$(id -u) podman system df +curl -s -o /dev/null -w '%{http_code}\n' http://127.0.0.1:7770/health +XDG_RUNTIME_DIR=/run/user/$(id -u) podman ps --format '{{.Names}} {{.Status}}' +``` + +All running containers must be untouched, archon back to 200, and the campaign gate recomputed with +its own command. ext4 reserves 5% (~90G on the 1.8T root): `df` Avail is what the unprivileged +campaign sees, so root's ability to write does not mean the gate passes; lowering the reserve +(`tune2fs -m 1`) releases ~72G but removes the fragmentation cushion on a nearly full disk — last +resort only. diff --git a/agent/managed-skills/fleet-maintenance-window-refresh/SKILL.md b/agent/managed-skills/fleet-maintenance-window-refresh/SKILL.md new file mode 100644 index 0000000..05fae75 --- /dev/null +++ b/agent/managed-skills/fleet-maintenance-window-refresh/SKILL.md @@ -0,0 +1,85 @@ +--- +name: fleet-maintenance-window-refresh +description: "Execute a fleet-wide OS + container image refresh on Dankclaw/Dankserver: apt without reboots, registry pin census, digest bump to main, per-service deploys, and proving the container actually moved." +--- + +"Update the servers / update all containers" on Dankclaw (192.168.0.143) + Dankserver (192.168.0.235). Both are Ubuntu 26.04, rootless podman quadlets, deploy via `scripts/deploy.sh`. + +## 0. Recon before touching anything + +```bash +for h in .143 .235; do ssh dank@192.168.0.$h 'uptime; systemctl --failed --plain --no-legend; sudo -n apt-get -s upgrade | grep -c ^Inst'; done +ssh dank@192.168.0.235 'ps -eo pid,etimes,args | grep -E "radiance|llama-server" | grep -v grep' +``` +- A radiance/llama process holding the GPUs is a campaign or a bench: **do not touch it**, and never reboot without explicit approval. +- Note pre-existing failed units so you do not blame them on your deploy later (`mnt-md0.mount`, `nvidia-cdi-refresh*`, `logrotate` fail on Dankserver by default). + +## 1. OS packages, no reboot + +```bash +sudo -n env DEBIAN_FRONTEND=noninteractive NOTIFY_SOCKET= apt-get update -qq +sudo -n env DEBIAN_FRONTEND=noninteractive NOTIFY_SOCKET= apt-get -y \ + -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold \ + -o APT::Get::AutomaticRemove=false full-upgrade +``` +- `needrestart` is NOT installed on these hosts — check before assuming daemons will not bounce. +- Dankserver's initramfs trigger takes ~9 minutes; "dpkg BUSY" for 20 minutes is normal, do not kill it. +- After: `cat /var/run/reboot-required`, `uname -r` vs newest `/boot/vmlinuz-*`. A staged kernel is not a running kernel — say so. +- **Dankclaw only:** `grep -o 'iommu=pt' /boot/grub/grub.cfg` must still match after the kernel's `update-grub`, else the XDNA2 NPU lane dies at the next boot. + +## 2. Registry pin census (full digests, not prefixes) + +Read every `Image=` in `**/quadlet/*.container` plus `FRANK_*_IMAGE=` in `scripts/*.env`, resolve `tag -> Docker-Content-Digest` and compare. + +- Docker Hub: token from `auth.docker.io/token?service=registry.docker.io&scope=repository::pull`, then `HEAD registry-1.docker.io/v2//manifests/` with an index+list+manifest `Accept`. +- ghcr.io: `https://ghcr.io/token?scope=repository::pull&service=ghcr.io`. +- **lscr.io answers 404 through its own `/v2` path.** Its images are byte-identical on `docker.io/library/linuxserver/` — enumerate there; pinned lscr digests match Hub digests exactly. +- Print the FULL digest. A 12-char prefix cannot be pasted into a pin. +- `latest`-tagged services differ constantly; a differing digest IS the update. Do not call a versioned pin stale without a newer version tag. + +## 3. Bump the pins in the repo + +- Digest-only change: `sed -i 's|sha256:OLD|sha256:NEW|g'` (verify with `git diff -U0`). +- Tag+digest change: edit the line. +- Some pins MUST stay tagless `repo@sha256:…` — a bats seam greps the exact form (e.g. `frank-dragoon.env`). Check `grep -n 'FRANK_DRAGOON_IMAGE=' tests/*.bats` before adding a tag. +- Repo-wide truth is the canonical checkout **`/home/blank/Projects/serverAppConfig`** (on `main`, has `tests/e2e/.env.local`). Session worktrees cannot run `deploy.sh`: it refuses a dirty tree and any branch but `main`, and `git checkout` is blocked by workspace policy. Edit in the worktree, then there: `git pull --ff-only`, `git merge --no-ff `, `git push`. +- Restore canonical-checkout files you edited by accident with `git show HEAD: > ` (checkout/restore are blocked). + +## 4. Pre-pull, then deploy + +Quadlets ship `Pull=never`/`missing` and the deployers refuse an absent image, so pull first as the **owner** (rootless user `dank`): + +```bash +ssh dank@host podman pull -q 'repo/name:tag@sha256:...' # exact pinned ref +``` +Then, from the canonical checkout, batch services per host in one invocation (each invocation also runs the ~1 min Playwright gate): + +```bash +./scripts/deploy.sh dankclaw --service grafana --service alertmanager --service seerr +``` +Order within a batch: DB before app (postgres/valkey/mariadb first), `gluetun` before `searxng` (shared netns). A failure aborts the rest of the batch, so group tightly. + +## 5. PROVE the container moved + +A green deploy is not evidence. Compare running digests to the pins: + +```bash +ssh dank@host podman ps --format '{{.Names}}|{{.Image}}' +``` +- Handlers that call `systemctl --user start` no-op on an active unit: the image installs, the old container keeps serving, and the health gate + e2e still pass. (This bit authentik server/worker on 2026-10-30; the handler now restarts.) If a digest did not move, `systemctl --user restart ` and re-check. +- `systemctl is-active` without `--user` reports system units — Dankclaw app quadlets are user units. Always `--user` for them. + +## 6. Locally built images (loki, alloy, caddy-porkbun) + +`build_compose_image` skips the build when the image's `cfhash` label matches the Containerfile hash, so a rolling base (`alpine:latest`, `grafana/loki:latest`, `caddy:2`) never refreshes on its own. Force it by rebuilding with the same label, then deploy to restart: + +```bash +h=$(sha256sum dankclaw/loki/Containerfile | cut -d' ' -f1) +podman build --format docker --no-cache --label "cfhash=$h" -t franksdank/${svc}:latest -f . +``` +- **zsh trap:** over `ssh` the remote shell is zsh, where `$svc:latest` is a history modifier (`:l` lowercases) — `-t franksdank/$svc:latest` tags `lokiatest`. Always brace: `${svc}:latest`. +- alloy is mirrored to the ROOT store by its handler (`mirror_image_to_root`), so deploy it rather than restarting the user unit. + +## 7. Close + +`./tests/run.sh` before each push (577 seams, ~85 s). Docs: update the endpoint/service page as the owner of the fact; `AGENTS.md` only for a cross-cutting law. Report the reboot as OWED, not done, when it is staged. diff --git a/agent/managed-skills/llamacpp-prism-fork-rebuild-swap/SKILL.md b/agent/managed-skills/llamacpp-prism-fork-rebuild-swap/SKILL.md new file mode 100644 index 0000000..9248fcd --- /dev/null +++ b/agent/managed-skills/llamacpp-prism-fork-rebuild-swap/SKILL.md @@ -0,0 +1,57 @@ +--- +name: llamacpp-prism-fork-rebuild-swap +description: "Rebuild the PrismML llama.cpp fork on Dankclaw from origin/prism and swap frank-zealot's LLAMA_BIN onto it, including the gcc-13 host-compiler and install-RPATH requirements." +--- + +Rebuild the PrismML llama.cpp fork on Dankclaw and swap `frank-zealot` onto it, without guessing the build flags. + +## Recover the build configuration first + +The binary dir carries no CMakeCache. The configure that produced the running build lives in the source tree: + +```bash +grep -E 'GGML_CUDA|CUDA_ARCH|BUILD_TYPE|SHARED|HOST_COMPILER|FA_QUANTS|GENERATOR' \ + /home/dank/GitPackages/llama-prism-src/build-*/CMakeCache.txt +``` +The pinned source clone is `/home/dank/GitPackages/llama-prism-src`, branch `prism`, remote `https://github.com/PrismML-Eng/llama.cpp.git`. `origin/HEAD -> origin/prism`; there is no `master`/`main`. + +## Build into a fresh worktree at a pinned commit + +```bash +SHA=$(git -C /home/dank/GitPackages/llama-prism-src rev-parse origin/prism) +git -C /home/dank/GitPackages/llama-prism-src worktree add --detach /home/dank/GitPackages/llama-prism-next "$SHA" +cmake -B .../llama-prism-next/build-cuda -S .../llama-prism-next -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86-real \ + -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/gcc-13 -DLLAMA_CURL=OFF -DGGML_NATIVE=ON \ + -DCMAKE_INSTALL_PREFIX=/home/dank/GitPackages/llama-prism- \ + -DCMAKE_INSTALL_RPATH='$ORIGIN;$ORIGIN/../lib' +cmake --build .../build-cuda -j 8 +cmake --install .../build-cuda +``` +- **`CMAKE_CUDA_HOST_COMPILER=/usr/bin/gcc-13` is mandatory**: the box default is gcc 15 and nvcc 12.4 rejects it. +- **`CMAKE_INSTALL_RPATH` is mandatory**: `cmake --install` strips the build-tree rpath, and the installed binary then dies on `libllama-server-impl.so`. `patchelf` is NOT installed, so there is no post-hoc fix. +- 672 targets on the first full build; run it detached (`setsid nohup … > log`) and keep `-j 8` so the co-tenant services stay usable. +- Disk: the box runs ~84% full; check `df -h /home/dank` before starting. + +## Verify the payload gate BEFORE swapping + +```bash +D=/home/dank/GitPackages/llama-prism- +$D/bin/llama-server --version # expect 0.x.y-dev (build N, commit ) +ldd $D/bin/llama-server | grep -c "not found" # must be 0 +strings $D/lib/libggml-base.so.* | grep -c PQ2_0 # custom tensor type 142 +strings $D/lib/libllama.so.* | grep -c prism.hadamard # dequant metadata +``` +A fork that dropped either cannot load `Ternary-Bonsai-2-9B PQ2_0` — stock llama.cpp refuses the file and a loader ignoring the metadata dequantizes wrong, silently. + +## Swap + +1. `scripts/frank-zealot.env`: `LLAMA_BIN=/bin/llama-server` (install layout is `bin/`+`lib/`, not the flat dir older builds shipped — the old path `/llama-server` no longer exists). +2. Update the build/version in the env header and `serve-frank-zealot.sh` (default + comments). +3. Land on `main` from the canonical checkout, then `./scripts/deploy.sh dankclaw --service frank-zealot`. +4. Prove it: process arg0 is the new path, and the journal shows `- type pq2_0: N tensors`, `load_tensors_hadamard: loaded N Hadamard-folded weight(s)`, the adapter's `prism.abliterate.base_gguf` line, and `draft acceptance` once MTP drafts. Then probes: chat, a `tools` turn returning valid JSON `tool_calls`, and vision (solid-colour PNG via the OpenAI `image_url` content part, `max_tokens` ≥512 — thinking tokens count against it). +5. Rollback: point `LLAMA_BIN` back at the previous `llama-prism-b` dir and redeploy. Keep that dir on disk. + +## Test seam + +`tests/frank_zealot_unit.bats` asserts `LLAMA_BIN` names a `llama-prism-*` **fork family** path, not a literal build directory: the suite runs on a box with no `/home/dank`, so existence cannot be checked there either. Do not re-pin a literal path. diff --git a/agent/managed-skills/niri-config-edit-validate/SKILL.md b/agent/managed-skills/niri-config-edit-validate/SKILL.md index c0bfca0..a476ca3 100644 --- a/agent/managed-skills/niri-config-edit-validate/SKILL.md +++ b/agent/managed-skills/niri-config-edit-validate/SKILL.md @@ -15,7 +15,9 @@ description: "Locate, edit, validate, and commit niri compositor config on Frank - Profiles: `cfg/desk--.kdl` (e.g. `desk-1-home.kdl`, `desk-2-portable.kdl`); ACTIVE profile is symlinked as `cfg/display.kdl` — `readlink` it to know which. `scripts/desk.sh` manages switching; `desk.sh capture` prints connected outputs' EDID long names. - Panels are keyed by **EDID long name** (`output "Lenovo Group Limited 0x40A9 Unknown"` = the laptop eDP-1 panel), not connector — connector names reshuffle between desks. Long-name blocks BEAT connector-keyed blocks when both match one output (legacy connector blocks like `output "eDP-1"` linger as duplicates; niri applies the EDID block's `position`). - Position math: x,y = top-left of each output in the logical layout; right edge = x + mode width. To place panel A left of panel B: `A.x = B.x - A.width`; bottom-align: `A.y = B.y + (B.height - A.height)`. -- Verify: `niri validate` → hot-reload → `niri msg outputs | grep -iE 'Output |Position'` (prints `Logical position: x, y`). +- **Changing `transform` rewrites every x, not one block.** A 1920x1200 panel rotated 90 is logically 1200 wide; normalizing it to `"normal"` makes it 1920 wide, which shifts the right edge +720 and therefore the x of every output to its right. Recompute the whole row from left to right and state the new x in the commit. Modes need no change (`mode` is the physical mode string, independent of transform). +- **"No matching profile" is a defect, not an idle state.** `desk.sh match` prints `no desk profile matches connected outputs` and `desk-autoswitch.sh` then exits at its initial match (the unit shows `inactive (dead)` with `status=0/SUCCESS`, minutes after start). The rig is not "unchanged" — it runs the *previous* desk's numbers, and a stale `mode` there falls back silently to preferred. Signature seen 2026-10-30: portable rig on `desk-1-home`, both 16" panels asked for `3440x1440@180.000` that no panel reported. Fix the profile, `desk.sh switch`, then `systemctl --user restart blankpad-desk-autoswitch` and confirm `is-active` — the watcher never comes back on its own. +- Verify: `niri validate` → hot-reload → `niri msg outputs | grep -iE 'Output |Position'` (prints `Logical position: x, y`). Compare `Logical size` and `Transform` too — that is what proves a rotation actually applied. ## Schema gotchas (niri 26.04) - `preset-column-widths` AND `preset-window-heights` are children of `layout { }` as SIBLING blocks — wiki wording misleads; nesting or `window-rule` placement fails `niri validate` with `expected one of proportion, fixed` / `unexpected node`. diff --git a/agent/managed-skills/radiance-calib-tap-sizing/SKILL.md b/agent/managed-skills/radiance-calib-tap-sizing/SKILL.md new file mode 100644 index 0000000..10e4b37 --- /dev/null +++ b/agent/managed-skills/radiance-calib-tap-sizing/SKILL.md @@ -0,0 +1,58 @@ +--- +name: radiance-calib-tap-sizing +description: "Decide whether a radiance calibration tap is needed, how large a token floor it wants, and which step/window shape makes it fast — before booking a multi-hour card window." +--- + +# Sizing a radiance calibration tap + +Read this before promising a tap run (hours to days of exclusive cards). It answers three questions: +*what is the tap for*, *how many tokens*, and *which shape makes it fast*. + +## 1. The tap is for the Hessian, not the imatrix + +`tools/rad_convert.cpp` help: `--imatrix F` is *"the expert-popularity profile, the column +importance"* — a diagonal summary. GPTQ's input is separate: the recipe names `gptq calib=`, +and `` is the tap's gram set: **d × d fp32 second moments (`E[x xᵀ]`) per expert per layer**. + +- An external/published imatrix **cannot** substitute. It carries no second moments, it is keyed by + another model's tensor names, and radiance warns that *"a legacy-format imatrix cannot carry + [per-expert counts] at all — the planner will start cold."* +- The imatrix you need is a free by-product: `scripts/grams-to-imatrix.py -o imatrix.gguf`. +- Without a tap the experts can only take the recipe's non-GPTQ rule (`rtn`) — a visibly worse arm. + So "we only want an imatrix" is not a reason to skip it, and "just fetch one" is not a path. + +## 2. The shape decides the throughput — window and step are one decision + +The arch prices a step against the window: **QSA score plane ≈ rows × (window ÷ 4) × 4 bytes**. + +| window | step (rows) | plane | verdict | +|---|---|---|---| +| 200,000 | 16,384 | 3,125 MiB | refused at boot (`the QSA score plane would be … Lower either`) | +| 32,768 | 16,384 | 512 MiB | fits | +| 32,768 | 32,768 | 1 GiB | fits | + +Tap prefill is bound by **expert re-reads per step**, not by tokens as such, so the small window is the +lever: calibration corpora are re-chunked to ≤7,168-token documents and need nowhere near a 200k +window, yet copying the production window is the reflex that disqualifies a wide step and pins the run +to 2,048-token steps (≈70 tok/s on 2× R9700 = ~48 h for 12 M tokens). A 8,192–16,384-token step at a +32,768 window re-reads experts 4–8× less often per token. + +Always boot once and read the ledger (`activation arena`, `experts`, `kv cache`) before believing any +speed estimate: a wider step grows the arena and steals from the experts. + +## 3. Before you spend the cards + +- **Arming proof**: the boot must print `recorded passes: off, the program accumulates calibration + Grams (gram_accum)`. A build that prints nothing writes nothing, however healthy `/health` and + `/stats` look (a 17-hour run on a shipped pin produced zero files). +- **The rig must own the socket**: `ss -ltnp 'sport = :7770'` → pid, and that pid must appear in + `podman top pid`. After the runner stops production, the dying process answers `/health` for a + moment while the new container is still loading — "container running + health 200" is not the rig. +- **Liveness is completions, not log mtime**: the engine logs one line per *completed* request, so a + 4–8 minute prefill looks like silence while `kv_tokens_used` steps in 2,048-token chunks. +- **Space**: one seal writes ~40 GiB in one burst; the preflight needs ≥60 GiB free in the tap dir. +- **The floor is a campaign decision, not a vendor default**: the engine default is 65,536 tokens; a + 12 M floor over a ~9.31 M-token corpus seals in round 1 plus part of round 2, and a round that seals + nothing is normal (the floor is pooled — never treat a flat round as a failure). +- **Exit means success**: at the seal the engine exits by itself; a mid-flight client gets + `429 rate_limit_error: the server is shutting down`. diff --git a/agent/managed-skills/radiance-calib-window-orchestrator/SKILL.md b/agent/managed-skills/radiance-calib-window-orchestrator/SKILL.md new file mode 100644 index 0000000..a2becd0 --- /dev/null +++ b/agent/managed-skills/radiance-calib-window-orchestrator/SKILL.md @@ -0,0 +1,68 @@ +--- +name: radiance-calib-window-orchestrator +description: "Run the fleet's radiance calibration window through the repo orchestrator (scripts/qwen38-calib-window.sh): its gates, the arming proof, the seal contract, and the traps that make a tap look healthy while writing nothing." +--- + +# Run the calibration window through the orchestrator, never a hand-rolled driver + +`serverAppConfigs/scripts/qwen38-calib-window.sh` is the only sanctioned driver for a radiance +calibration tap. Run it from the deploy checkout on the inference host: + +```sh +cd /home/dank/serverAppConfigs +bash scripts/qwen38-calib-window.sh # add --dry-run to print the plan only +``` + +## What it owns (do not re-implement any of it) + +| Gate | Detail | +|---|---| +| Inputs | reference recording `kld-ref/kld.json`, the BF16 `.rad`, the corpus (`Q38_WINDOW_CALIB_CORPUS`, default `calib-7168.jsonl` — the re-chunked file the rig's pool can hold) | +| **Space** | tap dir must have `Q38_CALIB_MIN_FREE_MIB` (default **61440** MiB) free **before a card is touched** | +| Rig | boots `scripts/run-qwen38-bf16.sh calib`, waits `/health` within `Q38_WINDOW_HEALTH_TIMEOUT` (3600 s), else kills the rig, restores the service, exits non-zero | +| Marker | the runner writes `calib/.rig-boot`; `calib-push.py` refuses without it, refuses if it names another container, refuses if that container is not running | +| Bank | the BF16 reference container moves to `~/.AGENTTRASH/-qwen38-bf16-rad/` **only** if both phases exit 0 | + +A hand-rolled `podman run` + push loop skips the space gate and the corpus choice — that is how a +43 GB seal lands on a volume with 35 GB free. + +## Prove the tap ARMS before spending an hour + +A build can carry every `RADIANCE_CALIB_*` string, seal message and plugin, and still never arm. +The arming proof is the boot ledger line: + +``` +I[0] recorded passes: off, the program accumulates calibration Grams (gram_accum) +``` + +No such line ⇒ nothing will be written, however healthy `/health`, `/stats` and the push look +(measured: 17 h, zero bytes, on a pin that lacked it). Diff the *sources* first if in doubt — the tap +code can be identical across versions while one shipped build drops it. + +## Reading a live tap run + +- **A quiet log is not a hang.** The engine logs one line per *completed* request. At 2–4-way + concurrency a 7.5 K-token prefill takes 4–8 minutes; `kv_tokens_used` steps in 2,048-token + chunks and `W step: nothing has been logged for 21 s` is step-level silence. Count completed + requests (`req cmpl`), never log mtime — a log-mtime watchdog kills healthy cycles, and every + reboot discards the accumulated tap. +- **Throughput is capped by expert streaming** (~60–70 tok/s on a disk-tier payload); concurrency + lengthens requests instead of raising throughput. +- **A port that answers is not the rig.** A dying production process answers `:7770` for seconds + after the stop. Gate on `podman ps` naming the rig container. + +## The seal contract + +- One seal writes **~40 GiB** (24,768 `gram.r.blk..ffn_down_exps..bin` files). +- `RADIANCE_CALIB_TOKENS` is a **pooled floor**: the engine sums Grams over prefill tokens and writes + at that count (engine default 65,536; the fleet runner defaults 1,500,000). The vendor's own + measurement: 10 M tokens beat 1 M by under 1 %. +- After writing, **the engine exits by itself**; in-flight clients get + `429 rate_limit_error: the server is shutting down` — that is success, not failure. + +## Card exclusivity + +A leftover engine keeps `/dev/kfd`; the next boot dies `E startup stopped: device error` with the +reason buried above the startup report. Kill by pid — `for p in $(sudo fuser /dev/kfd); do kill -9 $p; done` — +and never `pkill -f ` (the pattern matches your own command line). Verify the device is empty +before any boot and after any kill. diff --git a/agent/managed-skills/radiance-calib-window-serve-gate/SKILL.md b/agent/managed-skills/radiance-calib-window-serve-gate/SKILL.md new file mode 100644 index 0000000..7b9ab76 --- /dev/null +++ b/agent/managed-skills/radiance-calib-window-serve-gate/SKILL.md @@ -0,0 +1,85 @@ +--- +name: radiance-calib-window-serve-gate +description: "Diagnose a radiance calibration window that dies at second zero or refuses its push: the health answer coming from the dying production socket, the listener-identity gate, the preflight trio (space, re-chunked corpus, KEEP_RAD), the 12M floor and the wide-step knob." +--- + +# radiance calibration window — the serve gate and its preflight + +Friction-earned 2026-10-30 on `dank/serverAppConfigs` (`scripts/qwen38-calib-window.sh`, +`scripts/run-qwen38-bf16.sh`). A 12 M-token window died at second zero; the cause was the gate, +not the engine. + +## The signature, and what it really means + +``` +phase=calib health ok waited=0s +the rig container qwen38-bf16 is not running <- pusher refuses +phase=calib push end rc=2 ... FAIL: serve rc=1 push rc=2 +``` + +`waited=0s` is the tell. The runner stops `frank-archon` **first**; the dying production process keeps +its listener for a moment, while the freshly created rig container is already `State.Running=true` but +has not bound the port yet. So `podman inspect` (container running) **and** `/health` 200 (production's +socket) both hold for the wrong process. + +**The gate must prove the listener belongs to the rig:** + +```bash +lpid=$(ss -H -ltnp 'sport = :7770' | grep -o 'pid=[0-9]*' | head -1 | cut -d= -f2) +[ -n "$lpid" ] || return 1 +podman top "$RIG" pid | grep -qw "$lpid" +``` + +Landed as PR #90 with two red-first tests (foreign listener ⇒ no push, window fails; in-container +listener ⇒ proceeds and banks). If you see `waited=0s` on a checkout older than that PR, the window +there is pre-fix: do not run a long window on it. + +## Preflight trio — the window refuses correctly, so do not "fix" it + +1. **Tap dir ≥ 60 GiB free.** One seal writes **43 GB in 24,768 files** + (`gram.r.blk..ffn_down_exps..bin`). Point it at a 35 GiB root and the seal fills the + filesystem to zero. `Q38_CALIB_MIN_FREE_MIB` overrides — for *tests only* (fixtures on `/tmp` tmpfs). +2. **The re-chunked corpus.** `Q38_WINDOW_CALIB_CORPUS` defaults to `calib-7168.jsonl`; pushing + `calib.jsonl` instead runs un-chunked documents through a window sized for chunks. +3. **`Q38_WINDOW_KEEP_RAD=1` when two windows run back to back.** The window banks the BF16 reference + container into `~/.AGENTTRASH/` on success; without this, stage 2 has no reference to tap. + +## Floor and the "add more tokens later" trap + +`RADIANCE_CALIB_TOKENS` is the campaign's number, not the vendor's: the reference container this arm +competes with was calibrated on 10 M tokens, so the fleet floor is **12 M**. Engine default is 65,536. + +**A sealed tap cannot be extended.** Hessians are VRAM accumulators; the seal writes and drops them. +So "run 1 M to verify, then add 11 M" is two windows — 1 M then a fresh 12 M — not an append. Budget +accordingly, and rotate the verification tap to `.AGENTTRASH/`. + +One round of the re-chunked corpus is **9.31 M tokens**, so a 12 M floor seals in round 1 plus part +of round 2; a flat round is normal (the floor is pooled; `round_verdict` says "continue"). + +## The wide-step knob (the tap's real cost) + +`Q38_CALIB_MNBT` / `Q38_CALIB_SEQS` set the calibration step and slots; everything else inherits +production. The tap is **read-bound**: a 7,168-token document cut into four 2,048-token steps +re-streams each layer's routed expert union four times, so a wider step is worth up to ~4× (the +runner's own header says this). + +Wide steps are refused by the arch, not the budget, and the refusal scales with the window: + +``` +QSA score plane = rows x (--max-model-len / 4) x 4 bytes +16,384 rows @ 200,000 window -> 3,125 MiB -> REFUSED at boot +16,384 rows @ 32,768 window -> 512 MiB -> legal +``` + +The calibration corpus is chunked to 7,168 tokens, so **32,768 is the window to calibrate at** — +copying production's 200,000 window is what disqualifies the wide step. + +## Judging a run + +- **A quiet log is not a hang.** The engine logs one line per *completed* request; at 2-way a 7.5 K + prefill takes ~4 min. Watch `kv_tokens_used` (steps of the chunk size) and the `req cmpl` count. + A watchdog on log mtime kills healthy cycles and throws away accumulated tokens. +- **Sealing is success.** The engine exits by itself; a mid-flight client gets + `429 rate_limit_error: the server is shutting down`. +- **Kill by pid** (`sudo fuser /dev/kfd`), never `pkill -f`: an orphaned engine keeps the device and + the next boot dies in ~20 s with `E startup stopped: device error`, which reads like a bad shape. diff --git a/agent/managed-skills/radiance-calibration-tap-latest/SKILL.md b/agent/managed-skills/radiance-calibration-tap-latest/SKILL.md new file mode 100644 index 0000000..f0aeedb --- /dev/null +++ b/agent/managed-skills/radiance-calibration-tap-latest/SKILL.md @@ -0,0 +1,85 @@ +--- +name: radiance-calibration-tap-latest +description: "Capture radiance calibration Grams for GPTQ (RADIANCE_CALIB_DIR) on the R9700 rig: the tap only arms on latest, the fitting shape, the 12M floor rule, and how to read a run that looks hung." +--- + +Friction-earned 2026-10-30 on `dank/serverAppConfigs` (qwen38-w5nl64 campaign, 2×R9700 TP=2). + +## The tap arms only on the latest radiance image + +A 17-hour calib run on the *production* pin (1.8.1) wrote **zero bytes** into `RADIANCE_CALIB_DIR` +while `/health`, `/stats` and the pusher all reported healthy. The tap's arming code is byte-identical +in both trees — the shipped 1.8.1 build drops it. The tell is the boot line: + +``` +I[0] recorded passes: off, the program accumulates calibration Grams (gram_accum) +``` + +Present on latest (v1.14.0 = `:latest` 2026-10-30), absent on 1.8.1. Verify with +`podman inspect IMG --format '{{json .Labels}}'` → `org.opencontainers.image.version` / +`revision`, and pin the **index digest**, not the tag: `:latest` moves under you, and a local stale +`:latest` can be a 3-week-old `dev` build (check `.Created`). + +Run the tap from a report container of the *same* model, but the tap needs `--ngram-placement disk` +for a calibration boot: the n-gram table pinned to RAM (`ram`) OOMs the rig's boot +(`startup stopped: out of memory`) while kldref/score boots succeed at the same pool. + +## The fitting shape (vendor recipe does not boot 2×32 GiB) + +The vendor's calibration line (`--max-num-batched-tokens 65536 --max-num-seqs 64`, ~930 tok/s) +assumes a bigger box: the activation arena alone is 12.83 GiB of 27.95 GiB claimable and every +variant stops in the VRAM resolver (short 3.19 GiB → 279 MiB → 59 MiB → 86 MiB → cache starved). +Wider steps are refused by the arch itself, pre-allocation: + +``` +qwen4exp: the QSA score plane would be 3125 MiB -- 16384 rows (--max-num-batched-tokens 16384) + by --max-model-len 200000 / compress ratio 4. Lower either. +``` + +What boots and seals: 4 slots, `--max-num-batched-tokens 2048` (the file-tier ceiling), +`--kv-cache-dtype fp8`, `--weights-disk-tier`, `--placement expert_tiered`, +`--ngram-placement disk`, `--max-model-len 200000`, `--num-speculative-tokens 0`, `--deterministic`. +Ledger at that shape: `elastic 22.8 GiB — experts first, kv cache 1.50 GiB … 1.0 full-length` +(pool ≈ 199,948 tokens). On 1.8.1 the same payload's pool was 7,436 tokens — the resolver, not the +window, is what changed. + +## Floor + +`RADIANCE_CALIB_TOKENS` is a **pooled** token floor. Engine default 65,536. Fleet rule (Frank, +2026-10-30): **12,000,000**, because the served quant this arm competes with was calibrated on a +10 M-token set. The vendor's "10 M beats 1 M by under 1 %" measures *their* plate, not ours. +Consequence: a ~9.3 M-token corpus round seals in round 1 plus part of round 2, and a round that +seals nothing is NORMAL (`round_verdict` → continue, never failure). + +## Reading a run (the trap that cost two cycles) + +- Throughput is **streaming-bound**, ~60–70 tok/s whatever the slot count; concurrency only lengthens + each request (2-way ≈ 4 min per 7.5 K-token prefill, 4-way ≈ 8). +- The engine logs **one line per completed request**. A log quiet for 8 minutes is therefore normal; + `W step: nothing has been logged for 21 s` is step-level silence. Liveness = **completed-request + count**, never log mtime. A watchdog on mtime kills healthy cycles and every false positive throws + away accumulated tap tokens. +- One seal writes **~43 GB** (24,768 `gram.r.blk..ffn_down_exps..bin`) and the engine + then **exits by itself**; a mid-flight client gets `429 rate_limit_error: the server is shutting + down` — success, not failure. +- A port that answers is not the rig: after the runner stops production the dying process answers + `:7770` briefly. Gate on `podman ps` naming the rig container, and on the `.rig-boot` marker. + +## Orchestration and space + +Use the repo's orchestrator, never a hand-rolled driver: +`bash scripts/qwen38-calib-window.sh` (gates: reference present, `.rad` present, tap dir ≥ 60 GiB, +healthy rig, marker, then push; banks the reference container only on success). Knobs: +`FRANK_QWEN38_CALIB_TOKENS`, `Q38_WINDOW_CALIB_CORPUS`, `Q38_WINDOW_ROUNDS`, +`Q38_CALIB_MIN_FREE_MIB`, `Q38_WINDOW_KEEP_RAD`. + +A hand-rolled driver skips the space preflight; on this host the tap dir sat on `/` with 35 G free +while one seal writes 43 GB — the orchestrator refuses that, which is the point. Frank's rule: keep +artifacts on `/`, never `/mnt/entropy`. + +## Then + +`gram_files → scripts/grams-to-imatrix.py DIR -o imatrix.gguf` (stdlib-only, skips non-gram names +like `.rig-boot`) → `scripts/convert-qwen38-w5nl64.sh --plan-only` (space bill, `--plan-only` first) +→ convert (card-less, CPU BLAS) → `score` each arm + the served payload at the **production shape** → +`scripts/kld-gate-qwen38.sh` (mean KL ≤ 0.03, p99.9, top-1 ≥ 95 %, every `by_source` slice). diff --git a/agent/managed-skills/radiance-tap-run-observability/SKILL.md b/agent/managed-skills/radiance-tap-run-observability/SKILL.md new file mode 100644 index 0000000..dd2162a --- /dev/null +++ b/agent/managed-skills/radiance-tap-run-observability/SKILL.md @@ -0,0 +1,63 @@ +--- +name: radiance-tap-run-observability +description: "Judge a live radiance calibration-tap run: which pin actually arms the tap, why a quiet engine log is not a hang, the watchdog that does not throw away accumulated grams, the seal's side effects, and the container-vs-port gate before pushing." +--- + +# Judging a live radiance calibration-tap run + +Use when a tap (layer Hessians for `rad-convert`'s `gptq:calib=`) is running for hours on a drained +endpoint and you must tell progress from a stall without destroying the accumulation. + +## Which image actually arms the tap + +- **Verify `:latest` by digest, not by tag or by a local label.** `podman pull` then + `podman inspect … --format '{{.Labels}} {{.Created}}'`: a working box can hold a *stale* local + `:latest` whose label reads `version=dev/revision=unknown` while the registry's `latest` is a real + release (on 2026-10-30 that was v1.14.0, revision `4a61d7296bc5`). Compare the two before blaming + a version. +- The arming line is the only ground truth that the tap is in the program: + `I[0] recorded passes: off, the program accumulates calibration Grams (gram_accum)`. + Its absence means no tap, however healthy `/health`, `/stats` and the pusher look — that is how a + 17-hour run on a production pin wrote zero bytes while everything reported fine. +- Reading source proves nothing about a build: the tap code was byte-identical in the version that + arms and the one that never writes. Judge the shipped image. + +## A quiet log is not a hang + +- The engine logs **one line per completed request** (`I req cmpl-… prompt= tok`). At 60–70 tok/s + of prefill a 7.5 K-token document takes ~2 min alone, ~4 min at 2-way and ~8 min at 4-way + concurrency, and **nothing is logged in between**. +- Concurrently, `W step: nothing has been logged for 21 s` fires on *step*-level silence. It is a + diagnostic, not a verdict: `/health` stays 200, `/stats` keeps reporting `running: N`, and + `kv_tokens_used` still steps in batch-sized chunks. +- So a watchdog keyed on log mtime kills healthy cycles, and every kill discards the accumulation + (grams only materialise at the seal). **Key the watchdog on the completed-request count** and give + it a threshold far above the worst legitimate request latency (45 min for a 1.5 M-token floor). +- Cheap liveness triple: `grep -c "req cmpl" log` climbing, `kv_tokens_used` moving, `podman ps` + showing the rig container. Throughput is bounded by expert streaming, so raising concurrency + lengthens requests without raising total tokens/s. + +## The seal and what it does + +- One seal is ~43 GB (24,768 `gram.r.blk..ffn_down_exps..bin` files) — size the tap + filesystem before starting, and rotate any previous tap out first. +- The engine **exits by itself when the tap seals**. A client mid-flight then sees + `429 rate_limit_error: the server is shutting down`: that is success, not failure. +- The seal line is `radiance: calibration wrote layer Hessians after tokens to /data/calib`. + Since the tap materialises all at once, "grams > 1000 files" is a valid post-seal trigger for a + chained imatrix → convert pipeline. + +## Gate on the container, not the port + +After a script stops the production service, the dying process still answers `:7770` for seconds, so +a health check can pass against the wrong thing. Require `podman ps` to name the rig container +**and** health to pass. A pusher that trusts the port alone pushes into a dead rig. + +## Cards must be exclusive + +An orphaned engine keeps `/dev/kfd`, and the next container boot dies with +`E startup stopped: device error` — which looks like a bad argument and is really a busy device. +Kill by pid from `sudo fuser /dev/kfd`, never `pkill -f`; verify `fuser` is empty before the next +boot. A genuinely wedged driver is a different failure: the *production* service then fails to serve +too, and only a reboot clears it — so prove the card healthy by restarting production and taking one +real chat turn before blaming your own shape. diff --git a/agent/memories/sharpshooter/omp-32c11n2wnglul/state.json b/agent/memories/sharpshooter/omp-32c11n2wnglul/state.json index 2ef7ea0..77923c9 100644 --- a/agent/memories/sharpshooter/omp-32c11n2wnglul/state.json +++ b/agent/memories/sharpshooter/omp-32c11n2wnglul/state.json @@ -1,6 +1,6 @@ { "v": 1, - "lastConsolidatedAt": 1793314235443, + "lastConsolidatedAt": 1793399844747, "lastResult": { "at": 1791599865775, "sessions": 1, diff --git a/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/architecture.md b/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/architecture.md index 7a574f1..4e09419 100644 --- a/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/architecture.md +++ b/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/architecture.md @@ -1,4 +1,13 @@ ## Structural Standards Quant quality must hold regardless of implementation choice. + Container versions must be digest-pinned to latest snapshot. + +## Model Corpus Constraints + +Use the 1M corpus threshold for payload serving decisions. + +## Merging Strategy + +Merge anything that hasn't been merged yet, and let us try that faster shape. \ No newline at end of file diff --git a/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/product.md b/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/product.md index 9654ad2..d46c053 100644 --- a/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/product.md +++ b/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/product.md @@ -2,4 +2,4 @@ ## Behavioral Updates -Merge any fixes immediately. +Merge fixes immediately. \ No newline at end of file diff --git a/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/state.json b/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/state.json index d95ce62..d0aa4ca 100644 --- a/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/state.json +++ b/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/state.json @@ -1,8 +1,8 @@ { "v": 1, - "lastConsolidatedAt": 1793398614775, + "lastConsolidatedAt": 1793402274788, "lastResult": { - "at": 1793397014344, + "at": 1793401924358, "sessions": 1, "deltas": 1, "model": "frank-zealot" diff --git a/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/style.md b/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/style.md index e69de29..bc78e06 100644 --- a/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/style.md +++ b/agent/memories/sharpshooter/wt-20261027-142335-6d1fc1f-1hpiehxnq73s0/style.md @@ -0,0 +1 @@ +## Style Standards diff --git a/agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/architecture.md b/agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/architecture.md new file mode 100644 index 0000000..e69de29 diff --git a/agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/product.md b/agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/product.md new file mode 100644 index 0000000..33bf2bd --- /dev/null +++ b/agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/product.md @@ -0,0 +1,2 @@ +## Builds +- clear out those harbor builds \ No newline at end of file diff --git a/agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/state.json b/agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/state.json index a9efb0b..35e16d0 100644 --- a/agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/state.json +++ b/agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/state.json @@ -1,4 +1,10 @@ { "v": 1, - "lastConsolidatedAt": 1793398691951 + "lastConsolidatedAt": 1793402231969, + "lastResult": { + "at": 1793400701944, + "sessions": 1, + "deltas": 1, + "model": "frank-zealot" + } } diff --git a/agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/style.md b/agent/memories/sharpshooter/wt-20261030-143311-6d1fc1f-3qg6yco5geckj/style.md new file mode 100644 index 0000000..e69de29 diff --git a/agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/state.json b/agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/state.json index 0bd20e4..5a74a1b 100644 --- a/agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/state.json +++ b/agent/memories/sharpshooter/wt-20261030-144848-6d1fc1f-1n8ll6xrxgcec/state.json @@ -1,6 +1,6 @@ { "v": 1, - "lastConsolidatedAt": 1793398788553, + "lastConsolidatedAt": 1793401788576, "lastResult": { "at": 1793397235249, "sessions": 1, diff --git a/agent/memories/sharpshooter/wt-20261030-153725-7c8a99d-2jvaw6dcu0qzt/architecture.md b/agent/memories/sharpshooter/wt-20261030-153725-7c8a99d-2jvaw6dcu0qzt/architecture.md new file mode 100644 index 0000000..e69de29 diff --git a/agent/memories/sharpshooter/wt-20261030-153725-7c8a99d-2jvaw6dcu0qzt/product.md b/agent/memories/sharpshooter/wt-20261030-153725-7c8a99d-2jvaw6dcu0qzt/product.md new file mode 100644 index 0000000..b17b386 --- /dev/null +++ b/agent/memories/sharpshooter/wt-20261030-153725-7c8a99d-2jvaw6dcu0qzt/product.md @@ -0,0 +1,2 @@ +## Product behavior +- Normalize monitor rotation: hdmi monitor left of eDP \ No newline at end of file diff --git a/agent/memories/sharpshooter/wt-20261030-153725-7c8a99d-2jvaw6dcu0qzt/state.json b/agent/memories/sharpshooter/wt-20261030-153725-7c8a99d-2jvaw6dcu0qzt/state.json new file mode 100644 index 0000000..e13ba35 --- /dev/null +++ b/agent/memories/sharpshooter/wt-20261030-153725-7c8a99d-2jvaw6dcu0qzt/state.json @@ -0,0 +1,10 @@ +{ + "v": 1, + "lastConsolidatedAt": 1793402365714, + "lastResult": { + "at": 1793401119393, + "sessions": 1, + "deltas": 1, + "model": "frank-zealot" + } +} diff --git a/agent/memories/sharpshooter/wt-20261030-153725-7c8a99d-2jvaw6dcu0qzt/style.md b/agent/memories/sharpshooter/wt-20261030-153725-7c8a99d-2jvaw6dcu0qzt/style.md new file mode 100644 index 0000000..e69de29 diff --git a/dotfiles/gtk-3.0/.config/gtk-3.0/noctalia.css b/dotfiles/gtk-3.0/.config/gtk-3.0/noctalia.css index 3f71f07..d639805 100644 --- a/dotfiles/gtk-3.0/.config/gtk-3.0/noctalia.css +++ b/dotfiles/gtk-3.0/.config/gtk-3.0/noctalia.css @@ -4,6 +4,9 @@ * stylelint-disable at-rule-no-unknown */ +/* App-scoped CSS of enabled community GTK extensions; Noctalia maintains this file. */ +@import url("noctalia-extensions.css"); + @define-color accent_color #f43841; @define-color accent_bg_color #f43841; @define-color accent_fg_color #181818; diff --git a/dotfiles/gtk-4.0/.config/gtk-4.0/noctalia.css b/dotfiles/gtk-4.0/.config/gtk-4.0/noctalia.css index eeee41b..7aed22d 100644 --- a/dotfiles/gtk-4.0/.config/gtk-4.0/noctalia.css +++ b/dotfiles/gtk-4.0/.config/gtk-4.0/noctalia.css @@ -4,6 +4,9 @@ * stylelint-disable at-rule-no-unknown */ +/* App-scoped CSS of enabled community GTK extensions; Noctalia maintains this file. */ +@import url("noctalia-extensions.css"); + @define-color accent_color #f43841; @define-color accent_bg_color #f43841; @define-color accent_fg_color #181818; diff --git a/dotfiles/noctalia/.local/state/noctalia/settings.toml b/dotfiles/noctalia/.local/state/noctalia/settings.toml index ed28f59..c17ee93 100644 --- a/dotfiles/noctalia/.local/state/noctalia/settings.toml +++ b/dotfiles/noctalia/.local/state/noctalia/settings.toml @@ -107,11 +107,11 @@ widget_order = [ [lockscreen_widgets.widget."lockscreen-login-box@DP-2"] box_height = 196.0 box_width = 810.0 - cx = 599.9998779296875 - cy = 1628.8001708984375 + cx = 959.99981689453125 + cy = 1018.0001220703125 output = "DP-2" - placement_height = 1920.0 - placement_width = 1200.0 + placement_height = 1200.0 + placement_width = 1920.0 rotation = 0.0 type = "login_box" @@ -135,11 +135,11 @@ widget_order = [ [lockscreen_widgets.widget."lockscreen-login-box@HDMI-A-1"] box_height = 196.0 box_width = 720.0 - cx = 600.0 - cy = 1628.800048828125 + cx = 960.0 + cy = 1018.0 output = "HDMI-A-1" - placement_height = 1920.0 - placement_width = 1200.0 + placement_height = 1200.0 + placement_width = 1920.0 rotation = 0.0 type = "login_box"