From d99c1c3c4fda96a09650a8e4d864252c0f80ef6d Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 19 Oct 2026 03:04:41 -0700 Subject: [PATCH] live: heartbeat sync 2026-10-19T03:04:41-07:00 (Refs #11) --- .../fleet-service-runtime-census/SKILL.md | 62 +++++++++++++++++++ .../strix-halo-npu-readiness-census/SKILL.md | 44 +++++++++++++ .../serverAppConfig-eu5lc7uqff33/state.json | 2 +- .../state.json | 2 +- .../architecture.md | 0 .../product.md | 0 .../state.json | 8 ++- .../style.md | 0 .../architecture.md | 0 .../product.md | 0 .../state.json | 10 +++ .../style.md | 0 .../architecture.md | 0 .../product.md | 0 .../state.json | 10 +++ .../style.md | 0 .../.local/state/noctalia/settings.toml | 8 +-- 17 files changed, 139 insertions(+), 7 deletions(-) create mode 100644 agent/managed-skills/fleet-service-runtime-census/SKILL.md create mode 100644 agent/managed-skills/strix-halo-npu-readiness-census/SKILL.md create mode 100644 agent/memories/sharpshooter/wt-20261019-011132-7c8a99d-y69mcckuzbqu/architecture.md create mode 100644 agent/memories/sharpshooter/wt-20261019-011132-7c8a99d-y69mcckuzbqu/product.md create mode 100644 agent/memories/sharpshooter/wt-20261019-011132-7c8a99d-y69mcckuzbqu/style.md create mode 100644 agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/architecture.md create mode 100644 agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/product.md create mode 100644 agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/state.json create mode 100644 agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/style.md create mode 100644 agent/memories/sharpshooter/wt-20261019-021028-6d1fc1f-3p69q8c3d94kw/architecture.md create mode 100644 agent/memories/sharpshooter/wt-20261019-021028-6d1fc1f-3p69q8c3d94kw/product.md create mode 100644 agent/memories/sharpshooter/wt-20261019-021028-6d1fc1f-3p69q8c3d94kw/state.json create mode 100644 agent/memories/sharpshooter/wt-20261019-021028-6d1fc1f-3p69q8c3d94kw/style.md diff --git a/agent/managed-skills/fleet-service-runtime-census/SKILL.md b/agent/managed-skills/fleet-service-runtime-census/SKILL.md new file mode 100644 index 0000000..c95a005 --- /dev/null +++ b/agent/managed-skills/fleet-service-runtime-census/SKILL.md @@ -0,0 +1,62 @@ +--- +name: fleet-service-runtime-census +description: "Answer \"what is service running right now?\" for a dank-fleet service: live endpoint probe plus host units, image ids, live-vs-repo env, and drop-ins." +--- + +# Fleet service runtime census + +Answer the question "what is `` running currently?" with evidence, not from docs. Docs say what *should* run; the host says what *does*. Run all five probes, then report one table. + +## 1. Probe the published port (works from any LAN box, no ssh) + +```bash +curl -s -m 10 http://:/health; echo +curl -s -m 10 http://:/v1/audio/voices # per-endpoint extras: /v1/models, /metrics +``` + +The extra route matters more than `/health` for identity claims: +- llama.cpp and radiance **echo the requested model name** — a 200 proves nothing; read `/v1/models`. +- halogen 404s `/completion`, `/ping`, `/props`, `/tokenize`. +- The frank-tts façade answers `/health` **without dialing the engine**, so "façade healthy, engine down" is a real state; the voice list in that same response is the honest census. + +## 2. Units + containers on the host (ssh as the deploy user, no sudo) + +```bash +ssh 'systemctl --user status .service .service --no-pager \ + | grep -E "^(●| Active| Loaded)"; \ + podman images --format "{{.Repository}}:{{.Tag}} {{.ID}} {{.Created}}"; \ + podman ps -a --format "{{.Names}}\t{{.Status}}\t{{.Image}}"' +``` + +- `active (running) since …` + `Up N hours (healthy)` is the answer; `activating (auto-restart)` with a ~50 ms exit is a **preflight refusal**, not an engine fault. +- **NEVER prefix these podman calls with `sudo`.** Containers live in the deploy user's rootless storage; `sudo podman inspect frank-tts-engine` returns `no such object` while the plain form succeeds. Treating that as "container missing" is a false finding. + +## 3. Drop-ins — the override that no health gate sees + +```bash +ssh 'systemctl --user show .service -p DropInPaths; \ + systemctl --user show .service -p DropInPaths' +``` + +`DropInPaths=` must be empty. A leftover bench drop-in (`Environment=FRANK__ENV_FILE=/tmp/.env`) makes the next deploy boot a different model at a different shape while `/health` still answers 200 and the e2e gate passes. + +## 4. Live env vs repo env — the only truth test + +```bash +# repo side (local checkout) +grep -vE '^\s*#|^\s*$' scripts/.env +# host side +ssh 'sudo cat /home/dank/serverAppConfigs/scripts/.env' | grep -vE '^\s*#|^\s*$' +``` + +Diff them. Per repo law the env file beside the serve script **is** the authority; a doc page that disagrees with it is the bug, and host-side drift means an uncommitted edit. Also check `git log --oneline -3 -- scripts/.env` to confirm the host isn't ahead of git. + +## 5. Report + +Table of: unit state + since, image repo:tag@short-ID + age, backend/alias actually served (from step 1, not from the name), drop-ins, and whether live env == repo env. State explicitly when a fact came from docs rather than from a probe. + +## Traps that produce false findings + +- Digest-pinned images: the local image ID's *creation date* answers "is this stale" better than the tag; `latest` + `Pull=never` is a locally built image, not a registry pull. +- `ssh` user context changes what podman sees (root vs rootless) — see step 2. +- Two ports on one host are two providers in `models.yml`; a port mismatch is a rename or drift, not a config typo. diff --git a/agent/managed-skills/strix-halo-npu-readiness-census/SKILL.md b/agent/managed-skills/strix-halo-npu-readiness-census/SKILL.md new file mode 100644 index 0000000..d0663d3 --- /dev/null +++ b/agent/managed-skills/strix-halo-npu-readiness-census/SKILL.md @@ -0,0 +1,44 @@ +--- +name: strix-halo-npu-readiness-census +description: "Engine-agnostic census of whether an AMD Strix Halo / Ryzen AI host can run anything on its XDNA2 NPU: driver bind, firmware, device perms, IOMMU, distro XRT, missing AIE compiler, fabric clock, RAM headroom." +--- + +Use when asked "can we run on the Strix Halo / Ryzen AI NPU?" for a runtime that is **not** halogen (halogen has its own gate + vendor catalog: use `halogen-npu-feasibility-census` for that). This answers only: is the *host* able, and what is missing. + +## Why it is usually the wrong door first + +XDNA2 executes **precompiled AIE partition programs** loaded through XRT → `amdxdna`. There is no generic tensor dispatch: an engine built on ggml/llama.cpp (or any GGUF loader) cannot "enable" the NPU — the port must be a re-export + vendor compile, not a flag. So pair this census with the accelerator the same engine *can* select by build flag (Vulkan/HIP on the gfx1151 iGPU) before proposing NPU work. + +## Census (all read-only, over ssh from the operator box) + +```bash +ssh ' +cat /proc/cmdline; ls /sys/kernel/iommu_groups | wc -l # iommu=pt required; groups==0 => IOMMU off => NPU dead +lsmod | grep -i amdxdna # in-tree driver +sudo dmesg | grep -iE "xdna|npu" | tail # "Initialized amdxdna_accel_driver for 0000:c6:00.1" +lspci -nn | grep -iE "npu|processing" # Strix Halo = 1022:17f0 +ls /sys/bus/pci/drivers/amdxdna # must list the BDF +ls -l /dev/accel # accel0, root:render 660 +id -nG | tr " " "\n" | grep -E "render|video" # rootless podman needs render, else --device fails at create +find /usr/lib/firmware -iname "*npu*" | head # firmware is /usr/lib/firmware/amdnpu/_/npu.sbin*.zst — match the PCI rev +dpkg -l | grep -iE "xrt|xdna"; ls /opt/xilinx/xrt/lib 2>&1 # distro libxrt-npu2 = XDNA2 runtime; vendor /opt/xilinx/xrt usually absent +apt-cache policy libxrt2 libxrt-npu2 | grep Candidate +apt-cache search onnxruntime | head # distro ONNX RT exists; the Ryzen AI EP does not +python3 -c "import onnxruntime" 2>&1 | tail -1 # absence of the EP + of ryzenai-compile is the real blocker +for c in /sys/class/drm/card*/device; do echo "-- $c"; cat $c/power_dpm_force_performance_level 2>&1; cat $c/pp_dpm_fclk 2>&1 | tr "\n" " "; echo; done +nproc; free -g | head -2 # RAM headroom + thread count the engine will pick +' +``` + +## Reading it + +- Driver bound + `accel0` + firmware dir + `iommu=pt` ⇒ **kernel side is ready**; do not re-litigate "is the NPU disabled". A missing device node is *not* the modern failure mode. +- `auto` on `pp_dpm_fclk` / `power_dpm_force_performance_level` ⇒ fabric clock **unheld**. GPU+NPU concurrently with a moving fclk hangs these hosts and corrupts NPU results; any NPU plan must pin the top fclk step (or run in a container holding it) first. +- No `ryzenai-compile`/ONNX Ryzen AI EP ⇒ nothing on the host can produce a partition program. That is the enablement delta, and it is a toolchain install + a model-specific export/quantize/compile project, not a config change. +- `free` headroom matters: NPU partitions want a few GB of host RAM each and can log benign `Failed to alloc dev bo memory, ret -28` under pressure. + +## Laws / traps + +- Read state only. Never probe or load the NPU beside a live inference service — stop + mask the service first (fleet benching policy); this host class has a precedent of a concurrent GPU/NPU experiment needing a power cycle. +- `sudo podman ...` finds nothing (per-user rootless storage); run podman as the deploy user. +- `/dev/accel/accel0` existing does not mean a context/PASID can be created; with IOMMU off the failure is at context creation, not at the node. diff --git a/agent/memories/sharpshooter/serverAppConfig-eu5lc7uqff33/state.json b/agent/memories/sharpshooter/serverAppConfig-eu5lc7uqff33/state.json index f446c9a..f620de2 100644 --- a/agent/memories/sharpshooter/serverAppConfig-eu5lc7uqff33/state.json +++ b/agent/memories/sharpshooter/serverAppConfig-eu5lc7uqff33/state.json @@ -1,4 +1,4 @@ { "v": 1, - "lastConsolidatedAt": 1792386375195 + "lastConsolidatedAt": 1792401025778 } diff --git a/agent/memories/sharpshooter/wt-20261017-181502-6d1fc1f-1oq2osmoyw03k/state.json b/agent/memories/sharpshooter/wt-20261017-181502-6d1fc1f-1oq2osmoyw03k/state.json index d08cf5c..9429915 100644 --- a/agent/memories/sharpshooter/wt-20261017-181502-6d1fc1f-1oq2osmoyw03k/state.json +++ b/agent/memories/sharpshooter/wt-20261017-181502-6d1fc1f-1oq2osmoyw03k/state.json @@ -1,6 +1,6 @@ { "v": 1, - "lastConsolidatedAt": 1792400553633, + "lastConsolidatedAt": 1792400853634, "lastResult": { "at": 1792393886960, "sessions": 1, diff --git a/agent/memories/sharpshooter/wt-20261019-011132-7c8a99d-y69mcckuzbqu/architecture.md b/agent/memories/sharpshooter/wt-20261019-011132-7c8a99d-y69mcckuzbqu/architecture.md new file mode 100644 index 0000000..e69de29 diff --git a/agent/memories/sharpshooter/wt-20261019-011132-7c8a99d-y69mcckuzbqu/product.md b/agent/memories/sharpshooter/wt-20261019-011132-7c8a99d-y69mcckuzbqu/product.md new file mode 100644 index 0000000..e69de29 diff --git a/agent/memories/sharpshooter/wt-20261019-011132-7c8a99d-y69mcckuzbqu/state.json b/agent/memories/sharpshooter/wt-20261019-011132-7c8a99d-y69mcckuzbqu/state.json index 9487df4..6321238 100644 --- a/agent/memories/sharpshooter/wt-20261019-011132-7c8a99d-y69mcckuzbqu/state.json +++ b/agent/memories/sharpshooter/wt-20261019-011132-7c8a99d-y69mcckuzbqu/state.json @@ -1,4 +1,10 @@ { "v": 1, - "lastConsolidatedAt": 1792400552453 + "lastConsolidatedAt": 1792404212471, + "lastResult": { + "at": 1792401456699, + "sessions": 1, + "deltas": 1, + "model": "frank-zealot" + } } diff --git a/agent/memories/sharpshooter/wt-20261019-011132-7c8a99d-y69mcckuzbqu/style.md b/agent/memories/sharpshooter/wt-20261019-011132-7c8a99d-y69mcckuzbqu/style.md new file mode 100644 index 0000000..e69de29 diff --git a/agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/architecture.md b/agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/architecture.md new file mode 100644 index 0000000..e69de29 diff --git a/agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/product.md b/agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/product.md new file mode 100644 index 0000000..e69de29 diff --git a/agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/state.json b/agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/state.json new file mode 100644 index 0000000..e73c19d --- /dev/null +++ b/agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/state.json @@ -0,0 +1,10 @@ +{ + "v": 1, + "lastConsolidatedAt": 1792404069868, + "lastResult": { + "at": 1792402864677, + "sessions": 1, + "deltas": 8, + "model": "frank-zealot" + } +} diff --git a/agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/style.md b/agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/style.md new file mode 100644 index 0000000..e69de29 diff --git a/agent/memories/sharpshooter/wt-20261019-021028-6d1fc1f-3p69q8c3d94kw/architecture.md b/agent/memories/sharpshooter/wt-20261019-021028-6d1fc1f-3p69q8c3d94kw/architecture.md new file mode 100644 index 0000000..e69de29 diff --git a/agent/memories/sharpshooter/wt-20261019-021028-6d1fc1f-3p69q8c3d94kw/product.md b/agent/memories/sharpshooter/wt-20261019-021028-6d1fc1f-3p69q8c3d94kw/product.md new file mode 100644 index 0000000..e69de29 diff --git a/agent/memories/sharpshooter/wt-20261019-021028-6d1fc1f-3p69q8c3d94kw/state.json b/agent/memories/sharpshooter/wt-20261019-021028-6d1fc1f-3p69q8c3d94kw/state.json new file mode 100644 index 0000000..4463703 --- /dev/null +++ b/agent/memories/sharpshooter/wt-20261019-021028-6d1fc1f-3p69q8c3d94kw/state.json @@ -0,0 +1,10 @@ +{ + "v": 1, + "lastConsolidatedAt": 1792404208554, + "lastResult": { + "at": 1792401394555, + "sessions": 1, + "deltas": 1, + "model": "frank-zealot" + } +} diff --git a/agent/memories/sharpshooter/wt-20261019-021028-6d1fc1f-3p69q8c3d94kw/style.md b/agent/memories/sharpshooter/wt-20261019-021028-6d1fc1f-3p69q8c3d94kw/style.md new file mode 100644 index 0000000..e69de29 diff --git a/dotfiles/noctalia/.local/state/noctalia/settings.toml b/dotfiles/noctalia/.local/state/noctalia/settings.toml index 1b0be25..b7b08a5 100644 --- a/dotfiles/noctalia/.local/state/noctalia/settings.toml +++ b/dotfiles/noctalia/.local/state/noctalia/settings.toml @@ -275,13 +275,13 @@ transition = [ "fade" ] interval_seconds = 1200 [wallpaper.default] - path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_g7r3vd.jpg" + path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_v9w8r5.jpg" [wallpaper.last] - path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_g7r3vd.jpg" + path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_v9w8r5.jpg" [wallpaper.monitors.DP-1] - path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_g7r3vd.jpg" + path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_v9w8r5.jpg" [wallpaper.monitors.DP-2] path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_oxex69.jpg" @@ -290,7 +290,7 @@ transition = [ "fade" ] path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_v9e2w3.jpg" [wallpaper.monitors.eDP-1] - path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_g7r3vd.jpg" + path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_v9w8r5.jpg" [weather] unit = "imperial"