Compare commits
53
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1aeb7af6df | ||
|
|
cd0be97b39 | ||
|
|
1bb6399524 | ||
|
|
aa75b45543 | ||
|
|
b1fc1113c6 | ||
|
|
9f7457e39e | ||
|
|
687dbfbabe | ||
|
|
961713e92d | ||
|
|
01db29e754 | ||
|
|
ab06e4c221 | ||
|
|
1006e96312 | ||
|
|
7d85fed1c9 | ||
|
|
d8196ce499 | ||
|
|
6f442a14a3 | ||
|
|
afe72781bc | ||
|
|
200039d589 | ||
|
|
91d9864bb0 | ||
|
|
980491668c | ||
|
|
e494617599 | ||
|
|
9809ed9c74 | ||
|
|
634fef242b | ||
|
|
2fe68e69af | ||
|
|
70dd7fbe6d | ||
|
|
2626332828 | ||
|
|
eb4d9910cb | ||
|
|
5cc2b3c25a | ||
|
|
18a29c40d2 | ||
|
|
73a7ec3fc1 | ||
|
|
130e9d739f | ||
|
|
854890a155 | ||
|
|
d9f3d1be1e | ||
|
|
afa3a4521c | ||
|
|
9cc8ca1c88 | ||
|
|
9c05c40aa0 | ||
|
|
63219787af | ||
|
|
e914ecbca6 | ||
|
|
1eebf3b129 | ||
|
|
af5a1f69a2 | ||
|
|
0930510f9b | ||
|
|
af327bcc5e | ||
|
|
fcf94562d1 | ||
|
|
861920797d | ||
|
|
3ef711f146 | ||
|
|
823fd52d0e | ||
|
|
bda011d26f | ||
|
|
b19f1c49d6 | ||
|
|
b1ca4e02fa | ||
|
|
7f65cf0252 | ||
|
|
346e1d22f4 | ||
|
|
53ef121180 | ||
|
|
fa093e617d | ||
|
|
8bac97ec87 | ||
|
|
ee6d8610b3 |
@@ -166,9 +166,11 @@ bare compose container never returned; 2026-09-11: five observability
|
||||
services had no units). The observability services now have units under
|
||||
`dankclaw/observability/quadlet/` and their compose definitions were removed —
|
||||
one owner per container name. The stack is **configured and delivering as of
|
||||
2026-09-17**: a Grafana datasource and the `llm-fleet` dashboard are
|
||||
provisioned from `shared/grafana/provisioning/`, `shared/prometheus/rules/
|
||||
llm-fleet.yml` loads 5 alert rules + 1 recording rule, and firing rules are
|
||||
2026-09-17**: Grafana datasources (Prometheus and Loki) and the `llm-fleet`
|
||||
dashboard are provisioned from `shared/grafana/provisioning/`, and
|
||||
`shared/prometheus/rules/llm-fleet.yml` loads two rule groups — `llm-fleet`
|
||||
(llama.cpp/halogen, `llamacpp:*`) and `vllm-fleet` (radiance, `vllm:*`) — alert
|
||||
+ recording rules, and firing rules are
|
||||
delivered: Prometheus → Alertmanager (`dankclaw/observability/quadlet/
|
||||
alertmanager.container`, loopback :9093) → host-side Signal relay
|
||||
(`scripts/signal-relay.py`, user unit, 192.168.0.143:18093 — the host LAN
|
||||
@@ -803,6 +805,20 @@ dies, its page is destroyed. Rollback is a commit, not a document.
|
||||
`serverAppConfigs-docs/index.md` is the locator for every page that survives;
|
||||
retired scripts live in `old/` and `.AGENTTRASH/`.
|
||||
|
||||
## Agent skills
|
||||
|
||||
### Issue tracker
|
||||
|
||||
Issues live in Gitea (`dank/serverAppConfigs` on Dankclaw), via its REST API with the stored credential. See `serverAppConfigs-docs/Agent/Issue Tracker.md`.
|
||||
|
||||
### Triage labels
|
||||
|
||||
The five canonical roles keep their default strings (`needs-triage`, `needs-info`, `ready-for-agent`, `ready-for-human`, `wontfix`). See `serverAppConfigs-docs/Agent/Triage Labels.md`.
|
||||
|
||||
### Domain docs
|
||||
|
||||
Single-context: root `GLOSSARY.md` + `docs/adr/`, created lazily; this vault stays the owner of operational facts. See `serverAppConfigs-docs/Agent/Domain Docs.md`.
|
||||
|
||||
## Why
|
||||
|
||||
Direct edits on the remote drift from version control, and future agents
|
||||
|
||||
Executable
+65
@@ -0,0 +1,65 @@
|
||||
#!/bin/sh
|
||||
# halogen-fabric-clock hold | release | status: the AMD GPU's fabric clock (FCLK), held at its top level.
|
||||
#
|
||||
# On Strix Halo the GPU and the NPU working at the same time can hang the machine, and corrupt the NPU's results,
|
||||
# while the GPU's fabric clock changes speed; held at its top level, the two run together cleanly. halogen-flash-server
|
||||
# with HALOGEN_NPU_MODELS refuses to start until the clock is held. This holds it: the performance level `manual` and
|
||||
# only the top pp_dpm_fclk level, on every amdgpu device that has the control (the GPU and SoC clocks stay automatic).
|
||||
# Cost: about 1 W at idle. Run as root; halogen-fabric-clock.service runs `hold` at every boot and after a resume.
|
||||
#
|
||||
# hold hold it (waits up to 60 s for the GPU driver to come up)
|
||||
# release give it back to the driver (the performance level `auto`)
|
||||
# status print each device's state; exit 0 when every device is held
|
||||
set -u
|
||||
SYS="${HALOGEN_SYSFS:-/sys}"
|
||||
|
||||
devices() {
|
||||
for d in "$SYS"/class/drm/card*/device; do
|
||||
[ -w "$d/pp_dpm_fclk" ] || [ -r "$d/pp_dpm_fclk" ] || continue
|
||||
[ -e "$d/power_dpm_force_performance_level" ] || continue
|
||||
echo "$d"
|
||||
done | sort -u
|
||||
}
|
||||
|
||||
held() { # held DEVICE_DIR: the performance level high, or manual with only the top fclk level selected
|
||||
perf=$(cat "$1/power_dpm_force_performance_level" 2>/dev/null)
|
||||
[ "$perf" = high ] && return 0
|
||||
[ "$perf" = manual ] || return 1
|
||||
top=$(grep . "$1/pp_dpm_fclk" | tail -n 1)
|
||||
[ "$(grep -c '\*' "$1/pp_dpm_fclk")" = 1 ] && [ "${top%\*}" != "$top" ]
|
||||
}
|
||||
|
||||
cmd="${1:-status}"
|
||||
case "$cmd" in
|
||||
hold)
|
||||
n=0
|
||||
while [ -z "$(devices)" ] && [ "$n" -lt 60 ]; do sleep 1; n=$((n + 1)); done
|
||||
[ -n "$(devices)" ] || { echo "halogen-fabric-clock: no amdgpu device with a fabric clock control (pp_dpm_fclk) under $SYS/class/drm" >&2; exit 1; }
|
||||
rc=0
|
||||
for d in $(devices); do
|
||||
lvl=$(grep . "$d/pp_dpm_fclk" | tail -n 1 | cut -d: -f1)
|
||||
# the driver applies the level several seconds after the write, so the read-back waits up to 30 s
|
||||
ok=0
|
||||
if echo manual > "$d/power_dpm_force_performance_level" && echo "$lvl" > "$d/pp_dpm_fclk"; then
|
||||
t=0; until held "$d" || [ "$t" -ge 300 ]; do sleep 0.1; t=$((t + 1)); done
|
||||
held "$d" && ok=1
|
||||
fi
|
||||
if [ "$ok" = 1 ]; then
|
||||
echo "halogen-fabric-clock: $d: held at level $lvl ($(grep . "$d/pp_dpm_fclk" | tail -n 1 | awk '{print $2}'))"
|
||||
else
|
||||
echo "halogen-fabric-clock: $d: could not hold it (level $lvl)" >&2; rc=1
|
||||
fi
|
||||
done
|
||||
exit "$rc" ;;
|
||||
release)
|
||||
for d in $(devices); do echo auto > "$d/power_dpm_force_performance_level" && echo "halogen-fabric-clock: $d: released (auto)"; done ;;
|
||||
status)
|
||||
[ -n "$(devices)" ] || { echo "halogen-fabric-clock: no amdgpu device with a fabric clock control"; exit 1; }
|
||||
rc=0
|
||||
for d in $(devices); do
|
||||
if held "$d"; then echo "$d: held ($(cat "$d/power_dpm_force_performance_level"))"
|
||||
else echo "$d: NOT held ($(cat "$d/power_dpm_force_performance_level"))"; rc=1; fi
|
||||
done
|
||||
exit "$rc" ;;
|
||||
*) echo "usage: halogen-fabric-clock hold | release | status" >&2; exit 2 ;;
|
||||
esac
|
||||
@@ -0,0 +1,19 @@
|
||||
# Holds the AMD GPU's fabric clock at its top level at boot and after every resume, for halogen-flash-server with
|
||||
# the NPU (HALOGEN_NPU_MODELS): on Strix Halo the GPU and the NPU at once are safe only with that clock held.
|
||||
# Install (as root), from this directory:
|
||||
# install -m 755 halogen-fabric-clock /usr/local/sbin/
|
||||
# install -m 644 halogen-fabric-clock.service /etc/systemd/system/
|
||||
# systemctl daemon-reload && systemctl enable --now halogen-fabric-clock.service
|
||||
# Check: halogen-fabric-clock status. Undo: systemctl disable halogen-fabric-clock.service, then
|
||||
# halogen-fabric-clock release (or reboot). The unit runs once at boot and once after each resume (a resume can hand
|
||||
# the clock back to the driver); it holds nothing open, so it shows inactive between runs.
|
||||
[Unit]
|
||||
Description=Hold the AMD GPU fabric clock at its top level (GPU and NPU together)
|
||||
After=systemd-modules-load.service suspend.target hibernate.target hybrid-sleep.target suspend-then-hibernate.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/halogen-fabric-clock hold
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target suspend.target hibernate.target hybrid-sleep.target suspend-then-hibernate.target
|
||||
@@ -1,55 +1,51 @@
|
||||
# Quadlet unit: alloy (rootless, boot-persistent).
|
||||
# Quadlet unit: alloy (ROOT system service, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/alloy.container
|
||||
# Unit name: alloy.service (container name: alloy)
|
||||
# Install path on dankclaw: /etc/containers/systemd/alloy.container
|
||||
# Unit name: alloy.service (container name: alloy) — SYSTEM manager, root.
|
||||
#
|
||||
# Log shipper. This replaces the compose service of the same name in
|
||||
# dankclaw/compose.yaml ("Observability" block).
|
||||
# WHY ROOT (not rootless, unlike its observability siblings). Alloy reads the
|
||||
# host journal. The journal files are root:systemd-journal 0640 (other::---);
|
||||
# a rootless container carries no supplementary groups and cannot open them —
|
||||
# a rootless read of system.journal returns DENIED (see the Alloy page in the
|
||||
# vault). Running as root, one loki.source.journal reads system.journal AND
|
||||
# user-1000.journal, so it carries both host logs and the rootless Quadlet
|
||||
# containers' logs. Validated 2026-10-20: a root probe read 31,825 journal
|
||||
# lines and pushed 26,966 into Loki in 20 s, zero drops.
|
||||
#
|
||||
# IMAGE. franksdank/alloy is built locally by deploy-remote.sh
|
||||
# (build_compose_image) from dankclaw/alloy/Containerfile: upstream
|
||||
# grafana/alloy plus a static busybox for the healthcheck. Pull=never keeps a
|
||||
# boot from reaching for that local tag at a registry.
|
||||
# NETWORK=host. Loki publishes 127.0.0.1:3100 (loopback only) on the host, and
|
||||
# no bridge container — rootless or root — can reach the host's loopback, so
|
||||
# this unit runs on the host network and pushes to 127.0.0.1:3100. The side
|
||||
# effect: the UI must bind loopback too, so Exec sets the listen-addr to
|
||||
# 127.0.0.1 (the old rootless unit used 0.0.0.0 only because container loopback
|
||||
# was not host loopback; under Network=host they are the same address, so the
|
||||
# unauthenticated UI stays loopback-only — nothing off-host reaches it).
|
||||
#
|
||||
# NETWORK. It joins dankclaw_default so its config's loki.write endpoint
|
||||
# resolves the container name loki:3100.
|
||||
# THE /etc/machine-id MOUNT IS LOAD-BEARING. sd_journal locates the journal
|
||||
# directory by the local machine id; without it the container opens nothing and
|
||||
# the journal source reads exactly 0 lines — silently, no error, health green.
|
||||
# This mount is the entire reason the pipeline carries data. Never remove it.
|
||||
#
|
||||
# THE UI PORT EXISTS ON PURPOSE. Alloy's UI listens on the container's own
|
||||
# 127.0.0.1 by default and cannot be moved from the config file (alloy 1.x
|
||||
# refuses the ui block), so the compose service had no reachable health
|
||||
# endpoint at all: the homepage status card for alloy reported "down"
|
||||
# continuously and there was nothing for a deploy to wait on. The command
|
||||
# below therefore moves the UI to the container's wildcard address and the
|
||||
# port is published on host loopback only, which puts alloy in the same class
|
||||
# as the other three: reachable from the host, by Caddy over the private
|
||||
# network, and by nothing off-host. Alloy has no authentication, which is
|
||||
# exactly why it is not published wider.
|
||||
#
|
||||
# SOURCES. The host journald socket and the host log/container-storage trees
|
||||
# are bind-mounted read-only; Alloy tails them and pushes to Loki. The config
|
||||
# lives at /opt/alloy/config/config.alloy, installed from
|
||||
# shared/alloy/config.alloy on every deploy.
|
||||
#
|
||||
# ORDERING. Wants/After loki.service: without a reachable Loki, Alloy's write
|
||||
# client fails and its log pipeline backs up.
|
||||
# IMAGE. Built rootless by build_compose_image alloy, then mirrored into the
|
||||
# root image store by the deployer (a root build runs in the build container's
|
||||
# own resolver and fails on DNS here). Pull=never: boot must not reach for the
|
||||
# local tag at a registry.
|
||||
|
||||
[Unit]
|
||||
Description=Alloy log shipper (Dankclaw)
|
||||
After=network-online.target loki.service
|
||||
Wants=network-online.target loki.service
|
||||
After=network-online.target
|
||||
|
||||
[Container]
|
||||
ContainerName=alloy
|
||||
Image=localhost/franksdank/alloy:latest
|
||||
Pull=never
|
||||
Network=dankclaw_default
|
||||
Exec=run --server.http.listen-addr=0.0.0.0:12345 /etc/alloy/config.alloy
|
||||
PublishPort=127.0.0.1:12345:12345
|
||||
Volume=/opt/alloy/config:/etc/alloy
|
||||
Network=host
|
||||
Exec=run --server.http.listen-addr=127.0.0.1:12345 /etc/alloy/config.alloy
|
||||
Volume=/opt/alloy/config:/etc/alloy:ro
|
||||
Volume=/etc/machine-id:/etc/machine-id:ro
|
||||
Volume=/run/systemd/journal/socket:/run/systemd/journal/socket:ro
|
||||
Volume=/var/log:/var/log:ro
|
||||
Volume=/var/lib/containers:/var/lib/containers:ro
|
||||
HealthCmd=/usr/local/bin/busybox wget --spider -q http://localhost:12345/-/healthy
|
||||
Volume=/var/log/journal:/var/log/journal:ro
|
||||
Volume=/run/log/journal:/run/log/journal:ro
|
||||
HealthCmd=/usr/local/bin/busybox wget --spider -q http://127.0.0.1:12345/-/healthy
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=3
|
||||
@@ -60,4 +56,4 @@ Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
WantedBy=multi-user.target
|
||||
|
||||
@@ -47,9 +47,24 @@ Environment=GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${GRAFANA_OAUTH_CLIENT_SECRET}
|
||||
# client_id query parameter — /application/o/<slug>/authorize/ does not exist.
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://auth.franksdank.cloud/application/o/authorize/
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://auth.franksdank.cloud/application/o/token/
|
||||
# ?api=1 = introspection-style userinfo: returns custom claims (groups) too.
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_API_URL=https://auth.franksdank.cloud/application/o/userinfo/?api=1
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=false
|
||||
# Standard OIDC userinfo. Grafana v13 fetches <api_url>/emails for the address
|
||||
# ONLY when the token carries no `email` claim (authentik has no /emails path and
|
||||
# 404s it, failing the login) — so the claim must actually be there: the fix is
|
||||
# the grafana-email-scope mapping (see scripts/authentik-grafana-sso-fix.sh),
|
||||
# not a Grafana flag. Do NOT append ?api=1 to this URL: Grafana then GETs
|
||||
# <...>/userinfo/?api=1/emails, a 403.
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_API_URL=https://auth.franksdank.cloud/application/o/userinfo/
|
||||
# authentik's scopes are openid/email/profile. `user:email` (a GitLab-ism) is
|
||||
# not a known scope here; requesting it makes the authorize view crash.
|
||||
# Quoted: quadlet word-splits Environment=, so an unquoted spaced list
|
||||
# truncates SCOPES to its first token.
|
||||
Environment="GF_AUTH_GENERIC_OAUTH_SCOPES=openid email profile"
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH=email
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH=sub
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH=preferred_username
|
||||
# First SSO login auto-provisions the user. Safe: unauthenticated traffic never
|
||||
# reaches this callback — the Anubis -> Authentik guard fronts every request.
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_AUTO_UPSERT=true
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_ORG=true
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_ORG_ID=1
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# /etc/udev/rules.d/98-halogen-npu-access.rules
|
||||
#
|
||||
# Why the NPU node is world-writable: `frank-dragoon` runs rootless podman under a
|
||||
# unit with `User=dank`, and the container's user namespace maps exactly one uid and
|
||||
# one gid (dank). `/dev/accel/accel0` is `root:render 0660`, so inside the container
|
||||
# the node reads `nobody:nogroup` and the engine dies with its own diagnosis:
|
||||
#
|
||||
# halogen npu: /dev/accel/accel0 is here, but this process cannot open it
|
||||
# (crw-rw---- nobody:nogroup).
|
||||
#
|
||||
# `--group-add keep-groups` is the vendor's advice and it does NOT work in this
|
||||
# shape: measured 2026-10-19 from the host as a `dank` shell that was already in
|
||||
# `render`, with and without the flag, the node still read `nobody:nogroup` and the
|
||||
# open was refused — an unmapped gid cannot be granted to a process inside a
|
||||
# namespace that does not map it. Same conclusion `99-comfyui-gpu-access.rules`
|
||||
# reached for `/dev/kfd` from the systemd-user-spawn direction: the mode is the only
|
||||
# spawn-context-independent door short of running the stack rootful.
|
||||
#
|
||||
# Blast radius: any process in this host's single uid domain (every rootless
|
||||
# container here shares it) can submit to the XDNA2 NPU. The NPU holds no user data
|
||||
# and no privileged context — the same trade already accepted for the GPU compute
|
||||
# nodes beside it. Installed by `deploy-remote.sh` when the NPU lane is armed and left
|
||||
# in place afterwards — the mode costs nothing while the lane is idle. `serve-frank-dragoon.sh`
|
||||
# refuses to arm the lane on a node that is not other-accessible, so a host that lost this
|
||||
# file fails loudly at boot instead of crash-looping.
|
||||
KERNEL=="accel[0-9]*", SUBSYSTEM=="accel", MODE="0666"
|
||||
Executable
+81
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env bash
|
||||
# Repair the Grafana OAuth2 relying-party SSO on Dankclaw's authentik.
|
||||
#
|
||||
# grafana.franksdank.cloud logs in through authentik as an OAuth2 relying party.
|
||||
# Two DB-side facts (the provider was created by hand via `ak shell`, never
|
||||
# repo-managed) silently broke that login, and BOTH had previously been
|
||||
# mis-recorded here as a single "the property-mapping evaluator is broken, so
|
||||
# userinfo returns only `sub`" myth. They are neither, and the evaluator is fine:
|
||||
#
|
||||
# 1. provider.authorization_flow was NULL. The authorize view builds a flow
|
||||
# planner from it and crashes with
|
||||
# `AttributeError: 'NoneType' object has no attribute 'slug'` — an HTTP 500
|
||||
# "Server Error — Powered by authentik", before any claim is touched.
|
||||
# Fix: point it at default-provider-authorization-implicit-consent, the
|
||||
# flow every other provider in this instance already uses.
|
||||
# 2. The `grafana-email-scope` ScopeMapping's expression was the bare string
|
||||
# `user.email`. authentik merges a scope mapping's result into the token as
|
||||
# a MAPPING: a returned dict adds its keys as claims; a returned string is
|
||||
# keyed under the mapping's NAME ("grafana-email-scope"), so no `email`
|
||||
# claim is ever emitted. Grafana, finding no email, falls back to
|
||||
# GET <api_url>/emails, authentik 404s that path, and the login dies with
|
||||
# "Login failed: InternalError". Fix: return
|
||||
# {"email": user.email, "email_verified": True} — the shape the stock
|
||||
# OpenID 'email' mapping uses. The dict-valued `grafana-profile-scope`
|
||||
# mapping has always worked, which is the proof the evaluator is fine.
|
||||
#
|
||||
# This is authentik DB state. It is NOT in any deployable file: a blueprint
|
||||
# re-import, or an ak-shell re-creation of the grafana provider, reverts BOTH
|
||||
# facts and the symptom returns verbatim. Re-run this after any such rebuild.
|
||||
# Every step is idempotent.
|
||||
#
|
||||
# Grafana's own half — SCOPES="openid email profile", a standard userinfo API_URL
|
||||
# with no `?api=1`, and ALLOW_SIGN_UP=true — lives in
|
||||
# dankclaw/observability/quadlet/grafana.container and deploys normally.
|
||||
#
|
||||
# Usage: authentik-grafana-sso-fix.sh (override host: AUTHENTIK_HOST=…)
|
||||
set -euo pipefail
|
||||
HOST="${AUTHENTIK_HOST:-dankclaw}"
|
||||
|
||||
ssh "$HOST" 'podman exec -i authentik-server ak shell' <<'PYEOF'
|
||||
from authentik.flows.models import Flow
|
||||
from authentik.providers.oauth2.models import OAuth2Provider
|
||||
|
||||
p = OAuth2Provider.objects.filter(name="grafana").first()
|
||||
if not p:
|
||||
raise SystemExit("no OAuth2Provider named 'grafana'")
|
||||
|
||||
# 1. The authorization flow every other provider uses.
|
||||
want = Flow.objects.filter(slug="default-provider-authorization-implicit-consent").first()
|
||||
if not want:
|
||||
raise SystemExit("default-provider-authorization-implicit-consent flow is missing")
|
||||
if p.authorization_flow_id != want.pk:
|
||||
p.authorization_flow = want
|
||||
p.save()
|
||||
print(f"set grafana.authorization_flow -> {want.slug}")
|
||||
else:
|
||||
print(f"grafana.authorization_flow already {want.slug}")
|
||||
|
||||
# 2. The email scope mapping must RETURN A DICT, not a bare string.
|
||||
m = p.property_mappings.filter(name="grafana-email-scope").first()
|
||||
if not m:
|
||||
raise SystemExit("grafana-email-scope mapping is missing")
|
||||
fixed = 'return {"email": user.email, "email_verified": True}'
|
||||
if m.expression.strip() != fixed:
|
||||
m.expression = fixed
|
||||
m.save()
|
||||
print("repaired grafana-email-scope -> returns {email, email_verified}")
|
||||
else:
|
||||
print("grafana-email-scope expression already correct")
|
||||
|
||||
# 3. The id_token must carry the scope claims; Grafana reads identity from it.
|
||||
if not p.include_claims_in_id_token:
|
||||
p.include_claims_in_id_token = True
|
||||
p.save()
|
||||
print("enabled include_claims_in_id_token")
|
||||
else:
|
||||
print("include_claims_in_id_token already on")
|
||||
PYEOF
|
||||
|
||||
echo "Done. Verify: sign in at https://grafana.franksdank.cloud via Authentik;"
|
||||
echo "the round trip must land on / and /api/user must answer 200 with an email."
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
# Set the authentik login-session TTL on Dankclaw.
|
||||
#
|
||||
# The login-session lifetime is governed by the `default-authentication-login`
|
||||
# stage's `session_duration`, NOT a global setting and NOT SESSION_COOKIE_AGE
|
||||
# (that is the pre-login session, sessions.unauthenticated_age). Authentik does
|
||||
# not expose SESSION_SAVE_EVERY_REQUEST at all, so the session is an absolute
|
||||
# window from login, not a sliding one.
|
||||
#
|
||||
# The field is a STRING in authentik's own idiom ("days=7", "seconds=30",
|
||||
# "hours=12") — not a Python timedelta. Writing str(timedelta(days=7)) stores
|
||||
# "7 days, 0:00:00", which authentik's timedelta_from_string parser REJECTS
|
||||
# (ValueError), and login then crashes computing the session expiry. Always pass
|
||||
# the "key=value" form; this script validates it before writing.
|
||||
#
|
||||
# Existing sessions keep their old expiry; the new TTL applies from the next
|
||||
# login. This is DB state — it is NOT repo-managed and a blueprint re-import or
|
||||
# rebuild-from-scratch reverts it. Run this again after such a rebuild.
|
||||
#
|
||||
# Usage: authentik-set-login-session-duration.sh ["days=7"] (default days=7)
|
||||
set -euo pipefail
|
||||
|
||||
DUR="${1:-days=7}"
|
||||
STAGE="default-authentication-login"
|
||||
HOST="${AUTHENTIK_HOST:-dankclaw}"
|
||||
|
||||
# Refuse anything that is not the safe key=value duration idiom.
|
||||
if ! printf '%s' "$DUR" | grep -Eq '^[A-Za-z]+=[0-9]+([ ]+[A-Za-z]+=[0-9]+)*$'; then
|
||||
echo "invalid duration '$DUR' — use authentik form, e.g. 'days=7' or 'seconds=30'" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
ssh "$HOST" 'podman exec -i authentik-server ak shell' <<PYEOF
|
||||
from django.apps import apps
|
||||
from authentik.lib.utils.time import timedelta_from_string
|
||||
dur = "$DUR"
|
||||
timedelta_from_string(dur) # validate the form before touching the row
|
||||
ULS = apps.get_model("authentik_stages_user_login", "UserLoginStage")
|
||||
s = ULS.objects.get(name="$STAGE")
|
||||
s.session_duration = dur
|
||||
s.save(update_fields=["session_duration"])
|
||||
back = ULS.objects.get(name="$STAGE").session_duration
|
||||
assert back == dur, "persisted value mismatch: %r" % back
|
||||
print("session_duration set to", repr(back), "->", timedelta_from_string(back))
|
||||
PYEOF
|
||||
|
||||
echo "Login-session TTL for '$STAGE' is now $DUR (applies on next login)."
|
||||
@@ -293,6 +293,17 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
if len(parts) >= 2 and parts[1] in {"caddy", "comfyui", "loki", "alloy", "hermes", "anubis", "seerr"}:
|
||||
_add(services, parts[1])
|
||||
return services
|
||||
if len(parts) >= 3 and parts[1] == "host":
|
||||
# Host state a service needs at boot lives under dankclaw/host/.
|
||||
# The fabric-clock hold is frank-dragoon's NPU precondition (the
|
||||
# engine refuses to start with HALOGEN_NPU_MODELS set while the
|
||||
# clock moves) and the deployer installs and gates it, so changing
|
||||
# it re-dispatches that endpoint. `grub` is the tracked boot line,
|
||||
# which moves only through scripts/set-dankclaw-bootline.sh and is
|
||||
# a reboot decision — it plans no service.
|
||||
if parts[2].startswith("halogen-fabric-clock"):
|
||||
_add(services, "frank-dragoon")
|
||||
return services
|
||||
if len(parts) >= 4 and parts[1] == "authentik" and parts[2] == "quadlet":
|
||||
name = parts[3].removesuffix(".container")
|
||||
if name in {"authentik-postgres", "authentik-server", "authentik-worker", "authentik-outpost", "authentik-outpost-ldap"}:
|
||||
|
||||
+185
-13
@@ -487,13 +487,15 @@ seed_grafana_env() {
|
||||
|| { echo "grafana env missing GRAFANA_OAUTH_CLIENT_SECRET; set it in dankclaw/.env" >&2; return 1; }
|
||||
}
|
||||
|
||||
# The five observability containers left compose on 2026-09-11 (they were
|
||||
# started by hand with podman-compose and never came back after a reboot).
|
||||
# They are ordinary rootless Quadlet units now: same images, same mounts, same
|
||||
# arguments, same container names, so nothing that addresses them by name —
|
||||
# Caddy, Prometheus' own scrape config, Alloy's loki.write endpoint — changed.
|
||||
# Only the published ports did: the three HTTP UIs bind 127.0.0.1, because the
|
||||
# compose services published on 0.0.0.0 and none of them authenticates.
|
||||
# The observability containers left compose on 2026-09-11 (they were started by
|
||||
# hand with podman-compose and never came back after a reboot). Five are ordinary
|
||||
# rootless Quadlet units: same images, same mounts, same arguments, same
|
||||
# container names, so nothing that addresses them by name — Caddy, Prometheus'
|
||||
# own scrape config — changed. Only the published ports did: the three HTTP UIs
|
||||
# bind 127.0.0.1, because the compose services published on 0.0.0.0 and none of
|
||||
# them authenticates. Alloy is the exception: a ROOT system service on
|
||||
# Network=host, because reading the host journal needs privileges a rootless
|
||||
# container cannot get. It is handled by deploy_alloy, not the loop below.
|
||||
deploy_observability() {
|
||||
local service="$1"
|
||||
# Observability units live under <target>/observability/quadlet: Dankclaw
|
||||
@@ -512,10 +514,6 @@ deploy_observability() {
|
||||
copy_loki_config
|
||||
build_compose_image loki
|
||||
;;
|
||||
alloy)
|
||||
copy_alloy_config
|
||||
build_compose_image alloy
|
||||
;;
|
||||
grafana)
|
||||
copy_grafana_provisioning
|
||||
seed_grafana_env || return 1
|
||||
@@ -534,11 +532,71 @@ deploy_observability() {
|
||||
alertmanager) wait_for_health alertmanager http://127.0.0.1:9093/-/healthy 60 ;;
|
||||
grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;;
|
||||
loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
|
||||
alloy) wait_for_health alloy http://127.0.0.1:12345/-/healthy 60 ;;
|
||||
node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Mirror an image the deployer built rootless into the ROOT image store. A root
|
||||
# quadlet reads root storage; a root build (build_compose_image is rootless) is
|
||||
# not possible on this host — the build container's resolver fails DNS — so the
|
||||
# only way to get a root-readable copy is save-and-load. Compares image ids so a
|
||||
# rebuilt alloy actually re-lands in root storage instead of leaving a stale tag.
|
||||
mirror_image_to_root() {
|
||||
local ref="$1" rid
|
||||
rid="$(podman image inspect "$ref" --format '{{.Id}}' 2>/dev/null)" || {
|
||||
echo "ERROR: rootless image $ref is not built; cannot mirror to root" >&2
|
||||
return 1
|
||||
}
|
||||
if [[ "$(run_privileged podman image inspect "$ref" --format '{{.Id}}' 2>/dev/null)" == "$rid" ]]; then
|
||||
return 0
|
||||
fi
|
||||
local tar; tar="$(mktemp /var/tmp/alloy-mirror.XXXXXX.tar)"
|
||||
podman save -o "$tar" "$ref"
|
||||
run_privileged podman load -i "$tar"
|
||||
rm -f "$tar"
|
||||
}
|
||||
|
||||
# Install a quadlet into the SYSTEM generator dir (root manager) instead of the
|
||||
# user dir. Boot persistence is the unit's own [Install] section (multi-user
|
||||
# target); the v1 system generator creates the .wants on daemon-reload, so —
|
||||
# like the user units — no `systemctl enable` (generated units cannot be enabled
|
||||
# here).
|
||||
install_system_quadlet() {
|
||||
local source_dir="$1" unit="$2"
|
||||
run_privileged mkdir -p /etc/containers/systemd
|
||||
run_privileged cp "$source_dir/$unit.container" /etc/containers/systemd/
|
||||
run_privileged systemctl daemon-reload
|
||||
}
|
||||
|
||||
# Alloy as a ROOT system service on Network=host. See the unit file: a rootless
|
||||
# container cannot read the 0640 journal files, and it cannot reach Loki's
|
||||
# 127.0.0.1:3100 publish from a bridge netns. Root + host network fixes both;
|
||||
# the price is that this one container runs as root and on the host network.
|
||||
deploy_alloy() {
|
||||
copy_alloy_config
|
||||
build_compose_image alloy # rootless build (resolver works)
|
||||
mirror_image_to_root localhost/franksdank/alloy:latest
|
||||
|
||||
# Retire the rootless unit BEFORE the system one starts: both managers
|
||||
# generate alloy.service, and two containers named "alloy" both binding
|
||||
# 127.0.0.1:12345 cannot coexist. Move the user quadlet out of the generator
|
||||
# dir (it regenerates from there) and stop the user unit.
|
||||
if [[ -f "$quadlet_unit_dir/alloy.container" ]] ||
|
||||
systemctl --user is-active --quiet alloy.service 2>/dev/null; then
|
||||
systemctl --user stop alloy.service 2>/dev/null || true
|
||||
if [[ -f "$quadlet_unit_dir/alloy.container" ]]; then
|
||||
local trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-alloy-root-cutover"
|
||||
mkdir -p "$trash"
|
||||
mv -- "$quadlet_unit_dir/alloy.container" "$trash/alloy.container"
|
||||
echo "retired rootless alloy.container -> $trash"
|
||||
fi
|
||||
systemctl --user daemon-reload
|
||||
fi
|
||||
|
||||
install_system_quadlet dankclaw/observability/quadlet alloy
|
||||
run_privileged systemctl restart alloy.service
|
||||
wait_for_health alloy http://127.0.0.1:12345/-/healthy 60
|
||||
}
|
||||
build_compose_image() {
|
||||
local service="$1" image cfhash oldhash containerfile
|
||||
# Every file the image COPIES IN, beside the Containerfile. The skip below
|
||||
@@ -1329,6 +1387,9 @@ deploy_anubis() {
|
||||
# port gate is the same as every cutover since 2026-09-16: nothing
|
||||
# starts until 7770 is released.
|
||||
LLM_DRAGOON_IMAGE="${LLM_DRAGOON_IMAGE:-ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84ac22308a53f5d1ca6b86806702d7039e5ebc51cae4c66621b92fe04a}"
|
||||
# The payload dir is its own twin because the NPU lane's containment gate below
|
||||
# reads it; every artifact path under it stays a full literal as usual.
|
||||
LLM_DRAGOON_MODEL_DIR="${LLM_DRAGOON_MODEL_DIR:-/home/dank/models/halogen-v2-abliterated}"
|
||||
LLM_DRAGOON_CKPT="${LLM_DRAGOON_CKPT:-/home/dank/models/halogen-v2-abliterated/qwen38-flash-next-v2-abliterated.hgn}"
|
||||
LLM_DRAGOON_CKPT_BYTES="${LLM_DRAGOON_CKPT_BYTES:-66687678432}"
|
||||
LLM_DRAGOON_NGRAM="${LLM_DRAGOON_NGRAM:-/home/dank/models/halogen-v2-abliterated/qwen38-flash-next-ngram.hgn}"
|
||||
@@ -1337,6 +1398,18 @@ LLM_DRAGOON_VISION="${LLM_DRAGOON_VISION:-/home/dank/models/halogen-v2-abliterat
|
||||
LLM_DRAGOON_VISION_BYTES="${LLM_DRAGOON_VISION_BYTES:-897916416}"
|
||||
LLM_DRAGOON_TOK_DIR="${LLM_DRAGOON_TOK_DIR:-/home/dank/models/halogen-v2-abliterated/tokenizer}"
|
||||
LLM_DRAGOON_TOK_FILES="${LLM_DRAGOON_TOK_FILES:-tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json}"
|
||||
# NPU lane (2026-10-19). HALOGEN_NPU_MODELS is the whole switch and the parity
|
||||
# twin of the env file's line: empty here AND in the env = the pre-lane
|
||||
# endpoint. The gates below mirror serve-frank-dragoon.sh, and the vendor's
|
||||
# fabric-clock host unit (dankclaw/host/, verbatim at tag v0.16.2) is installed
|
||||
# ONLY while the lane is named — on Strix Halo the iGPU and the NPU co-running
|
||||
# hang the machine while the fabric clock moves, so halogen refuses to boot with
|
||||
# the lane armed unless that clock is held.
|
||||
LLM_DRAGOON_NPU_MODELS="${LLM_DRAGOON_NPU_MODELS:-qwen3-embedding-0.6b}"
|
||||
LLM_DRAGOON_NPU_DIR="${LLM_DRAGOON_NPU_DIR:-/home/dank/models/halogen-v2-abliterated/npu}"
|
||||
LLM_DRAGOON_DEV_ACCEL="${LLM_DRAGOON_DEV_ACCEL:-/dev/accel/accel0}"
|
||||
LLM_DRAGOON_XRT_DIR="${LLM_DRAGOON_XRT_DIR:-/usr/lib/x86_64-linux-gnu}"
|
||||
LLM_DRAGOON_NETWORK="${LLM_DRAGOON_NETWORK:-dankclaw_default}"
|
||||
|
||||
retire_halogen_dragoon_quadlet() {
|
||||
local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
|
||||
@@ -1415,6 +1488,77 @@ deploy_llm_dragoon() {
|
||||
fi
|
||||
done
|
||||
|
||||
# The container network has to exist before the run line names it (a typo
|
||||
# there is a restart-looping unit, not a config error anyone reads).
|
||||
if [[ -n "$LLM_DRAGOON_NETWORK" ]] && ! podman network exists "$LLM_DRAGOON_NETWORK"; then
|
||||
echo "ERROR: podman network $LLM_DRAGOON_NETWORK does not exist" >&2
|
||||
echo "frank-dragoon: create it or blank FRANK_DRAGOON_NETWORK (pasta default)" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
# --- NPU lane: the same predicates the serve script checks --------------
|
||||
if [[ -n "$LLM_DRAGOON_NPU_MODELS" ]]; then
|
||||
# One door into the container: the read-only /models mount of the
|
||||
# payload dir. A staging dir outside it needs a nested bind whose
|
||||
# mountpoint runc cannot create through a read-only parent, so the
|
||||
# parity twin of the serve gate refuses the shape before the restart.
|
||||
if [[ "$LLM_DRAGOON_NPU_DIR/" != "$LLM_DRAGOON_MODEL_DIR"/* ]]; then
|
||||
echo "ERROR: NPU staging dir $LLM_DRAGOON_NPU_DIR is outside the payload dir $LLM_DRAGOON_MODEL_DIR — the lane's files reach the container only through the read-only /models mount" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -e "$LLM_DRAGOON_DEV_ACCEL" ]]; then
|
||||
echo "ERROR: NPU node $LLM_DRAGOON_DEV_ACCEL absent — amdxdna is not bound (is iommu=pt still on the boot line?)" >&2
|
||||
return 1
|
||||
fi
|
||||
# Access, not existence: a rootless container cannot open a root:render
|
||||
# 0660 node (its namespace maps one uid/gid, and keep-groups cannot grant
|
||||
# an unmapped gid — measured 2026-10-19). Install the mode rule, re-trigger
|
||||
# the node, then PROVE the node is other-accessible before the restart.
|
||||
run_privileged install -m 0644 dankclaw/udev/98-halogen-npu-access.rules \
|
||||
/etc/udev/rules.d/98-halogen-npu-access.rules
|
||||
run_privileged udevadm control --reload-rules
|
||||
run_privileged udevadm trigger --sysname-match="$(basename "$LLM_DRAGOON_DEV_ACCEL")"
|
||||
# (`-o FILE` is not "other-writable" — it means "exists", and `-o` is also
|
||||
# the OR operator inside test), so the bit is read straight off the mode.
|
||||
if [[ $(( 8#$(stat -c '%a' "$LLM_DRAGOON_DEV_ACCEL") & 8#2 )) -eq 0 ]]; then
|
||||
echo "ERROR: $LLM_DRAGOON_DEV_ACCEL is still $(stat -c '%a %U:%G' "$LLM_DRAGOON_DEV_ACCEL") after installing the access rule — the lane cannot open it rootless" >&2
|
||||
return 1
|
||||
fi
|
||||
for xl in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
|
||||
if [[ ! -e "$LLM_DRAGOON_XRT_DIR/$xl" ]]; then
|
||||
echo "ERROR: XRT NPU library $LLM_DRAGOON_XRT_DIR/$xl absent (Ubuntu libxrt2 + libxrt-npu2)" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
for nm in ${LLM_DRAGOON_NPU_MODELS//,/ }; do
|
||||
if [[ ! -d "$LLM_DRAGOON_NPU_DIR/$nm" ]]; then
|
||||
echo "ERROR: NPU model $nm not staged in $LLM_DRAGOON_NPU_DIR" >&2
|
||||
echo "frank-dragoon: stage it: bash scripts/download-frank-dragoon-npu.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
# The hold must exist BEFORE the engine starts and survive every boot
|
||||
# (a resume hands the clock back too — the unit's WantedBy covers it).
|
||||
run_privileged install -m 0755 dankclaw/host/halogen-fabric-clock \
|
||||
/usr/local/sbin/halogen-fabric-clock
|
||||
run_privileged install -m 0644 dankclaw/host/halogen-fabric-clock.service \
|
||||
/etc/systemd/system/halogen-fabric-clock.service
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable --now halogen-fabric-clock.service
|
||||
if ! run_privileged /usr/local/sbin/halogen-fabric-clock status; then
|
||||
echo "ERROR: fabric clock is not held — halogen refuses to start with HALOGEN_NPU_MODELS set" >&2
|
||||
echo "frank-dragoon: journalctl -u halogen-fabric-clock.service -n 40" >&2
|
||||
return 1
|
||||
fi
|
||||
elif [[ -x /usr/local/sbin/halogen-fabric-clock ]]; then
|
||||
# Lane rolled back: hand the clock back to the driver and drop the boot
|
||||
# hook. `disable` deletes the INSTALLED copy of this unit — unlike the
|
||||
# tracked LLM unit symlinks the fleet's disable law protects, this file
|
||||
# is repo-owned and the next lane-on deploy reinstalls it.
|
||||
run_privileged systemctl disable halogen-fabric-clock.service 2>/dev/null || true
|
||||
run_privileged /usr/local/sbin/halogen-fabric-clock release || true
|
||||
fi
|
||||
|
||||
retire_halogen_dragoon_quadlet
|
||||
# The predecessor on a re-deploy is THIS unit: stop it (ExecStop stops
|
||||
# the container) before waiting on the port. The stray sweep is keyed
|
||||
@@ -1434,6 +1578,33 @@ deploy_llm_dragoon() {
|
||||
run_privileged systemctl enable frank-dragoon.service
|
||||
run_privileged systemctl restart frank-dragoon.service
|
||||
wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200
|
||||
# The lane's own gate. /health proves the engine is up, nothing about the
|
||||
# NPU programs; one real embedding is the observable output. The first
|
||||
# request after a boot is a warmup artifact (fleet law), so retry before
|
||||
# calling a booted lane dead.
|
||||
if [[ -n "$LLM_DRAGOON_NPU_MODELS" ]]; then
|
||||
for nm in ${LLM_DRAGOON_NPU_MODELS//,/ }; do
|
||||
lane_ok=0
|
||||
for _try in 1 2 3 4 5; do
|
||||
if curl -fsS -m 60 -H 'content-type: application/json' \
|
||||
-d "{\"model\":\"$nm\",\"input\":[\"frank-dragoon npu lane gate\"]}" \
|
||||
http://127.0.0.1:7770/v1/embeddings 2>/dev/null \
|
||||
| python3 -c 'import json,sys
|
||||
v = json.load(sys.stdin)["data"][0]["embedding"]
|
||||
raise SystemExit(0 if isinstance(v, list) and len(v) and all(isinstance(x, (int, float)) for x in v) else 1)'; then
|
||||
lane_ok=1
|
||||
break
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
if [[ "$lane_ok" != 1 ]]; then
|
||||
echo "ERROR: NPU model $nm answered no vector on /v1/embeddings" >&2
|
||||
echo "frank-dragoon: read the boot ledger: tail -100 /home/dank/logs/frank_dragoon.log" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "frank-dragoon: NPU lane verified — $nm embedded"
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
# deploy_llm_zealot — frank-zealot is llama.cpp (PrismML fork build) on the
|
||||
@@ -1828,7 +1999,8 @@ deploy_service() {
|
||||
copyparty) deploy_copyparty ;;
|
||||
plex) deploy_plex ;;
|
||||
tautulli) deploy_tautulli ;;
|
||||
prometheus|grafana|loki|alloy|node-exporter|alertmanager) deploy_observability "$service" ;;
|
||||
alloy) deploy_alloy ;;
|
||||
prometheus|grafana|loki|node-exporter|alertmanager) deploy_observability "$service" ;;
|
||||
comfyui) deploy_comfyui ;;
|
||||
*) deploy_compose_service "$service" ;;
|
||||
esac
|
||||
|
||||
Executable
+117
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env bash
|
||||
# download-frank-dragoon-npu.sh — stage the NPU half of frank-dragoon: the
|
||||
# vendor-precompiled AIE programs halogen-flash-server runs on the XDNA2 NPU
|
||||
# when HALOGEN_NPU_MODELS names them. These are NOT model weights you can
|
||||
# quantize yourself: XDNA2 executes only programs compiled by AMD's AIE
|
||||
# compiler, so the only source is peonist-ai's per-model NPU repo, and the
|
||||
# engine checks every file's size and sha256 against its own catalog at boot
|
||||
# (/opt/halogen/npu/models.txt inside the image) and refuses to load a model
|
||||
# whose files do not match.
|
||||
#
|
||||
# The manifest below IS that catalog block (dumped from the pinned 0.16.2
|
||||
# image on 2026-10-19), so the files land in the layout the entrypoint
|
||||
# expects — <dir>/<model-id>/devices/*, <dir>/<model-id>/<model-id>.hnpw and
|
||||
# <dir>/<model-id>/tokenizer/tokenizer.json — and the endpoint can keep
|
||||
# /models read-only and open no outbound connection at boot (HALOGEN_DOWNLOAD
|
||||
# stays unset; the vendor's own README: with it unset the container makes no
|
||||
# outbound connections at all).
|
||||
#
|
||||
# The fetch law, same as every sibling script: curl -f (an access error must
|
||||
# not become a "successful" 153-byte artifact), the exact byte count and the
|
||||
# sha256 from the manifest, --verify-only to re-check a staged set.
|
||||
#
|
||||
# Adding a second NPU model: read its block out of the image's catalog and add
|
||||
# a case below —
|
||||
# podman exec frank-dragoon-halogen grep "^file <model-id> " /opt/halogen/npu/models.txt
|
||||
# podman exec frank-dragoon-halogen grep "^model <model-id> " /opt/halogen/npu/models.txt
|
||||
# and add the model id to HALOGEN_NPU_MODELS in frank-dragoon.env.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}"
|
||||
# The env file owns the staging path (it must sit inside the payload dir, which
|
||||
# is the container's only /models mount); an exported FRANK_DRAGOON_NPU_DIR wins
|
||||
# so the bats seam can stage somewhere disposable.
|
||||
DEST="${FRANK_DRAGOON_NPU_DIR:-}"
|
||||
if [ -z "$DEST" ] && [ -f "$ENV_FILE" ]; then
|
||||
DEST="$(sed -nE 's/^FRANK_DRAGOON_NPU_DIR=(.*)$/\1/p' "$ENV_FILE" | tail -1)"
|
||||
fi
|
||||
: "${DEST:?FRANK_DRAGOON_NPU_DIR is not set and $ENV_FILE names no FRANK_DRAGOON_NPU_DIR}"
|
||||
HF="https://huggingface.co"
|
||||
|
||||
MODELS="${HALOGEN_NPU_MODELS:-}"
|
||||
if [ -z "$MODELS" ] && [ -f "$ENV_FILE" ]; then
|
||||
MODELS="$(sed -nE 's/^HALOGEN_NPU_MODELS=(.*)$/\1/p' "$ENV_FILE" | tail -1)"
|
||||
fi
|
||||
|
||||
VERIFY_ONLY=0
|
||||
case "${1:-}" in
|
||||
--verify-only) VERIFY_ONLY=1 ;;
|
||||
"") ;;
|
||||
*) echo "usage: ${0##*/} [--verify-only] (models come from HALOGEN_NPU_MODELS)" >&2; exit 2 ;;
|
||||
esac
|
||||
[ -n "$MODELS" ] || { echo "no NPU model named — set HALOGEN_NPU_MODELS or run with it in the environment" >&2; exit 1; }
|
||||
|
||||
# manifest <model-id> -> repo revision + one "path bytes sha256" per file.
|
||||
manifest() {
|
||||
case "$1" in
|
||||
qwen3-embedding-0.6b)
|
||||
NPU_REPO="peonist-ai/halogen-npu-qwen3-embedding-0.6b"
|
||||
NPU_REV="4960fd9d4423dd15373ad21c83db48dbdb3b2798"
|
||||
FILES=(
|
||||
"devices/NOTICE.md 3248 0f267b3d2523ddf14997ee58660a492a9eaf10ccf6b08bf19dfdc85b9a478b91"
|
||||
"devices/consts.hnpb 5086496 ae430e41545a0b1012938cda373b0c450364d3be631111cca3a8c43679f93336"
|
||||
"devices/devices.hnpm 24047 4e9bca9d95d084fb89ce5b99f579a00666c8ca1efae2613c53e3a4878cc9f174"
|
||||
"devices/u0.elf 4931968 b38f165886cb037b7d3951d9cb878473eaba58a432acd45416ff85fd88aa4e46"
|
||||
"devices/u1.elf 5456000 b1811feee851b8af833b37721da92099d89772110815bbbe2c3fa1c276f0ff42"
|
||||
"devices/u2.elf 6505552 406ea9a95b470628a7382e376f4a72cdf4d7aa4c59b4ddf0a4bdd139d63d6ff2"
|
||||
"devices/u3.elf 6817216 bc205d51b523d770151a0c61968f7c70bf624412827580bb6fe0aaf4650dc2ee"
|
||||
"devices/u4.elf 7444032 a5a91338a2603acdcd5c11df532033fda2d3f5479ea44ae87f98fafbdff9a62d"
|
||||
"devices/u5.elf 7432416 6c2bf41835569e0eba25ded7c05c7264ae69c6700bdeb4791057d884f2a4c126"
|
||||
"devices/u6.elf 4941232 919d0038ca088eafddf195f3466ff38db27d3635be1cffa5d4d4b5140902a68e"
|
||||
"devices/u7.elf 5488304 d904cf8afae9428dd167960549e18cce5441745f18c62f8c50975dffc077d3c2"
|
||||
"devices/u8.elf 4941232 db3aef69b17825854ee0dd2df475125a3f5c872f95c686793d6af1ee83596fa7"
|
||||
"devices/u9.elf 5488304 d182727e551084db8fb9600441d8aa373c7f09a3d267ae0a5e921ef92326db1c"
|
||||
"qwen3-embedding-0.6b.hnpw 813032248 b9f9901429df7b46ec17153aa3899f738adbea8e4356080da0c6e6586fb750d1"
|
||||
"tokenizer/tokenizer.json 11423705 def76fb086971c7867b829c23a26261e38d9d74e02139253b38aeb9df8b4b50a"
|
||||
) ;;
|
||||
*)
|
||||
echo "unknown NPU model '$1' — add its catalog block to manifest() in this script" >&2
|
||||
return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
for MODEL in ${MODELS//,/ }; do
|
||||
manifest "$MODEL" || exit 1
|
||||
echo "== $MODEL ($NPU_REPO@$NPU_REV) =="
|
||||
for entry in "${FILES[@]}"; do
|
||||
read -r f want sha <<<"$entry"
|
||||
path="$DEST/$MODEL/$f"
|
||||
mkdir -p "$(dirname "$path")"
|
||||
if [ "$VERIFY_ONLY" -eq 0 ]; then
|
||||
if [ -f "$path" ] && [ "$(stat -c %s "$path")" = "$want" ]; then
|
||||
echo "HAVE $f ($want bytes) — skipping fetch"
|
||||
else
|
||||
echo "FETCH $f -> $path"
|
||||
# HF answers /resolve/ with a 302 to the Xet CDN and curl
|
||||
# DROPS the Range header across redirects, so resolve the
|
||||
# effective URL first and resume against the CDN (the sibling
|
||||
# 62 GiB fetch hit this on 2026-10-18).
|
||||
url=$(curl -sfIL -o /dev/null -w '%{url_effective}' "$HF/$NPU_REPO/resolve/$NPU_REV/$f") \
|
||||
|| { echo "cannot resolve $NPU_REPO/$f" >&2; exit 1; }
|
||||
curl -fL --retry 5 -C - -o "$path" "$url"
|
||||
fi
|
||||
fi
|
||||
[ -f "$path" ] || { echo "MISSING $path (run without --verify-only)" >&2; exit 1; }
|
||||
if [ -L "$path" ]; then
|
||||
echo "SYMLINK $path — the serve gate refuses symlinks; replace with a real file" >&2
|
||||
exit 1
|
||||
fi
|
||||
got="$(stat -c %s "$path")"
|
||||
[ "$got" = "$want" ] || { echo "SIZE $path: $got != $want" >&2; exit 1; }
|
||||
got="$(sha256sum "$path" | cut -d' ' -f1)"
|
||||
[ "$got" = "$sha" ] || { echo "SHA256 $path: $got != $sha" >&2; exit 1; }
|
||||
echo "OK $f ($want bytes, sha256)"
|
||||
done
|
||||
done
|
||||
echo "DOWNLOAD_DONE $DEST ($MODELS)"
|
||||
+49
-31
@@ -46,31 +46,35 @@ FRANK_ARCHON_TP=2
|
||||
# stays EXACT for single-sequence decode and any step of <= 25 rows — see
|
||||
# MAX_SEQS x (1 + SPEC_TOKENS) below.
|
||||
FRANK_ARCHON_TP_WIRE=wht6
|
||||
# Decode slots. A slot costs KV, never decode; the pool is sized to this
|
||||
# contract and nothing more. At 1.8.1 the budget resolver caps the cache at
|
||||
# 556,348 fp8 tokens — 2.8 sessions at 200,000 — so a THIRD full-height
|
||||
# conversation answers 400 while /health and every shorter chat work fine
|
||||
# (3 x 50k streams run at 77.6 t/s aggregate). Frank's trade: declare the two
|
||||
# slots the engine can hold and let the freed ~1.1 GiB go to the expert plane,
|
||||
# instead of advertising a third slot that only exists at shorter prompts.
|
||||
FRANK_ARCHON_MAX_SEQS=2
|
||||
# Decode slots. A slot costs KV and pinned RAM, never decode: the cache target is
|
||||
# exactly MAX_SEQS x MAX_MODEL_LEN and the checkpoint store asks (1 + what the
|
||||
# retention policy keeps) per slot. Ordered 2026-10-20: FOUR slots over a pool of at
|
||||
# least 700,000 fp8 tokens, rounding UP if the resolver cannot land it exactly. The
|
||||
# target is therefore 4 x 200,000 = 800,000 tokens and HOST_POOL_MIB decides whether
|
||||
# the cache gets it (expert_vram_floor, core/place/planner.cpp:100-104: the card must
|
||||
# keep expert-plane-minus-host-room). The 2-slot shape was the answer to a
|
||||
# 365,204-token pool that backed 1.8 full-height sessions; at this pool the third and
|
||||
# fourth conversations are backed by the cache instead of over-committing it.
|
||||
FRANK_ARCHON_MAX_SEQS=4
|
||||
# Served window. The engine treats `kv group tokens >= max-model-len` as a HARD
|
||||
# limit, and models.yml contextWindow MUST quote this number (over-length is a
|
||||
# refusal, not a truncation). The pool is sized for seqs x maxlen, not one
|
||||
# sequence.
|
||||
FRANK_ARCHON_MAX_MODEL_LEN=200000
|
||||
FRANK_ARCHON_PLACEMENT=expert_tiered
|
||||
# Pinned host pool for expert rows that do not fit VRAM. Under the budget
|
||||
# resolver this is ALSO the lever that decides which experts stay on the card:
|
||||
# the card's leftover goes to the expert plane first and the KV cache takes the
|
||||
# rest, so growing the pool moves experts off the card and shrinking it spills
|
||||
# them onto VRAM the cache wanted. Under-sizing it is a boot failure (36,864
|
||||
# refused a boot). The ledger's coverage line is the referee, not this file.
|
||||
# Measured, not guessed: 45,568 (PR #30) moved host_pool to 45.09 GiB and left
|
||||
# vram_kv byte-identical at 4.20 GiB / 556,348 tokens on both ranks — extra
|
||||
# host RAM above what the expert plane's footprint needs buys no cache. Back at
|
||||
# 44,928, the size that boots clean.
|
||||
FRANK_ARCHON_HOST_POOL_MIB=44928
|
||||
# Pinned host pool for expert rows that do not fit VRAM, PINNED ONCE PER RANK: the
|
||||
# host must pin tp x this number, and the refusal that names it is
|
||||
# core/engine_bringup.cpp:250-265. Under the budget resolver this is ALSO the KV
|
||||
# lever -- the card keeps expert-plane-minus-this-room, so every MiB added moves one
|
||||
# MiB of experts off the card and into the cache. Derived from the boot ledger (a
|
||||
# 23.73 GiB floor at 44,928 room puts plane + 2 units at 67.60 GiB a rank), 49,152
|
||||
# drops the floor to 19.6 GiB and leaves the cache ~6.9 GiB, which is enough to back
|
||||
# the whole 4 x 200,000 target; the leftover returns to the experts. Cost is paid
|
||||
# twice over: pinned RAM (tp x 48 GiB, against the TTM target below) and expert
|
||||
# residency -- ~3.2 GiB a card of experts now lives in host RAM and is read over the
|
||||
# x8 link when the router picks it. The ledger's coverage line is the referee, not
|
||||
# this file: if vram_kv comes back under 700,000 tokens, raise this 1024 and redeploy.
|
||||
FRANK_ARCHON_HOST_POOL_MIB=49152
|
||||
# VRAM left unclaimed per card (publisher default 96 MiB); the engine's settle
|
||||
# floor is built from it.
|
||||
FRANK_ARCHON_GPU_HEADROOM_MIB=96
|
||||
@@ -93,16 +97,29 @@ FRANK_ARCHON_SPEC_TOKENS=2
|
||||
FRANK_ARCHON_MAX_BATCHED_TOKENS=2048
|
||||
# Step sharing. A prefill chunk that shares a step with decoding sequences holds
|
||||
# the device for the chunk's whole time, and every decode row in that step waits
|
||||
# that long for its next token (the census: [[LLM Performance Diagnosis]] §9).
|
||||
# `auto` leaves each decode row behind the chunk a row plus its draft window and
|
||||
# keeps a share for whatever is waiting to be admitted; a number pins the chunk's
|
||||
# cap; `off` is the pre-1.4 behaviour this endpoint ran for two eras, where the
|
||||
# chunk took the step and admission never ran. Interleave inserts N decode-only
|
||||
# steps after a chunk step taken beside decoders: it protects their cadence and
|
||||
# holds a waiting request out of those steps, so it stays 0 until the post-migration
|
||||
# decode-vs-prefill buckets say it is wanted.
|
||||
# that long for its next token: measured at this shape, decode-only steps 15.5 ms
|
||||
# against chunk steps 166 ms, which is the harness's ~146 t/s falling to ~11 t/s
|
||||
# while another session prefills (law and census: [[frank-archon]]).
|
||||
# `auto` is a PREFILL-SPEED dial, not a decoder dial: it reserves one row plus the
|
||||
# draft window per decoding sequence (3 tokens) and an eighth of the budget per
|
||||
# waiting request or waiting prefill, and the source states it will not halve for
|
||||
# decode rows alone. Live proof: prefill_chunks_capped 5,018 across 62,386 chunk
|
||||
# steps. A number is the only value that applies whenever the chunk shares a step,
|
||||
# and it must not go under the prefill stager's smallest pass -- 1025 rows here,
|
||||
# named on the boot line "prefill staging: ... a step of more than 1024 tokens" --
|
||||
# because below it every routed layer reads its pooled experts synchronously, so
|
||||
# 512 is a trap and the vendor measures the whole job slower there. Interleave
|
||||
# inserts N decode-only steps after a chunk step taken beside decoders: it protects
|
||||
# their cadence and holds a waiting request out of those steps. That is the decoder
|
||||
# dial. ARMED at 1 on 2026-10-20: at N=1 a stream takes its token from the chunk step
|
||||
# AND one from the held step, ~2 tokens per 182 ms instead of one per 166 ms; the
|
||||
# price falls on the prefilling prompt (its chunks lose every held step) and on any
|
||||
# request still waiting to enter. `--prefill-span` re-funds a prefill only when it is
|
||||
# ALONE, so it cannot buy this back while decoders run. Meter:
|
||||
# radiance:engine_steps_held_total must leave zero, and the §9 buckets must move
|
||||
# before N rises.
|
||||
FRANK_ARCHON_BUSY_PREFILL_CHUNK=auto
|
||||
FRANK_ARCHON_PREFILL_INTERLEAVE=0
|
||||
FRANK_ARCHON_PREFILL_INTERLEAVE=1
|
||||
# Prefix cache: pinned host memory only. The disk tier was removed outright
|
||||
# (--prefix-cache-dir and --prefix-cache-disk-mib are refusals), so what used to
|
||||
# be restored from NVMe is now re-prefilled. The tier is sized in ENTRIES and one
|
||||
@@ -118,9 +135,10 @@ FRANK_ARCHON_NGRAM_PLACEMENT=ram
|
||||
# RESETS pages_limit to half of RAM on every boot; below the plan the load dies
|
||||
# mid-pin with an anonymous allocator error. ttm-preflight.sh raises the knob
|
||||
# itself before refusing by name, so recovery is a boot, not a visit.
|
||||
# Floor = 2 x HOST_POOL + 48,824 (static weights + arena) + PREFIX_CACHE + MNBT
|
||||
# per the bats ledger test; at the 44,928 pool that computes to 148,920 MiB.
|
||||
FRANK_ARCHON_TTM_MIN_MIB=150000
|
||||
# Floor = 2 x HOST_POOL + 48,824 (static weights + arena) + PREFIX_CACHE + MNBT + the
|
||||
# auto checkpoint store (MAX_SEQS x 10 slots of 111.98 MiB), per the bats ledger test;
|
||||
# at the 49,152 pool with 4 slots that computes to 161,848 MiB.
|
||||
FRANK_ARCHON_TTM_MIN_MIB=162000
|
||||
FRANK_ARCHON_TTM_TARGET_MIB=204800
|
||||
# Escape hatch for experiments. Unset -> no extra argv; the serve script expands
|
||||
# this unquoted on purpose, so keep multi-flag values as one string.
|
||||
|
||||
@@ -80,3 +80,48 @@ HALOGEN_TOP_K=20
|
||||
# Weights stay pinned: the engine pins by default and only the retired w4b build
|
||||
# honored HALOGEN_FLASH_PIN_TRUNK=0, which this payload rejects.
|
||||
HALOGEN_WEIGHTS_LOCK=1
|
||||
|
||||
# --- NPU lane (XDNA2) -------------------------------------------------------
|
||||
# HALOGEN_NPU_MODELS is the whole switch: EMPTY or unset = the endpoint exactly
|
||||
# as it ran before this lane existed (no accel device, no XRT mount, no
|
||||
# fabric-clock dependency, no /v1/embeddings). Rolling the lane back is blanking
|
||||
# this one line and re-deploying — nothing else moves.
|
||||
# Names are the vendor catalog's folder names (`/opt/halogen/npu/models.txt`
|
||||
# inside the image) and the engine reads a model ONLY at /models/npu/<name>/ —
|
||||
# HALOGEN_NPU_DIR is gone from the image and setting it is a boot refusal.
|
||||
# The engine cannot fetch anything here: /models is read-only and
|
||||
# HALOGEN_DOWNLOAD stays unset so the container opens no outbound connection at
|
||||
# boot, so scripts/download-frank-dragoon-npu.sh must stage the files first
|
||||
# (the serve gate and the deployer both refuse by name when a named model is
|
||||
# missing). FRANK_DRAGOON_NPU_DIR therefore lives INSIDE the payload dir: the
|
||||
# payload mount is the container's only handle on /models, and runc cannot
|
||||
# mkdir a nested /models/npu mountpoint through a read-only bind — the lane's
|
||||
# first boot died on precisely that (2026-10-19). Swap the payload dir and this
|
||||
# path moves with it.
|
||||
# Cost: ~1.5 GiB extra pinned host RAM per named model, and a hard dependency on
|
||||
# the host holding the GPU fabric clock — on Strix Halo the iGPU and the NPU
|
||||
# together hang the machine (and corrupt NPU results) while FCLK changes speed,
|
||||
# so halogen refuses to start with this variable set unless the clock is held
|
||||
# (dankclaw/host/halogen-fabric-clock; the deployer installs and gates it).
|
||||
HALOGEN_NPU_MODELS=qwen3-embedding-0.6b
|
||||
FRANK_DRAGOON_NPU_DIR=/home/dank/models/halogen-v2-abliterated/npu
|
||||
# Node paths and the XRT directory are variables so the bats seam can drive
|
||||
# these gates on a box with no NPU. Ubuntu's distro XRT (libxrt2 + libxrt-npu2)
|
||||
# is mounted FILE BY FILE and TWICE EACH — once where the image looks for XRT,
|
||||
# once at its own path, because those libraries find each other at their install
|
||||
# path; the vendor forbids mounting an /opt/xilinx/xrt of symlinks into the
|
||||
# system dir, which dangle inside the container.
|
||||
FRANK_DRAGOON_DEV_ACCEL=/dev/accel/accel0
|
||||
FRANK_DRAGOON_XRT_DIR=/usr/lib/x86_64-linux-gnu
|
||||
# sysfs root the fabric-clock preflight reads (same knob the vendor's
|
||||
# halogen-fabric-clock status reads; the seam points it at a fixture).
|
||||
FRANK_DRAGOON_SYSFS=/sys
|
||||
# Container network. Empty = podman's rootless default (`pasta`): a private
|
||||
# netns that nothing else can join — podman refuses a second network while the
|
||||
# primary is pasta, the published port is refused from dankclaw_default
|
||||
# containers, and host.containers.internal resolves link-local, which
|
||||
# open-notebook's URL guard refuses to store. Named = the app bridge, so
|
||||
# open-notebook can dial http://frank-dragoon-halogen:8731 for the NPU
|
||||
# endpoints. The engine's unauthenticated 8730 stays private either way (it
|
||||
# binds the container's own loopback); only the API port becomes name-resolvable.
|
||||
FRANK_DRAGOON_NETWORK=dankclaw_default
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
# whichever TTS engine wins the engine decision (dankclaw/voiceservice/app.py
|
||||
# is a single stdlib-only file in a locally built image). Consumers — omp's
|
||||
# TTS device, OpenWebUI, open-notebook — address a VOICE BY NAME
|
||||
# (voice=narrator1, or the harness' voice_id=narrator1) and never see a path:
|
||||
# (voice=wilwheaton, or the harness' voice_id=wilwheaton) and never see a path:
|
||||
# a name is real exactly when /voices/<name>/reference.wav exists, and
|
||||
# anything else is a 400 that lists the names that are.
|
||||
#
|
||||
@@ -65,7 +65,7 @@ TTS_UPSTREAM_MODEL=voxcpm2
|
||||
# name still 400s with the list that exists — this covers the absent case only —
|
||||
# and the journal marks such a line `(default)` so an unrequested voice never
|
||||
# reads as a requested one. Any pack directory in /home/dank/voices is legal.
|
||||
DEFAULT_VOICE=narrator1
|
||||
DEFAULT_VOICE=wilwheaton
|
||||
|
||||
# CLI-backend knobs, inert while TTS_BACKEND=omni-http. Empty means "not
|
||||
# configured": the endpoint answers a 500 that says so instead of running
|
||||
|
||||
@@ -147,10 +147,21 @@ def main():
|
||||
deep = corpus(160_000)
|
||||
wide = corpus(1_100_000)
|
||||
|
||||
# 3.28 chars/token is the measured ratio of this corpus through the
|
||||
# froggeric template (646,000 chars -> 196,919 tokens at the 2026-10-11
|
||||
# deploy); 98.5 % of the window leaves the response room under the ceiling.
|
||||
chars = int(0.985 * maxlen * 3.28)
|
||||
# The corpus's chars/token ratio is MEASURED here, not stored. A stored 3.28
|
||||
# (646,000 chars -> 196,919 tokens, 2026-10-11) had drifted to 3.20 by
|
||||
# 2026-10-20 (646,299 chars -> 201,876 tokens) as the repo's docs grew, and a
|
||||
# stale constant sends leg B past --max-model-len, where the engine answers a
|
||||
# perfectly legitimate 400 ("the request (201900 tokens) exceeds the available
|
||||
# context size (200000)") and the leg reads as a capacity failure. One small
|
||||
# request prices the ratio; 97 % of the window leaves the response room.
|
||||
sample = prompt(wide, 4000, "CAL", "cal-marker")
|
||||
cal, _, _ = post({"model": MODEL,
|
||||
"messages": [{"role": "user", "content": sample}],
|
||||
"max_tokens": 1, "temperature": 0.0,
|
||||
"chat_template_kwargs": {"enable_thinking": False}})
|
||||
ratio = len(sample) / cal["usage"]["prompt_tokens"]
|
||||
print(f"[cal] corpus ratio {ratio:.2f} chars/token")
|
||||
chars = int(0.97 * maxlen * ratio)
|
||||
# Each marker's FIRST hyphen-separated word is unique, because show()
|
||||
# scores the hit on that word — a shared prefix would score every slot HIT
|
||||
# on any answer that says the word.
|
||||
@@ -176,7 +187,7 @@ def main():
|
||||
print(f"[B-{s}] FAILED after {time.time() - t0:.1f}s: {e!r}")
|
||||
print(f"[B] total {time.time() - t0:.1f}s")
|
||||
# D — concurrency decode: seqs streams, 512 tokens each, ~25 % depth.
|
||||
dchars = int(0.25 * maxlen * 3.28)
|
||||
dchars = int(0.25 * maxlen * ratio)
|
||||
dbodies = [chat(deep, dchars, f"D{i + 1}-9137", f"flint-marker-{i}", 512)
|
||||
for i in range(seqs)]
|
||||
print(f"[D] {seqs} concurrent streams x 512 tokens, "
|
||||
|
||||
@@ -15,18 +15,21 @@
|
||||
# log states every budget before anything is allocated (weights vs KV
|
||||
# split, the expert host pool's coverage of the worst case, the PLE pin,
|
||||
# the P2P answer per pair) — read it after every resize.
|
||||
# Deployed shape (2026-10-19, radiance 1.8.1): THREE decode slots x 200,000
|
||||
# tokens, fp8 KV, 44 GiB/rank pinned expert pool, MTP depth 2, prefix cache in
|
||||
# pinned host RAM, PLE pinned in RAM. The per-card split between the expert slab
|
||||
# and the KV pool is NO LONGER an operator number — 1.3.0 removed the VRAM pair
|
||||
# and the budget resolver derives it from headroom, the cache bound and the host
|
||||
# pool, so the slab size and the pool's token count are LEDGER lines: after every
|
||||
# boot read them out of the log and check the pool still backs 3 x 200,000.
|
||||
# Deployed shape (2026-10-20, radiance 1.8.1): FOUR decode slots x 200,000 tokens over a
|
||||
# >= 700,000-token fp8 KV pool, 48 GiB/rank pinned expert pool, MTP depth 2, prefix cache
|
||||
# in pinned host RAM, PLE pinned in RAM. The per-card split between the expert slab and the KV
|
||||
# pool is NO LONGER an operator number — 1.3.0 removed the VRAM pair and the budget
|
||||
# resolver derives it from headroom, the cache bound and the host pool, so the slab
|
||||
# size and the pool's token count are LEDGER lines. After every boot read them, plus
|
||||
# the two lines that size a prefill step — "activation arena: ... lends ... while
|
||||
# steps are at most N tokens" (the arena levels the expert slab borrows) and
|
||||
# "prefill staging: ... a step of more than N tokens ..." (the chunk size below which
|
||||
# pooled experts are read synchronously) — and the cache's session backing.
|
||||
#
|
||||
# The root unit (scripts/frank-archon.service, User=dank,
|
||||
# LimitMEMLOCK=infinity) grants the pin budget: 2 x 44 GiB expert pools +
|
||||
# the 47.7 GiB PLE table + 8 GiB prefix host tier — ~147 GiB of pinned
|
||||
# pages, which is ALSO why this script refuses to start when ttm's
|
||||
# LimitMEMLOCK=infinity) grants the pin budget: 2 x 48 GiB expert pools + the 47.7 GiB PLE
|
||||
# table + 8 GiB prefix host tier + the checkpoint store (slots x 112 MiB) — ~159 GiB of
|
||||
# pinned pages, which is ALSO why this script refuses to start when ttm's
|
||||
# pages_limit/pages_max cap is under FRANK_ARCHON_TTM_MIN_MIB (see below).
|
||||
set -euo pipefail
|
||||
|
||||
|
||||
@@ -27,6 +27,11 @@
|
||||
# the flash-server line reads it; it ships as a container; the root unit
|
||||
# keeps LimitMEMLOCK=infinity for parity and to outlive the login session,
|
||||
# exactly like frank-archon's.
|
||||
# Since 2026-10-19 the container also carries the XDNA2 NPU lane
|
||||
# (HALOGEN_NPU_MODELS): the vendor's precompiled AIE programs for the models
|
||||
# named there run on the NPU inside THIS container and answer /v1/embeddings,
|
||||
# beside the Flash model, on the same port. Opt-in — the variable empty is the
|
||||
# pre-lane endpoint, byte for byte.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -115,6 +120,119 @@ if [ ! -e "$DEV_KFD" ] || [ ! -d "$DEV_DRI" ]; then
|
||||
echo "frank-dragoon: $DEV_KFD or $DEV_DRI missing — no gfx1151 to serve on" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- NPU lane (XDNA2), opt-in via HALOGEN_NPU_MODELS ------------------------
|
||||
# Empty = the endpoint exactly as it ran before the lane existed: no device, no
|
||||
# extra mounts, no gate below. Set = three host preconditions, each refused BY
|
||||
# NAME (the 2026-10-01 law: the serve gate and the deployer gate must agree):
|
||||
# 1. the staged model files under $FRANK_DRAGOON_NPU_DIR — a path that MUST
|
||||
# live INSIDE $FRANK_DRAGOON_MODEL_DIR, because the container's only
|
||||
# handle on them is the read-only /models mount. A nested bind at
|
||||
# /models/npu cannot work: runc has to mkdir its mountpoint and the parent
|
||||
# bind is ro (`mkdirat .../merged/models/npu: read-only file system`,
|
||||
# measured 2026-10-19), and the image ships no /models at all for it to
|
||||
# land on. `$MODEL_DIR/npu/<id>` is visible at /models/npu/<id> through
|
||||
# the existing mount, which is also where the vendor's catalog expects it.
|
||||
# The engine re-checks size+sha256 against that catalog at boot; it cannot
|
||||
# fetch anything here because /models is read-only and HALOGEN_DOWNLOAD
|
||||
# stays unset — no outbound connection at boot),
|
||||
# 2. the distro XRT's three libraries, each mounted twice (where the image
|
||||
# looks for XRT, and at its own path — the libraries find each other there;
|
||||
# an /opt/xilinx/xrt of symlinks into the system dir dangles inside the
|
||||
# container, which the vendor calls out explicitly),
|
||||
# 3. the GPU fabric clock HELD: on Strix Halo the iGPU and the NPU co-running
|
||||
# hang the machine while FCLK changes speed, and the engine refuses to
|
||||
# start when it moves — it cannot read host sysfs from inside the
|
||||
# container, so the proof lives here, checked exactly the way the vendor's
|
||||
# `halogen-fabric-clock status` checks it.
|
||||
NPU_ARGS=()
|
||||
NPU_MODELS="${HALOGEN_NPU_MODELS:-}"
|
||||
if [ -n "$NPU_MODELS" ]; then
|
||||
DEV_ACCEL="${FRANK_DRAGOON_DEV_ACCEL:-/dev/accel/accel0}"
|
||||
NPU_DIR="${FRANK_DRAGOON_NPU_DIR:?HALOGEN_NPU_MODELS is set, so FRANK_DRAGOON_NPU_DIR must name the staged model dir}"
|
||||
XRT_DIR="${FRANK_DRAGOON_XRT_DIR:-/usr/lib/x86_64-linux-gnu}"
|
||||
SYSFS="${FRANK_DRAGOON_SYSFS:-/sys}"
|
||||
# The lane's files have exactly one door into the container: the read-only
|
||||
# payload mount. A staging dir OUTSIDE it would need a nested bind whose
|
||||
# mountpoint runc cannot create through that read-only parent, so refuse the
|
||||
# shape by name here rather than let runc die at container init (2026-10-19).
|
||||
case "$NPU_DIR/" in
|
||||
"$FRANK_DRAGOON_MODEL_DIR"/*) ;;
|
||||
*) echo "frank-dragoon: FRANK_DRAGOON_NPU_DIR=$NPU_DIR is outside the payload dir $FRANK_DRAGOON_MODEL_DIR — the lane's files reach the container only through the read-only /models mount; move the staging dir inside it" >&2
|
||||
exit 1 ;;
|
||||
esac
|
||||
if [ ! -e "$DEV_ACCEL" ]; then
|
||||
echo "frank-dragoon: $DEV_ACCEL missing — the NPU lane needs the amdxdna node (is iommu=pt still on the boot line?)" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Existence is not access. This endpoint runs rootless, so the container's
|
||||
# namespace maps exactly one uid/gid (dank) and a root:render 0660 node reads
|
||||
# `nobody:nogroup` inside it — the engine then dies naming the node itself
|
||||
# (measured 2026-10-19). --group-add keep-groups cannot help: an unmapped gid
|
||||
# cannot be granted into a namespace that does not map it, verified with and
|
||||
# without the flag from a shell already in `render`. The door is the udev mode.
|
||||
# NB: `[ -o FILE ]` is NOT "other-writable" — it means "exists", and `-o` is
|
||||
# also the OR operator inside test — so the permission bit is read directly.
|
||||
if [ $(( 8#$(stat -c '%a' "$DEV_ACCEL") & 8#2 )) -eq 0 ]; then
|
||||
echo "frank-dragoon: $DEV_ACCEL is $(stat -c '%a %U:%G' "$DEV_ACCEL") — a rootless container cannot open it; install dankclaw/udev/98-halogen-npu-access.rules and re-trigger udev (sudo udevadm control --reload-rules && sudo udevadm trigger --sysname-match=accel0)" >&2
|
||||
exit 1
|
||||
fi
|
||||
for M in ${NPU_MODELS//,/ }; do
|
||||
if [ ! -d "$NPU_DIR/$M" ]; then
|
||||
echo "frank-dragoon: NPU model $M not staged in $NPU_DIR;" >&2
|
||||
echo " fetch it: bash scripts/download-frank-dragoon-npu.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
for L in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
|
||||
if [ ! -e "$XRT_DIR/$L" ]; then
|
||||
echo "frank-dragoon: XRT NPU library $XRT_DIR/$L missing — the NPU lane cannot load (Ubuntu libxrt2 + libxrt-npu2)" >&2
|
||||
exit 1
|
||||
fi
|
||||
REAL="$(readlink -f "$XRT_DIR/$L")"
|
||||
NPU_ARGS+=(-v "$REAL:/opt/xilinx/xrt/lib/$L:ro" -v "$REAL:$XRT_DIR/$L:ro")
|
||||
done
|
||||
HELD=0
|
||||
for D in "$SYSFS"/class/drm/card*/device; do
|
||||
[ -e "$D/pp_dpm_fclk" ] || continue
|
||||
[ -e "$D/power_dpm_force_performance_level" ] || continue
|
||||
PERF="$(cat "$D/power_dpm_force_performance_level" 2>/dev/null || true)"
|
||||
TOP="$(grep . "$D/pp_dpm_fclk" | tail -n 1)"
|
||||
if [ "$PERF" = high ] || { [ "$PERF" = manual ] \
|
||||
&& [ "$(grep -c '\*' "$D/pp_dpm_fclk")" = 1 ] && [ "${TOP%\*}" != "$TOP" ]; }; then
|
||||
HELD=1
|
||||
else
|
||||
echo "frank-dragoon: fabric clock at $D is '$PERF', not held — the NPU lane refuses to start (GPU+NPU together need FCLK pinned)" >&2
|
||||
echo " hold it: sudo halogen-fabric-clock hold" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
if [ "$HELD" = 0 ]; then
|
||||
echo "frank-dragoon: no amdgpu device with a fabric-clock control under $SYSFS/class/drm — cannot prove FCLK is held" >&2
|
||||
exit 1
|
||||
fi
|
||||
NPU_ARGS+=(--device "$DEV_ACCEL" \
|
||||
-e HALOGEN_NPU_MODELS="$NPU_MODELS")
|
||||
echo "frank-dragoon: NPU lane armed: $NPU_MODELS on $DEV_ACCEL (fabric clock held)" >&2
|
||||
fi
|
||||
|
||||
# Container network. Empty = podman's rootless default (`pasta`), which is a
|
||||
# private netns and CANNOT be joined by anything else: podman refuses a second
|
||||
# network while the primary is pasta ("cannot set multiple networks without
|
||||
# bridge network mode"), the endpoint's published port answers 200 on the host
|
||||
# and on the LAN but is REFUSED from a dankclaw_default container (measured
|
||||
# 2026-10-19), and `host.containers.internal:7770` answers 200 yet resolves to
|
||||
# 169.254.1.2 — link-local, which open-notebook's URL guard refuses to store.
|
||||
# So an endpoint on pasta has no address an app container can use; naming the
|
||||
# app bridge here is the only shape that lets open-notebook dial
|
||||
# http://frank-dragoon-halogen:8731 for the NPU endpoints. It does not widen
|
||||
# the surface: the engine's unauthenticated 8730 is bound to the container's
|
||||
# OWN loopback (/proc/net/tcp: 127.0.0.1:8730 vs 0.0.0.0:8731, checked the same
|
||||
# day), so only the API port is reachable by name.
|
||||
NET_ARGS=()
|
||||
if [ -n "${FRANK_DRAGOON_NETWORK:-}" ]; then
|
||||
NET_ARGS=(--network "$FRANK_DRAGOON_NETWORK")
|
||||
fi
|
||||
# Host-RAM preflight — the engine's pin gate, checked BEFORE podman run.
|
||||
# The engine refuses to pin when MemAvailable is below the checkpoint plus
|
||||
# a 16 GiB floor: pinned pages cannot be swapped, and under the floor the
|
||||
@@ -189,6 +307,8 @@ podman run --name "$FRANK_DRAGOON_CONTAINER" \
|
||||
${HALOGEN_WEIGHTS_LOCK:+-e HALOGEN_WEIGHTS_LOCK="$HALOGEN_WEIGHTS_LOCK"} \
|
||||
-v "$FRANK_DRAGOON_MODEL_DIR:/models:ro" \
|
||||
-v "$FRANK_DRAGOON_MODEL_DIR/tokenizer:/tokenizer:ro" \
|
||||
"${NET_ARGS[@]}" \
|
||||
"${NPU_ARGS[@]}" \
|
||||
"$FRANK_DRAGOON_IMAGE" \
|
||||
> "$LOG" 2>&1
|
||||
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
---
|
||||
type: reference
|
||||
title: Domain Docs
|
||||
description: "Consumer rules for GLOSSARY.md and docs/adr/, and where this vault out-ranks them on operational facts."
|
||||
tags: [agent, glossary, adr, documentation]
|
||||
timestamp: 2026-10-20T00:00:00Z
|
||||
---
|
||||
|
||||
# Domain Docs
|
||||
|
||||
How the engineering skills (`/grill-with-docs`, `/domain-modeling`,
|
||||
`/improve-codebase-architecture`, `/to-spec`) should consume this repo's design
|
||||
vocabulary when exploring the codebase.
|
||||
|
||||
## Before exploring, read these
|
||||
|
||||
- **`GLOSSARY.md`** at the repo root
|
||||
- **`docs/adr/`**: read ADRs that touch the area you're about to work in
|
||||
|
||||
If either doesn't exist, **proceed silently**. Don't flag their absence; don't
|
||||
suggest creating them upfront. `/domain-modeling` (reached via
|
||||
`/grill-with-docs` and `/improve-codebase-architecture`) creates them lazily
|
||||
when terms or decisions actually get resolved. As of 2026-10-20 neither exists;
|
||||
this page is the pointer, not a promise.
|
||||
|
||||
## Ownership against this vault
|
||||
|
||||
This repo already carries `serverAppConfigs-docs/` — the operational knowledge
|
||||
base and, per repo law, the owner of operational facts (endpoint shapes, Eras,
|
||||
Laws, runbooks). `GLOSSARY.md` / `docs/adr/` hold **design vocabulary and
|
||||
decisions**. Where a term is defined in both, the vault page that owns the fact
|
||||
wins — one fact, one owner. A glossary entry restating an endpoint number is
|
||||
the bug, not the second source.
|
||||
|
||||
## Use the glossary's vocabulary
|
||||
|
||||
When output names a domain concept (an issue title, a refactor proposal, a
|
||||
hypothesis, a test name), use the term as defined in `GLOSSARY.md`. Don't drift
|
||||
to synonyms the glossary explicitly avoids.
|
||||
|
||||
If the concept you need isn't in the glossary yet, that's a signal: either
|
||||
you're inventing language the project doesn't use (reconsider) or there's a
|
||||
real gap (note it for `/domain-modeling`).
|
||||
|
||||
## Flag ADR conflicts
|
||||
|
||||
If output contradicts an existing ADR, surface it explicitly rather than
|
||||
silently overriding:
|
||||
|
||||
> _Contradicts ADR-0007 (…), but worth reopening because…_
|
||||
|
||||
## Related
|
||||
|
||||
- [[Issue Tracker]] — where specs and tickets built on this vocabulary are published.
|
||||
- [[Agent Docs and Edits]] — the edit-safety rules that apply while touching any doc in this checkout.
|
||||
@@ -0,0 +1,85 @@
|
||||
---
|
||||
type: reference
|
||||
title: Issue Tracker
|
||||
description: "How agent skills read and write Gitea issues here: credential-safe curl, label IDs, and wayfinding without sub-issues."
|
||||
tags: [agent, gitea, issues, triage, api]
|
||||
timestamp: 2026-10-20T00:00:00Z
|
||||
---
|
||||
|
||||
# Issue Tracker
|
||||
|
||||
Issues and specs for this repo live as Gitea issues in `dank/serverAppConfigs`
|
||||
on `http://192.168.0.143:3000` (Dankclaw). There is no `gh`/`glab` CLI here —
|
||||
the door is the GitHub-compatible REST API driven with `curl`. The
|
||||
mattpocock-lineage skills (`/triage`, `/to-tickets`, `/to-spec`, `/wayfinder`)
|
||||
read this page as their tracker contract. The git workflow around issues and
|
||||
PRs (branches, PR-only merges, attribution footer) is owned by
|
||||
[[Deployment Flow]]; this page owns only the API shape.
|
||||
|
||||
## Auth
|
||||
|
||||
The token lives in the git credential file `~/.config/secrets/gitea`: one
|
||||
line in git-credential URL form, `http://user:TOKEN@192.168.0.143`. Extract it
|
||||
without echoing:
|
||||
|
||||
```bash
|
||||
tok=$(sed -nE 's#^http://[^:/@]+:([^@]+)@192\.168\.0\.143.*#\1#p' ~/.config/secrets/gitea | head -1)
|
||||
```
|
||||
|
||||
**A token on a command line is world-readable in `/proc/<pid>/cmdline`** (the
|
||||
same leak class AGENTS.md records for HF tokens). Never pass it with `-H` —
|
||||
put it in a mode-600 curl config and pass `-K`:
|
||||
|
||||
```bash
|
||||
cfg=$(mktemp); chmod 600 "$cfg"
|
||||
printf 'header = "Authorization: token %s"\n' "$tok" > "$cfg"
|
||||
# ... every curl below uses -K "$cfg"; rm -f "$cfg" when done
|
||||
```
|
||||
|
||||
**GETs also accept HTTP Basic; writes do not** — a POST/PATCH with Basic auth
|
||||
is refused with `422 {"message":"token is required"}` (measured 2026-10-20).
|
||||
Use the `Authorization: token` header for everything.
|
||||
|
||||
## Conventions
|
||||
|
||||
`$API=http://192.168.0.143:3000/api/v1/repos/dank/serverAppConfigs`
|
||||
|
||||
- **Create an issue**: `curl -s -K "$cfg" -X POST -H 'Content-Type: application/json' -d "$(jq -n --arg t "$TITLE" --rawfile b "$BODY" '{title:$t, body:$b}')" $API/issues`
|
||||
- **Read an issue**: `curl -s -K "$cfg" $API/issues/<number>` (fields: `number`, `title`, `body`, `labels[].name`, `state`)
|
||||
- **List issues**: `curl -s -K "$cfg" "$API/issues?state=open&limit=50"` — add `&labels=<id>` to filter
|
||||
- **Comment**: `curl -s -K "$cfg" -X POST -d "$(jq -n --rawfile b "$BODY" '{body:$b}')" $API/issues/<number>/comments`
|
||||
- **Apply labels**: `POST $API/issues/<number>/labels` with `{"labels":[<label-ID numbers>]}` — Gitea wants numeric label **IDs**, not names; resolve them from `GET $API/labels` (see [[Triage Labels]] for which labels exist)
|
||||
- **Remove a label**: `DELETE $API/issues/<number>/labels/<label-id>`
|
||||
- **Close**: `PATCH $API/issues/<number>` with `{"state":"closed"}`
|
||||
- **Blocking relationships**: this instance has no native sub-issues or dependencies — put `Part of #<parent>` / `Blocked by: #<n>, #<n>` lines at the top of the child body. A ticket is unblocked when every listed blocker is closed.
|
||||
- **Attribution footer (repo law)**: every issue body you open ends with `cwd: <working directory> · session: <omp session id>` — full rule in [[Deployment Flow]].
|
||||
|
||||
**PRs as a request surface: no.** Issues and PRs have separate number spaces on
|
||||
Gitea; a bare `#42` is an issue, a PR is spelled "PR #42".
|
||||
|
||||
## When a skill says "publish to the issue tracker"
|
||||
|
||||
Create a Gitea issue as above.
|
||||
|
||||
## When a skill says "fetch the relevant ticket"
|
||||
|
||||
`GET $API/issues/<number>` and read `title` + `body` + `labels[].name` +
|
||||
comments.
|
||||
|
||||
## Wayfinding operations
|
||||
|
||||
Used by `/wayfinder`. The **map** is a single issue labelled `wayfinder:map`,
|
||||
holding the Notes / Decisions-so-far / Fog body.
|
||||
|
||||
- **Map**: create an issue, apply the `wayfinder:map` label (create it via `POST $API/labels` `{"name":"wayfinder:map","color":"#6f42c1"}` if missing).
|
||||
- **Child ticket**: an issue carrying `Part of #<map>` at the top of its body, plus a `wayfinder:<type>` label (`research`/`prototype`/`grilling`/`task`).
|
||||
- **Blocking**: the `Blocked by:` body line (see Conventions).
|
||||
- **Frontier**: list open issues (`?state=open`), drop any with an open `Blocked by` target or an assignee; lowest `number` wins.
|
||||
- **Claim**: `POST $API/issues/<n>/assignees` with `{"assignees":["dank"]}` — actually assign it or it stays in the frontier; the session's first write.
|
||||
- **Resolve**: post the answer as a comment, `PATCH` to `closed`, then update the map issue's body (GET → edit Decisions-so-far → PATCH) with a context pointer.
|
||||
|
||||
## Related
|
||||
|
||||
- [[Triage Labels]] — the role strings these operations apply.
|
||||
- [[Domain Docs]] — what a skill must read before touching the codebase.
|
||||
- [[SSH and Remote Access]] — reaching the host when an issue needs live evidence.
|
||||
@@ -0,0 +1,39 @@
|
||||
---
|
||||
type: reference
|
||||
title: Triage Labels
|
||||
description: "The five canonical triage roles mapped to this repo's Gitea label strings, plus the numeric-ID quirk."
|
||||
tags: [agent, gitea, triage, labels]
|
||||
timestamp: 2026-10-20T00:00:00Z
|
||||
---
|
||||
|
||||
# Triage Labels
|
||||
|
||||
The mattpocock-lineage skills speak in terms of five canonical triage roles.
|
||||
This page maps those roles to the actual label strings used in this repo's
|
||||
tracker ([[Issue Tracker]] owns the API shape).
|
||||
|
||||
| Label in mattpocock/skills | Label in our tracker | Meaning |
|
||||
| -------------------------- | -------------------- | ---------------------------------------- |
|
||||
| `needs-triage` | `needs-triage` | Maintainer needs to evaluate this issue |
|
||||
| `needs-info` | `needs-info` | Waiting on reporter for more information |
|
||||
| `ready-for-agent` | `ready-for-agent` | Fully specified, ready for an AFK agent |
|
||||
| `ready-for-human` | `ready-for-human` | Requires human implementation |
|
||||
| `wontfix` | `wontfix` | Will not be actioned |
|
||||
|
||||
When a skill mentions a role (e.g. "apply the AFK-ready triage label"), use the
|
||||
corresponding label string from this table. Edit the "Label in our tracker"
|
||||
column to match whatever vocabulary you actually use.
|
||||
|
||||
## Instance quirks (Dankclaw Gitea)
|
||||
|
||||
- Labels are applied by **numeric ID**, never by name: `POST /issues/<n>/labels`
|
||||
with `{"labels":[<id>]}`; resolve names via `GET /labels`.
|
||||
- The repo also carries the pre-existing `human-only` label from the manual era
|
||||
(e.g. the podcast-destination issues). `ready-for-human` maps to itself; the
|
||||
`(HUMAN ONLY)` title prefixes on existing issues stay as they are — the label
|
||||
is what the skills read.
|
||||
- An issue with **no** triage label is simply untriaged; absence is not a state.
|
||||
|
||||
## Related
|
||||
|
||||
- [[Issue Tracker]] — the auth-safe curl shape that applies these labels.
|
||||
@@ -6,6 +6,9 @@ docs get edited safely, and how the agent's home is deployed.
|
||||
- [[OMP Harness]] — The omp harness as configured here: compaction chain arithmetic, per-model sampler routing, and the context-lean audits.
|
||||
- [[Agent Docs and Edits]] — Two self-inflicted doc damage classes from parallel agent edits, with the audit checks and recovery order for each.
|
||||
- [[Command Center Dotfiles]] — How the dotfiles repo deploys: stow-equivalent packages, full-replica rsync, the pre-overwrite audit, transport gotchas.
|
||||
- [[Issue Tracker]] — How agent skills read and write Gitea issues here: credential-safe curl, label IDs, and wayfinding without sub-issues.
|
||||
- [[Triage Labels]] — The five canonical triage roles mapped to this repo's Gitea label strings, plus the numeric-ID quirk.
|
||||
- [[Domain Docs]] — Consumer rules for GLOSSARY.md and docs/adr/, and where this vault out-ranks them on operational facts.
|
||||
|
||||
## Related
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ tags:
|
||||
- netavark
|
||||
- rootless
|
||||
- dankclaw
|
||||
timestamp: 2026-09-02
|
||||
timestamp: 2026-10-19
|
||||
---
|
||||
|
||||
# Container Network Namespaces
|
||||
@@ -99,18 +99,42 @@ private `systemd-open-notebook`), each address aimed at a live listener:
|
||||
| Dial | Result | Mechanism |
|
||||
|---|---|---|
|
||||
| `192.168.0.143:3000` (Gitea) | **200** | published by a **rootless** container — rootless egress crosses the host netfilter where netavark installed that DNAT |
|
||||
| `192.168.0.143:7770` (`frank-dragoon`) | refused | published by a **root** container. Root and rootless Podman carry separate netavark rule sets, so a rootless container's traffic never crosses root's and a root-published port is invisible to it |
|
||||
| `192.168.0.143:7770` (`frank-dragoon`) | refused | the container is **rootless on podman's default `pasta` netns** (the root unit's `User=dank` runs `podman run`, so it lands in dank's storage). Pasta forwards the published port in the host netns — the host and the LAN both answer 200 — but installs no netavark DNAT for traffic arriving from a rootless bridge, so the same address is refused from a container. This is NOT a root/rootless rule-set split: `podman ps` as `dank` lists it |
|
||||
| `host.containers.internal:7770` (`169.254.1.2`) | **200** | the pasta forwarder is bound `0.0.0.0` on the host, so the gateway alias reaches it — and the alias is link-local, which is the dead end below |
|
||||
| `192.168.0.143:7771` (`frank-zealot`) | refused | host-netns listener, unpublished — the rule above, reconfirmed |
|
||||
| `10.89.1.1:7771` | refused | the bridge gateway address belongs to the rootlesskit netns, not to the host |
|
||||
| `host.containers.internal:7771` (`169.254.1.2`) | **200** | the gateway alias is the one address that carries a rootless container to a host-netns listener. Why `hermes-dashboard` timed out on this alias in 2026-09-02 while `open-notebook` answers today is **not established**; the difference to check first is the second network and where the default route lands |
|
||||
| `192.168.0.235:7770` (`frank-archon`) | **200** | a different machine: plain routing, no DNAT in the path |
|
||||
|
||||
The consequence that changes designs: **a rootless container's menu of local model
|
||||
endpoints is `frank-tts` (container DNS, same network) and `frank-archon` (LAN
|
||||
address on another host).** Both Dankclaw endpoints are out of reach —
|
||||
`frank-dragoon` behind root's publish, `frank-zealot` behind the host netns — and
|
||||
the only address that does reach them, `169.254.1.2`, is link-local, which some
|
||||
applications refuse to store at all ([[Open Notebook]] meets exactly that refusal).
|
||||
The consequence that changed designs: **a rootless container's menu of local model
|
||||
endpoints was only `frank-tts` (container DNS, same network) and `frank-archon` (LAN
|
||||
address on another host).** A `pasta` endpoint has no address an app container can
|
||||
use: the LAN address is refused, and the one alias that answers is link-local,
|
||||
which some applications refuse to store at all ([[Open Notebook]] meets exactly
|
||||
that refusal). Pasta cannot be *added to* either —
|
||||
|
||||
```
|
||||
podman run --network pasta --network dankclaw_default …
|
||||
Error: cannot set multiple networks without bridge network mode, selected mode pasta
|
||||
```
|
||||
|
||||
so the only shape that makes such an endpoint nameable is **moving it onto
|
||||
`dankclaw_default`** (`FRANK_DRAGOON_NETWORK` in `scripts/frank-dragoon.env`,
|
||||
2026-10-19). For frank-dragoon that is not an exposure change: the engine's
|
||||
unauthenticated token port binds the container's **own loopback**
|
||||
(`/proc/net/tcp` on 2026-10-19: `127.0.0.1:8730`, `0.0.0.0:8731`), so only the
|
||||
API port becomes name-resolvable, which is the same trust the published 7770
|
||||
already carries on the LAN. `frank-zealot` (7771) stays a host-netns binary and
|
||||
remains unreachable except through the link-local alias.
|
||||
|
||||
The move is verified from the consumer side, so the table above reads as the
|
||||
**pre-move** census and not as today's menu: inside `open-notebook`,
|
||||
`getent hosts frank-dragoon-halogen` → `10.89.1.39`, `GET :8731/v1/models`
|
||||
lists both served models, and the engine's log carries the app's
|
||||
`POST /v1/embeddings 200` from that address (2026-10-19) — see
|
||||
[[Open Notebook]] for what runs over it. What the table still owns is the
|
||||
pasta rule itself: nothing about pasta's forwarding changed, the endpoint was
|
||||
moved out of it.
|
||||
|
||||
## Operational rules
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ never learn a path.
|
||||
**Deployed 2026-10-18** — `./scripts/deploy.sh dankclaw --service frank-tts`
|
||||
both units `active`, no drop-ins, and the homepage card's `wip` gate came off in
|
||||
the same change. Probes on the deployed endpoint: `/health` answers
|
||||
`{"status":"ok","backend":"omni-http","voices":["narrator1","narrator2","narrator3"]}`
|
||||
`{"status":"ok","backend":"omni-http","voices":["dr.kern","garrickhagon","jeffhays","vaati","wilwheaton"]}`
|
||||
and two different voices rendered the same sentence into **different** bytes
|
||||
(24 kHz mono PCM16, 4.24 s, healthy RMS) — so name resolution is real, not a
|
||||
pass-through of one default. The operator harness routes its `speech` role here;
|
||||
@@ -42,8 +42,8 @@ render needs the app installed, not this endpoint.
|
||||
|
||||
## Naming a voice
|
||||
|
||||
A voice **is** a directory name. `voice=narrator1` resolves to
|
||||
`/voices/narrator1/reference.wav`, and that file — not a manifest, not a config
|
||||
A voice **is** a directory name. `voice=wilwheaton` resolves to
|
||||
`/voices/wilwheaton/reference.wav`, and that file — not a manifest, not a config
|
||||
entry — is what makes the name real. omp's TTS device may send `voice_id`; the
|
||||
shim accepts either. A call that sends **neither** — the harness's normal shape,
|
||||
because no harness setting carries a voice for this transport — speaks
|
||||
@@ -59,10 +59,11 @@ loudly instead of letting it half-work.
|
||||
|
||||
Names are user-visible: an unknown voice comes back as
|
||||
`400 unknown voice 'x'; available voices: …`, and that list is the honest
|
||||
census. The podcast speakers are `narrator1`, `narrator2`, `narrator3` —
|
||||
[[Open Notebook]] is configured with exactly those three, so renaming them
|
||||
breaks episode rendering, not just speech. A personal voice gets the person's
|
||||
name, lowercase.
|
||||
census. As of 2026-10-20 the live packs are `dr.kern`, `garrickhagon`,
|
||||
`jeffhays`, `vaati`, `wilwheaton`. [[Open Notebook]]'s `dank-duo` profile speaks
|
||||
`wilwheaton` (Frank) and `vaati` (Comrade), and `DEFAULT_VOICE` is `wilwheaton`.
|
||||
Renaming a pack that a speaker profile names breaks episode rendering, not just
|
||||
speech. A personal voice gets the person's name, lowercase.
|
||||
|
||||
## Voice packs
|
||||
|
||||
@@ -221,5 +222,5 @@ harness bundle and measured the day of the flip:
|
||||
| Endpoint answers, synthesis fails | the engine: `podman healthcheck run frank-tts-engine`, then `systemctl --user status frank-tts-engine.service` |
|
||||
| First request of the day is ~0.6 s slower than the rest — or every name returns 400 | the warm pass did not run; it carries `-`, so it cannot fail the unit. `journalctl --user -u frank-tts-engine` for the warmer's lines, then `podman exec frank-tts-engine /app/warm-voices.sh` — the registry is memory-only (law 1 of [[Frank TTS Engine]]) |
|
||||
| The unit never appears after a deploy | the generator dropped it: `frank-tts-engine.network` is missing from `~/.config/containers/systemd/` |
|
||||
| A consumer rejects `narrator1` by listing 29 names (`af_heart … bm_fable`) and no request ever reaches this endpoint | that consumer is synthesizing **in the browser** with Kokoro.js, whose voice pack is fixed — OpenWebUI 0.11.4 ships it as the `browser-kokoro` TTS engine; see [[OpenWebUI]] |
|
||||
| A consumer rejects `wilwheaton` by listing 29 names (`af_heart … bm_fable`) and no request ever reaches this endpoint | that consumer is synthesizing **in the browser** with Kokoro.js, whose voice pack is fixed — OpenWebUI 0.11.4 ships it as the `browser-kokoro` TTS engine; see [[OpenWebUI]] |
|
||||
---
|
||||
|
||||
@@ -45,6 +45,8 @@ open-notebook ([lfnovo/open-notebook](https://github.com/lfnovo/open-notebook))
|
||||
|
||||
The app's own login stays on behind the chain. It is not redundant: publishing the API on loopback does **not** hide it from this host's other containers, which can dial `open-notebook:5055` by name on `dankclaw_default`. The chain decides *who from outside* gets in; `OPEN_NOTEBOOK_PASSWORD` decides what anything that already reached the port may do.
|
||||
|
||||
**Single tenant — there are no user accounts, so this is an admin-side tool.** Upstream ships one global password and nothing else: `PasswordAuthMiddleware` compares `Bearer <OPEN_NOTEBOOK_PASSWORD>`, and the SurrealDB schema has no user/account/owner table (verified in the running image, 2026-10-20). One shared password therefore means **one shared workspace** — the guard chain can gate the door but cannot partition the data, and Authentik identity is never consumed. Multi-user is *approved but unbuilt* upstream (issue **#712**, "design phase, targeted post-v2"); OIDC/SSO was declined for now (**#607** closed). Real per-person isolation means **one deployment per person**, not a flag to flip. For that reason the homepage lists open-notebook under **admin**, not as a user-facing app.
|
||||
|
||||
## What the image actually runs
|
||||
|
||||
One container, three processes under supervisord (`supervisord.conf` upstream): `uvicorn api.main:app --port 5055` (`:12`), the job worker (`:25`), and the Next.js frontend (`:39`). Two consequences the config depends on:
|
||||
@@ -64,9 +66,11 @@ Credentials live in the database, encrypted with `OPEN_NOTEBOOK_ENCRYPTION_KEY`,
|
||||
|--------|-------|
|
||||
| Credential `frank-tts` | modality `text_to_speech`, base URL `http://frank-tts:7772/v1`, no API key (the shim does not authenticate) |
|
||||
| Credential `frank-archon` | modality `language`, base URL `http://192.168.0.235:7770/v1` |
|
||||
| Credential `frank-dragoon NPU` | modality `embedding`, base URL `http://frank-dragoon-halogen:8731/v1`, dummy API key (the lane does not authenticate) — a container name on `dankclaw_default`, which is the address shape that clears the URL guard below |
|
||||
| Model row `voxcpm2` | type `text_to_speech` on the frank-tts credential. The name is decorative: frank-tts ignores the request's `model` field and synthesizes whatever `voice` names ([[Frank TTS]]) |
|
||||
| Model row `frank-archon` | type `language`; `PUT /api/models/defaults` points chat, transformation and large-context at it. **The embedding default stays null — nothing in the fleet serves `/v1/embeddings`** |
|
||||
| Speaker profile `dank-duo` | two speakers, Frank → `voice_id: narrator1`, Comrade → `narrator2`, `voice_model` = the TTS row |
|
||||
| Model row `frank-archon` | type `language`; `PUT /api/models/defaults` points chat, transformation and large-context at it |
|
||||
| Model row `qwen3-embedding-0.6b` | type `embedding` on the NPU-lane credential; `default_embedding_model` points at it, so every note save embeds through the XDNA2 lane ([[frank-dragoon]]) |
|
||||
| Speaker profile `dank-duo` | two speakers, Frank → `voice_id: wilwheaton`, Comrade → `vaati`, `voice_model` = the TTS row |
|
||||
| Episode profile `dank-daily` | `num_segments: 4`, `speaker_config` = `dank-duo`, outline and transcript LLM = the archon row, briefing that ends "Never invent numbers." |
|
||||
|
||||
Order that works over REST: credentials → model rows (`POST /api/models` with `credential: <credential id>`) → `PUT /api/models/defaults` → speaker profile (`voice_model: <model id>`) → episode profile (`speaker_config` / `outline_llm` / `transcript_llm`). `POST /api/models/{id}/test` proves each row: the language row answered with a real completion from frank-archon; the TTS row reached frank-tts and came back `unknown voice 'alloy'; available voices: narrator1, narrator2, narrator3` — the app's own test path hard-codes an OpenAI voice, so that message *is* the proof the leg works.
|
||||
@@ -78,11 +82,16 @@ Order that works over REST: credentials → model rows (`POST /api/models` with
|
||||
- `192.168.0.143:7771` (frank-zealot, a host-netns listener) and `192.168.0.143:7770` (frank-dragoon, a **root**-published port) both refuse the connection; `host.containers.internal:7771` answers. Full census: [[Container Network Namespaces]].
|
||||
- The app refuses to **store** that one working address: its URL guard rejects any base URL that resolves inside link-local `169.254.0.0/16` (`open_notebook/utils/url_validation.py`, "Link-local addresses are not allowed (metadata endpoints)"), unconditionally — no env var touches it, and `AddHost=` cannot dodge it because the check runs on the resolved address.
|
||||
|
||||
`http://192.168.0.235:7770/v1` — another machine, plain RFC1918 — is the only fleet LLM address that passes both tests. Cost: frank-archon runs `--max-num-seqs 2` and the operator session itself lives there, so a render is a slow-down for whoever else is talking to that endpoint, not a stall; `ESPERANTO_LLM_TIMEOUT=180` covers an outline or transcript call comfortably.
|
||||
`http://192.168.0.235:7770/v1` — another machine, plain RFC1918 — was, when this was first written, the only fleet LLM address that passed both tests; a container name on `dankclaw_default` passes them as well (the embedding credential below is stored that way), but frank-archon stays the scriptwriter for the reason in the next paragraph. Cost: frank-archon runs `--max-num-seqs 2` and the operator session itself lives there, so a render is a slow-down for whoever else is talking to that endpoint, not a stall; `ESPERANTO_LLM_TIMEOUT=180` covers an outline or transcript call comfortably.
|
||||
|
||||
**frank-dragoon can write transcripts, but only greedily — and open-notebook cannot ask for greedy.** Measured 2026-10-19 against the live endpoint: `response_format` as `json_schema` or `json_object` returns schema-valid JSON at `temperature: 0` and a **400** at any sampled temperature — halogen decodes greedy under a schema, and because its sampler default is server-side, omitting `temperature` is sampled, not greedy. That 400 is the trap: esperanto's `is_json_schema_unsupported_error` matches the words *not implemented* in it, so `_SchemaFallbackModel` in `podcast_creator/nodes.py` reads it as "this endpoint has no schema support", downgrades to `json_object`, gets the same 400, and **fails the run** — esperanto's own rescue (drop `response_format` entirely) fires only on the different message `'response_format.type' must be 'json_schema'`. The full law is on [[frank-dragoon]]. And the fix is not reachable from here: a model record is `{credential, name, provider, type}` and a model response adds only timestamps (`GET /openapi.json`, `ModelCreate`/`ModelResponse`) — **there is no per-row config field**, no way to attach `temperature: 0` to one row from the UI or the API. Making `frank-dragoon` the writer would therefore need something in the path that rewrites request bodies, which is a heavier tenant than the contention it removes. **frank-archon stays the scriptwriter.** Reachability would be a separate problem anyway: `frank-dragoon` is not on `dankclaw_default` (the serve script passes no `--network`), so nothing on this network has a name for it, and the one address that does answer — `host.containers.internal:7770` — is the link-local the guard refuses to store ([[Container Network Namespaces]]).
|
||||
**frank-dragoon can write transcripts, but only greedily — and open-notebook cannot ask for greedy.** Measured 2026-10-19 against the live endpoint: `response_format` as `json_schema` or `json_object` returns schema-valid JSON at `temperature: 0` and a **400** at any sampled temperature — halogen decodes greedy under a schema, and because its sampler default is server-side, omitting `temperature` is sampled, not greedy. That 400 is the trap: esperanto's `is_json_schema_unsupported_error` matches the words *not implemented* in it, so `_SchemaFallbackModel` in `podcast_creator/nodes.py` reads it as "this endpoint has no schema support", downgrades to `json_object`, gets the same 400, and **fails the run** — esperanto's own rescue (drop `response_format` entirely) fires only on the different message `'response_format.type' must be 'json_schema'`. The full law is on [[frank-dragoon]]. And the fix is not reachable from here: a model record is `{credential, name, provider, type}` and a model response adds only timestamps (`GET /openapi.json`, `ModelCreate`/`ModelResponse`) — **there is no per-row config field**, no way to attach `temperature: 0` to one row from the UI or the API. Making `frank-dragoon` the writer would therefore need something in the path that rewrites request bodies, which is a heavier tenant than the contention it removes. **frank-archon stays the scriptwriter.** Reachability stopped being the objection on 2026-10-19 — the endpoint joined `dankclaw_default`, which is exactly why the embedding lane below is dialable by name — but no network change reaches the no-per-row-config wall.
|
||||
|
||||
**No embedding model is deployed, and saving a note is where that shows up.** Every note save submits an `embed_note` command that fails in the worker with `No embedding model configured. Please configure one in the Models section.` The note itself is stored and readable; semantic search, source ingestion and notebook-context questions stay dead until something serves `/v1/embeddings`. Podcast generation from `content` does not need embeddings, which is why renders work today. The cheapest door is **halogen's NPU lane, not a new endpoint**: `qwen3-embedding-0.6b` is already in the catalog of the image `frank-dragoon` runs and answers `/v1/embeddings` on the same port (1024 unit-length floats; the instruct prefix goes in the text, and `check_embedding_ctx_length=False` for LangChain). What enabling it costs — the accel device, the host XRT mount, and the fabric-clock hold that GPU+NPU co-run requires — is on [[frank-dragoon]]; after that, the embedding row still has to clear the reachability rule above.
|
||||
**Embeddings run on the XDNA2 lane — provisioned 2026-10-19.** `qwen3-embedding-0.6b` answers `/v1/embeddings` at `http://frank-dragoon-halogen:8731/v1`, the flash container's own bridge address; a container name on `dankclaw_default` resolves to `10.89.1.39`, plain RFC1918, so it clears the URL guard that ruled out `host.containers.internal` above. The lane's shape (one program, the token ceiling, what it costs the Flash model) is owned by [[frank-dragoon]]; the two shapes that bite on the app side are open-notebook's own:
|
||||
|
||||
- **`PUT /api/models/defaults` stores the record ID, not the name.** `"default_embedding_model": "qwen3-embedding-0.6b"` is accepted and echoed back, and then every embed dies in the worker with `Model with ID qwen3-embedding-0.6b not found`. The value has to be the row's `model:…` id — the other defaults in that same object are ids, so mirror them.
|
||||
- **`POST /api/embed` answers 500 while succeeding.** Upstream's `EmbedResponse.command_id` declares `str` and the worker hands it a `RecordID`, so FastAPI raises on serialization *after* the vector is written. The truth is the worker line `Successfully embedded note … in 0.19s`, never the HTTP code. Saving a note (`POST /api/notes`) submits `embed_note` by itself, so that route is only needed to backfill old rows.
|
||||
|
||||
Proof from the app's own paths, 2026-10-19: `POST /api/models/{id}/test` → `Embedding dimensions: 1024`; two notes with nothing in common, then `POST /api/search` with `type: "vector"` (the enum is `text` or `vector`; the score field is `similarity`) — "a yellow waterbird beside a glowing beacon tower" put the grebe note at **0.659** and the boiler note at 0.292, and "quarterly depreciation schedule for industrial boilers" inverted it to **0.906** / 0.318. Embed cost 0.19–0.43 s per note. Semantic search, source ingestion and notebook-context questions are live; podcast generation never needed any of it.
|
||||
|
||||
**Secrets never go in a command line.** The API's bearer token *is* `OPEN_NOTEBOOK_PASSWORD`, and a `-H "Authorization: Bearer …"` argument is world-readable in `/proc/<pid>/cmdline`; put it in a mode-600 curl config and pass `-K`:
|
||||
|
||||
@@ -99,13 +108,13 @@ A **speaker profile** carries one to four speakers (`docs/3-USER-GUIDE/creating-
|
||||
|
||||
| Speaker | Voice ID to type |
|
||||
|---------|------------------|
|
||||
| Speaker 1 | `narrator1` |
|
||||
| Speaker 2 | `narrator2` |
|
||||
| Speaker 3 | `narrator3` |
|
||||
| Speaker 1 | `wilwheaton` |
|
||||
| Speaker 2 | `vaati` |
|
||||
| Speaker 3 | `dr.kern` |
|
||||
|
||||
Those are the pack names frank-tts resolves from `/home/dank/voices` (`scripts/frank-tts.env` header: a voice is real exactly when `/voices/<name>/reference.wav` exists). Two traps, both from upstream's shipped defaults:
|
||||
|
||||
- The three seeded speaker profiles carry **OpenAI voice names** (`nova`, `alloy`, …) and **no voice model at all** — they show a "Setup required" badge, and generation fails with a voice error until they are edited (`creating-podcasts.md:13`, `:33`; `local-tts.md:89`). Add a `narrator1`/`narrator2` pair (or a third/fourth) rather than editing the seeded ones in place.
|
||||
- The three seeded speaker profiles carry **OpenAI voice names** (`nova`, `alloy`, …) and **no voice model at all** — they show a "Setup required" badge, and generation fails with a voice error until they are edited (`creating-podcasts.md:13`, `:33`; `local-tts.md:89`). Add a `wilwheaton`/`vaati` pair (or a third/fourth) rather than editing the seeded ones in place.
|
||||
- Two speakers with the same voice are hard to follow (`creating-podcasts.md:89`). Distinct packs per speaker is the point of the 1–4 shape.
|
||||
|
||||
## Two buttons that fail against frank-tts, and why that is cosmetic
|
||||
@@ -164,6 +173,7 @@ podman pull docker.io/lfnovo/open_notebook:v1-latest@sha256:54b2025a4a4853c188f7
|
||||
- Local measurements (2026-10-18, workstation, pinned digests): `surreal start --help` (credential and bind options carry `[env: SURREAL_*]`); running the pinned SurrealDB digest with env-only credentials (credential init in the log, correct login authenticates, wrong password refused, `/surreal is-ready` → `OK` exit 0, named volume owned by the mapped uid); the v1-compatible Quadlet generator dry-run of all three unit files (two `--env-file` in order, two `--network` in order, named volume + host bind, both health probes, `--pull never`)
|
||||
- Host evidence (Dankclaw, 2026-10-18, read-only): `ss -ltn` (`127.0.0.1:5055` held by seerr, `0.0.0.0:7771` a host listener, nothing on 7772/8502/5056/8000), `podman ps -a` (no open-notebook container, `dankclaw_default` present), subnet registry from `dankclaw/open-webui/quadlet/open-webui.network`
|
||||
- Deploy + provisioning (Dankclaw, 2026-10-19): first-boot log — migrations 1→25, `Credentials were provided, and no root users were found. The root user 'open_notebook' will be created`, `api`/`worker`/`frontend` all RUNNING under supervisord; `GET :8502/config` → the `API_URL` value; `GET /api/providers` (the `openai_compatible` modality list); `GET /openapi.json` schemas for `CreateCredentialRequest` / `ModelCreate` / `SpeakerProfileCreate` / `PodcastGenerateRequest`; the POSTs listed above; the note-save worker error `No embedding model configured.`
|
||||
- Embedding lane (Dankclaw, 2026-10-19): `POST /api/credentials` → `credential:7fdqz09tt0c4xp68x7cm`, `POST /api/models` → `model:p3ki598dydfinu93tbny`, `POST /api/models/{id}/test` → `Embedding dimensions: 1024`; worker lines `Successfully embedded note … in 0.43s` / `0.19s`; the name-not-id failure `Model with ID qwen3-embedding-0.6b not found`; the `/api/embed` 500 body (`EmbedResponse.command_id`, `input_value=RecordID(table_name=command…)`); `POST /api/search` `similarity` 0.6590/0.2921 and 0.9059/0.3176 for the two-note pair; `podman logs frank-dragoon-halogen` showing `POST /v1/embeddings 200` from `10.89.1.21`. Both probe notes deleted afterwards.
|
||||
- First render (2026-10-19): job `command:9rdojq5ne7dj9caicq20`, `execution_time: 301.21 s`, `episode:7ft03thj98sn0ek9xf29`; `episodes/d5fed29b-33e9-40ac-b46a-e0537c56c256/clips/0001.mp3 … 0023.mp3`; `…/audio/d5fed29b-….mp3` = 11,957,420 bytes, mp3 24 kHz mono, `ffprobe` duration 747.31 s; `GET /api/podcasts/episodes/{id}/audio` → 200 `audio/mpeg` with the bearer, 401 without.
|
||||
- Upstream code read inside the pinned image (2026-10-19): `frontend/next.config.ts:26-43` (the `/api/*` rewrite and the one-port claim), `frontend/src/app/config/route.ts:122-173` (`/config` priority order; autodetect appends `:5055`), `frontend/src/app/api/_sse-proxy.ts` (SSE proxied with `no-transform`), `open_notebook/utils/url_validation.py` (link-local refusal, `is_blocked_ip`, no environment override anywhere in the file).
|
||||
- Guard chain (2026-10-19): provider `notebook-proxy` created through `podman exec -i authentik-server ak shell` with the flows copied from `embedded-outpost` (pk 1), Application `Open Notebook`/`notebook-guard`, embedded-outpost provider set 9 → 10; afterwards the outpost logged zero `Failed to fetch outpost from API` and `seerr`/`grafana` still answered 200. Before the route existed, `notebook.franksdank.cloud` resolved to nothing (`000`).
|
||||
|
||||
@@ -290,9 +290,12 @@ on large sizes ⇒ check ComfyUI resources; 401 ⇒ the token is bad.
|
||||
|
||||
## Text-to-speech (frank-tts)
|
||||
|
||||
TTS is the fleet's one OpenAI-compatible speech endpoint —
|
||||
`http://host.containers.internal:7772/v1` (`audio.tts.openai.api_base_url`
|
||||
in the config DB, 2026-10-18), model `frank-tts`, voice `narrator1`. That
|
||||
TTS is the fleet's one OpenAI-compatible speech endpoint. The config DB
|
||||
`audio.tts.*` is set to engine `openai`, `audio.tts.openai.api_base_url`
|
||||
`http://192.168.0.143:7772/v1`, `audio.tts.voice` `wilwheaton` (re-keyed
|
||||
2026-10-20 from the retired `narrator1`; 7772 is a published port so
|
||||
`host.containers.internal` also reaches it, but the LAN IP is the canonical
|
||||
on-host form). That
|
||||
port is the rootless quadlet `frank-tts` (this repo:
|
||||
`dankclaw/voiceservice/quadlet/frank-tts.container`
|
||||
`scripts/frank-tts.env`), and its shim resolves `voice=<name>` against
|
||||
@@ -306,8 +309,8 @@ ships an in-page Kokoro implementation (chunk carrying a frozen voice table
|
||||
`onnx-community/Kokoro-82M-v1.0-ONNX`, plus a personal *Kokoro.js dtype*
|
||||
setting). Its engine select takes `""` (Default) or `browser-kokoro`; when
|
||||
`browser-kokoro` is chosen the page synthesizes locally and rejects any
|
||||
voice outside its fixed pack — so `narrator1` fails with
|
||||
`Voice: "narrator1" not found. Should be one of: af_heart, …` **without a
|
||||
voice outside its fixed pack — so `wilwheaton` fails with
|
||||
`Voice: "wilwheaton" not found. Should be one of: af_heart, …` **without a
|
||||
single request reaching the server**. Personal audio settings are
|
||||
browser: no `user.settings` row and no container env carries an audio
|
||||
override, yet the toast still appears ⇒ the choice lives in that browser's
|
||||
|
||||
@@ -3,70 +3,111 @@ type: reference
|
||||
description: "Alloy is Grafana's log and telemetry shipper."
|
||||
tags:
|
||||
- infrastructure
|
||||
timestamp: 2026-09-11
|
||||
timestamp: 2026-10-20
|
||||
---
|
||||
|
||||
# Alloy
|
||||
|
||||
Alloy is Grafana's log and telemetry shipper. It replaces Promtail (end of life as of 2026-03-02, https://grafana.com/docs/loki/latest/send-data/promtail/). It is wired to forward container and journal logs to Loki, and every unit in the path is healthy — but as of 2026-10-18 it is delivering **zero** log lines. See "Delivery state" below.
|
||||
Alloy is Grafana's log and telemetry shipper. It replaces Promtail (end of life
|
||||
as of 2026-03-02, https://grafana.com/docs/loki/latest/send-data/promtail/). It
|
||||
forwards the host journal — including the rootless Quadlet containers' logs —
|
||||
to Loki. Unlike its observability siblings it runs as a **root system service on
|
||||
the host network**, because a rootless container cannot read the journal (see
|
||||
*Why root* below).
|
||||
|
||||
**Host:** Dankclaw (one instance per host) — **Port:** 12345 (loopback) — **Docs:** https://grafana.com/docs/alloy/latest
|
||||
**Host:** Dankclaw (one instance per host) — **UI/health:** 127.0.0.1:12345
|
||||
(loopback) — **Docs:** https://grafana.com/docs/alloy/latest
|
||||
|
||||
## Facts
|
||||
|
||||
| Item | Value |
|
||||
|------|-------|
|
||||
| Image | `franksdank/alloy:latest` (local build: upstream + a static busybox for the healthcheck) |
|
||||
| Published | `127.0.0.1:12345` → 12345 (its own UI/health port) |
|
||||
| Unit | `dankclaw/observability/quadlet/alloy.container` → `alloy.service` |
|
||||
| Config | `shared/alloy/config.alloy`, installed to `/opt/alloy/config/config.alloy` on every deploy (`scripts/deploy-remote.sh:281-284`) |
|
||||
| Inputs | host journald socket, podman container logs, `/var/log` |
|
||||
| Output | `http://loki:3100/loki/api/v1/push` (`shared/alloy/config.alloy:13-17`) |
|
||||
| Successor to | Promtail (EOL 2026-03-02, https://grafana.com/docs/loki/latest/send-data/promtail/) |
|
||||
| Boot persistence | `Restart=always`/`RestartSec=10` + `WantedBy=default.target` (`dankclaw/observability/quadlet/alloy.container:58-63`); installed at `~/.config/containers/systemd/alloy.container`, reporting `generated` + `active` (`systemctl --user is-enabled|is-active alloy` 2026-10-18) |
|
||||
| Delivery | **Configured and healthy, currently delivering zero data** — see "Delivery state" |
|
||||
| Image | `localhost/franksdank/alloy:latest` — built rootless, then mirrored into the root image store by the deployer (a root build fails on DNS in this host's build container) |
|
||||
| Unit | `dankclaw/observability/quadlet/alloy.container` → system `alloy.service`, installed at `/etc/containers/systemd/` (root manager) |
|
||||
| Config | `shared/alloy/config.alloy` → `/opt/alloy/config/config.alloy` on every deploy |
|
||||
| Network | `host` — required so it can reach Loki's `127.0.0.1:3100` publish |
|
||||
| UI | `--server.http.listen-addr=127.0.0.1:12345` (loopback; nothing off-host reaches it) |
|
||||
| Inputs | host journal: `system.journal` + `user-*.journal` (root reads both) |
|
||||
| Output | `http://127.0.0.1:3100/loki/api/v1/push` |
|
||||
| Boot persistence | `Restart=always`/`RestartSec=10` + `WantedBy=multi-user.target`; boot persistence is the `[Install]` section (generated system units cannot be `enable`d here) |
|
||||
| Successor to | Promtail (EOL 2026-03-02) |
|
||||
| Deploy status | root cutover built and tested in-repo (2026-10-20); the **live** instance is still the old rootless unit until `./scripts/deploy.sh dankclaw --service alloy` runs |
|
||||
|
||||
## Why Alloy and not Promtail
|
||||
## Why root (the load-bearing decision)
|
||||
|
||||
Promtail reached end-of-life on 2026-03-02 and Grafana Labs directs new installs to Alloy. This stack uses Alloy from day one: no Promtail config or unit has ever existed in this repository (`git log -S promtail` over the whole history returns no commit that added one; the compose-era service of the same name was already Alloy — `dankclaw/observability/quadlet/alloy.container:6-7`).
|
||||
A rootless Alloy delivered **zero** lines, silently and with a green health
|
||||
check. Two independent causes, both proven on the host 2026-10-20:
|
||||
|
||||
1. **The journal is not readable by a rootless container.** The files are
|
||||
`root:systemd-journal 0640` (`group:adm`, `other::---`). Even bind-mounted, a
|
||||
rootless container has no supplementary groups, so opening them returns
|
||||
`DENIED`. `loki.source.journal` then sees an empty view and the counter just
|
||||
sits at 0 — no error logged.
|
||||
2. **Container logs live in the *user* journal, not `ctr.log`.** The rootless
|
||||
Quadlet containers log via the journald driver into `user-1000.journal`; the
|
||||
old file globs (`/var/lib/containers/.../ctr.log`, `/var/log/containers/*.log`)
|
||||
point at a root-storage layout that does not exist here.
|
||||
|
||||
**Root fixes both at once.** As root the container opens every journal file
|
||||
under the machine-id directory — `system.journal` *and* `user-1000.journal` —
|
||||
so a single `loki.source.journal` carries host services and the rootless
|
||||
containers' logs together. Validated with a throwaway root probe before writing
|
||||
the unit: **31,825 journal lines read, 26,966 pushed to Loki in 20 s, zero
|
||||
drops**, and Loki's label set went from empty to populated.
|
||||
|
||||
### The two mounts that make it work
|
||||
|
||||
- **`/etc/machine-id` (ro) — load-bearing.** `sd_journal` locates the journal
|
||||
directory by the *local* machine id. Without it the container opens nothing and
|
||||
the journal source reads exactly 0 lines — silently, health green. This single
|
||||
mount is the whole difference between a working and an empty pipeline; do not
|
||||
remove it. (A root probe without it reads 0; with it, 31,825.)
|
||||
- **`Network=host`.** Loki publishes `127.0.0.1:3100` (loopback only). No bridge
|
||||
container — rootless or root — can reach the host's loopback, so Alloy must
|
||||
share the host network to push to `127.0.0.1:3100`. Consequence: the UI binds
|
||||
`127.0.0.1` directly (no `PublishPort`), so it stays loopback-only as before.
|
||||
|
||||
## Configuration
|
||||
|
||||
`shared/alloy/config.alloy` is the source of truth:
|
||||
|
||||
- Reads journald logs from the host's `/run/systemd/journal/socket`, labelled `job="systemd-journal"` (`shared/alloy/config.alloy:20-26`).
|
||||
- Reads container logs from two globs: `/var/lib/containers/storage/overlay-containers/*/userdata/ctr.log` (`job="podman"`) and `/var/log/containers/*.log` (`job="containers"`) (`config.alloy:29-35`).
|
||||
- Forwards everything to `http://loki:3100/loki/api/v1/push` by container name on `dankclaw_default` (`config.alloy:13-17`).
|
||||
`shared/alloy/config.alloy` is the source of truth — one `loki.source.journal`
|
||||
(labelled `job="systemd-journal"`) forwarding to the loopback `loki.write`
|
||||
endpoint. There is deliberately **no** container-file source: container stdout is
|
||||
in the journal.
|
||||
|
||||
## Volumes
|
||||
|
||||
| Container path | Host path |
|
||||
|----------------|-----------|
|
||||
| `/etc/alloy` | `/opt/alloy/config` |
|
||||
| `/run/systemd/journal/socket` | host journald socket (read-only) |
|
||||
| `/var/log` | host logs (read-only) |
|
||||
| `/var/lib/containers` | podman storage (read-only) |
|
||||
| `/etc/alloy` | `/opt/alloy/config` (ro) |
|
||||
| `/etc/machine-id` | host machine id (ro) — **required** |
|
||||
| `/run/systemd/journal/socket` | host journald socket (ro) |
|
||||
| `/var/log/journal` | persistent journal files (ro) |
|
||||
| `/run/log/journal` | runtime journal files (ro) |
|
||||
|
||||
## The UI port is published on purpose
|
||||
## The UI stays on loopback
|
||||
|
||||
Alloy's UI listens on the container's own `127.0.0.1:12345` by default, and Alloy 1.x refuses to configure the `ui` service from a config block ("service ui does not support being configured", `shared/alloy/config.alloy:8-10`). Under compose that meant **no reachable health endpoint at all**: the homepage status card for alloy read `down` continuously and a deploy had nothing to wait on. The unit therefore passes `--server.http.listen-addr=0.0.0.0:12345` (`dankclaw/observability/quadlet/alloy.container:46`) and publishes `127.0.0.1:12345` (`alloy.container:47`, confirmed by `ss -ltnp` 2026-10-18) — same posture as the other three: reachable from the host and from Caddy's network, and from nothing off-host. Alloy has no authentication, which is exactly why it is not published wider. The homepage card probes that same loopback port (`shared/homepage/services.json:233`).
|
||||
|
||||
## Delivery state (2026-10-18): healthy, shipping nothing
|
||||
|
||||
The unit is `active`, the container reports `healthy`, the log says `Alloy is ready`, and the pipeline carries no data at all. Measured on 2026-10-18:
|
||||
|
||||
- `curl -s http://127.0.0.1:12345/metrics` → `loki_source_journal_target_lines_total{component_id="loki.source.journal.journal"} 0` and `loki_source_file_files_active_total{component_id="loki.source.file.containers"} 0`. The journal source logs no error; the counter simply never moves. The journal files it would have to read through the `/var/log` bind are `root:systemd-journal 0640` with an ACL (`ls -l /var/log/journal/64a287d7…/` 2026-10-18).
|
||||
- Both file globs were refused at start (`journalctl --user -u alloy`, 2026-10-18T08:25:27Z), verbatim: `failed to create source, skipping … component_id=loki.source.file.containers error="failed to tail file, stat failed: stat /var/lib/containers/storage/overlay-containers/*/userdata/ctr.log: permission denied"` and the same message for `/var/log/containers/*.log: no such file or directory`.
|
||||
- Downstream that means Loki holds zero streams — `GET /loki/api/v1/labels` returns `{"status":"success"}` with no `data` — see [[Loki]].
|
||||
|
||||
No fix is claimed here: the unit, the config and the push endpoint are all correct as written, and the gap is source-side access (and a Kubernetes-style glob that does not exist on this host). Anyone closing it should re-check the two counters above, not the health status — health is exactly what hides this failure mode.
|
||||
Alloy 1.x refuses to configure the `ui` service from a config block, so the UI
|
||||
address is set on the command line instead. Under the old rootless unit that
|
||||
meant `0.0.0.0` inside the container plus a published `127.0.0.1:12345`; under
|
||||
`Network=host` the listen-addr is `127.0.0.1:12345` directly — the host's
|
||||
loopback, reachable from the host and from nothing off-host. Alloy has no
|
||||
authentication, which is exactly why it is never published wider. The homepage
|
||||
card and the deploy health gate both dial `http://127.0.0.1:12345/-/healthy`.
|
||||
|
||||
## Gotchas
|
||||
|
||||
- The journald socket mount is required for journal scraping.
|
||||
- Container log paths differ between docker (`/var/lib/docker/containers`) and podman (`/var/lib/containers/storage/...`); point the config at the right one per host.
|
||||
- One Alloy per host; do not run a single instance that tries to reach both hosts' logs.
|
||||
- The unit `Wants=`/`After=` `loki.service`: without a reachable Loki the write client fails and the pipeline backs up.
|
||||
- **No `/etc/machine-id` mount ⇒ silent zero.** The health check still passes.
|
||||
This is the failure mode that hid the broken pipeline; check
|
||||
`loki_source_journal_target_lines_total` and the Loki label query, never
|
||||
`/healthy`.
|
||||
- **`After=loki.service` is gone.** Alloy is on the system manager; Loki is a
|
||||
rootless user unit — a system unit cannot order against a user unit. Alloy
|
||||
retries its push until Loki answers, so ordering is unnecessary.
|
||||
- **The image must be in the ROOT store.** The deployer builds rootless and
|
||||
mirrors (`mirror_image_to_root`); a `Pull=never` root quadlet with only a
|
||||
rootless tag fails at create.
|
||||
- One Alloy per host; do not run a single instance that tries to reach both
|
||||
hosts' logs.
|
||||
|
||||
## See Also
|
||||
|
||||
|
||||
@@ -81,7 +81,7 @@ Two provisioning rules, each verified live by an outage:
|
||||
appears. Create the Application with `provider_id` set (the UI does this
|
||||
automatically; raw API/ORM drives must do it explicitly).
|
||||
|
||||
## OAuth2 relying-party quirks (this fork, verified live 2026-08-31)
|
||||
## OAuth2 relying-party quirks (this fork; corrected live 2026-10-20)
|
||||
|
||||
The Grafana RP provider (`grafana`, pk 9, confidential client) was created and
|
||||
verified end-to-end via `ak shell` ORM drives. This fork's OAuth2 provider
|
||||
@@ -100,23 +100,32 @@ implementation diverges from upstream in ways that bit us; all verified live:
|
||||
- **Provider resolution is by `client_id` query param** on the global
|
||||
`/application/o/authorize/` route; per-slug authorize paths do not exist in
|
||||
this build. Token/userinfo live at `/application/o/token/` and
|
||||
`/application/o/userinfo/?api=1`.
|
||||
- **Property-mapping evaluator is broken → userinfo returns `sub` only.**
|
||||
`PropertyMappingEvaluator.evaluate(*args, **kwargs)` forwards kwargs to
|
||||
`BaseEvaluator.evaluate(expression_source)`, so the first context kwarg
|
||||
becomes the *expression source* instead of the context; every call raises a
|
||||
swallowed `PropertyMappingExpressionException` and the claim is skipped.
|
||||
Scope-grant filtering has the same class of bug: `ScopeMapping` has no
|
||||
provider FK (linkage lives in the `ProviderPropertyMapping` through model)
|
||||
so `filter(provider=...)` always yields an empty queryset and tokens asked
|
||||
to carry scopes get zero. Net effect: userinfo/introspection answers with
|
||||
`{sub}` only — email/groups/entitlements claims cannot be delivered by this
|
||||
build until the evaluator is patched or upgraded. Clients consuming this
|
||||
provider must key identity off `sub`.
|
||||
- Scope mappings for this provider exist — `grafana-email-scope` and
|
||||
`grafana-profile-scope` are both rows in `authentik_core_propertymapping`
|
||||
(read-only `SELECT`, 2026-10-18) — and will start emitting claims the moment
|
||||
the evaluator bug is fixed; no config change is needed then.
|
||||
`/application/o/userinfo/` — do **not** append `?api=1` to a Grafana API URL.
|
||||
- **NULL `authorization_flow` → authorize 500, `AttributeError: 'NoneType'
|
||||
object has no attribute 'slug'`.** A provider created via raw `ak shell`
|
||||
`create()` leaves `authorization_flow` unset; the authorize view then crashes
|
||||
building the flow planner (`flows/planner.py`) and returns "Server Error —
|
||||
Powered by authentik" before any claim is built. This was previously
|
||||
mis-attributed to a broken evaluator; the crash is the missing flow. Fix: set
|
||||
`authorization_flow = default-provider-authorization-implicit-consent` (the
|
||||
flow every other provider in this instance already uses).
|
||||
- **A scope mapping that returns a non-dict emits no claim.** authentik merges a
|
||||
granted scope mapping's evaluated result into the token **as a mapping**: a
|
||||
dict contributes its keys as claims; a bare string is keyed under the
|
||||
mapping's NAME, so nothing usable appears. `grafana-email-scope` returned the
|
||||
bare string `user.email`, so no `email` claim landed — the evaluator was
|
||||
never broken (the dict-valued `grafana-profile-scope` mapping has always
|
||||
delivered, and the stock OpenID mappings return dicts). Fix:
|
||||
`return {"email": user.email, "email_verified": True}`. With no email, Grafana
|
||||
v13 falls back to GET `<api_url>/emails`; authentik 404s it → "Login failed:
|
||||
InternalError".
|
||||
- **Grafana reads identity from the standard userinfo endpoint** (`API_URL` with
|
||||
no `?api=1`); no `USE_ID_TOKEN` is needed once the `email` claim exists.
|
||||
- **Net (2026-10-20):** the evaluator is fine; email/profile claims reach RP
|
||||
clients. Both IdP fixes are idempotent in
|
||||
`scripts/authentik-grafana-sso-fix.sh` (re-run after an ak-shell re-creation of
|
||||
the provider or a blueprint import, which revert them). Grafana's half plus the
|
||||
verified outcome (Viewer, `login = <sub hex>`, email present) are in [[Grafana]].
|
||||
|
||||
## LDAP provider objects (provisioned via `ak shell` ORM)
|
||||
|
||||
|
||||
@@ -267,6 +267,12 @@ The silent re-auth path was proven end-to-end: after deleting
|
||||
`qa-e2e-admin`'s rows, the e2e admin spec ran against the stale storage
|
||||
state and passed with no login prompt (2 tests, 2.9 s).
|
||||
|
||||
## Login-session lifetime (how often you re-authenticate)
|
||||
|
||||
How long a login stays valid is the **`default-authentication-login` stage's `session_duration`** (`UserLoginStage`) — not a global setting, and not `SESSION_COOKIE_AGE` (that is the *pre-login* session). Raised to **`days=7`** on 2026-10-20 (was stage value `0` = the 24 h framework default). It is an **absolute** window, never sliding: authentik implements `SESSION_SAVE_EVERY_REQUEST` nowhere, so activity does not extend it and no supported knob makes it one.
|
||||
|
||||
**`session_duration` is a STRING** in authentik's idiom (`"days=7"`, `"seconds=30"`, `"hours=12"`). A Python `timedelta` stores `"7 days, 0:00:00"`, which `authentik.lib.utils.time.timedelta_from_string` rejects with `ValueError` and login then crashes computing the expiry — so set it with **`scripts/authentik-set-login-session-duration.sh`** (validates the form first). Any non-zero value applies to *every* login (the "remember me" box only adds `remember_me_offset` on top) and a positive `set_expiry` also overrides `SESSION_EXPIRE_AT_BROWSER_CLOSE`, so closing the browser no longer logs you out; **existing sessions keep their old expiry** — delete the user's `authentik_core_session`/`…proxysession` rows and re-login to force it now. This is **DB state, not repo-managed**: a blueprint re-import or rebuild reverts the stage to `0`, so re-run the script after one. The other two login stages (`default-source-*`) drive external-source/enrollment logins and stay at the framework default.
|
||||
|
||||
## Guard-chain flip recipe (adding a guarded vhost)
|
||||
|
||||
Move a public vhost from `reverse_proxy <app>:<port>` onto the chain:
|
||||
|
||||
@@ -4,7 +4,7 @@ description: "Grafana visualizes the metrics that Prometheus collects from every
|
||||
tags:
|
||||
- infrastructure
|
||||
- grafana
|
||||
timestamp: 2026-09-11
|
||||
timestamp: 2026-10-20
|
||||
---
|
||||
|
||||
# Grafana
|
||||
@@ -64,19 +64,28 @@ Grafana responds on `/api/health` with 200 and `"database":"ok"`; measured 2026-
|
||||
- Change the default admin credentials at first boot (env or UI).
|
||||
- Dashboards provisioned as code (dashboard JSON plus a YAML provider, `shared/grafana/provisioning/dashboards/`) survive container recreation; UI-created ones live in the data volume — back it up.
|
||||
- **Auth = OAuth2 relying-party, not auth_proxy.** The embedded outpost carries
|
||||
no provider token (`token=None`), so no identity header ever reaches Grafana;
|
||||
RP mode is topology-independent. The full verified contract (provider pk 9,
|
||||
client_id `grafana-21ca190cde75d60f`, global authorize/token/userinfo paths,
|
||||
Basic client auth, form-encoded token body, and the fork's userinfo-claims
|
||||
bug that leaves identity at the `sub` hash only) is in
|
||||
[[Authentik Outpost Reference]] → "OAuth2 relying-party quirks". Consequences for Grafana:
|
||||
The unit sets `ALLOW_SIGN_UP=false`, `AUTO_UPSERT=true`, `ALLOW_ASSIGN_ORG=true`
|
||||
`ORG_ID=1` and **no** role-attribute mapping (`grafana.container:52-55`), so
|
||||
users land as Viewer with `login = <sub hex>`; no email/groups/role claims
|
||||
reach Grafana until the evaluator bug is fixed — do not write role mappings
|
||||
that depend on those claims. The local admin account stays the only Admin.
|
||||
(The Viewer-default outcome and the `<sub hex>` login are not readable from
|
||||
the host without the database or an admin password. *(unverified 2026-10-18)*)
|
||||
no provider token (`token=None`), so no identity header reaches Grafana; the
|
||||
login is Grafana's own "Sign in with Authentik" button against the `grafana`
|
||||
confidential-client provider. **Verified end-to-end 2026-10-20** after four
|
||||
independent breakages had made it fail. The two Authentik-side causes (a NULL
|
||||
`authorization_flow` → authorize 500; an email scope mapping that returned a
|
||||
bare string so no `email` claim was emitted) and the full wire contract live in
|
||||
[[Authentik Outpost Reference]] → "OAuth2 relying-party quirks"; the IdP half is
|
||||
re-applied by `scripts/authentik-grafana-sso-fix.sh` (idempotent — re-run after
|
||||
an ak-shell re-creation of the provider or a blueprint import, both of which
|
||||
revert it silently). The Grafana half is the unit's own env (`grafana.container`):
|
||||
`SCOPES="openid email profile"` (a GitLab-style `user:email` scope crashes the
|
||||
authorize view), the standard userinfo `API_URL` with **no `?api=1`**, the
|
||||
attribute paths `email`/`sub`/`preferred_username`, and `ALLOW_SIGN_UP=true`
|
||||
(safe: the guard fronts the callback). Grafana v13 only GETs `<api_url>/emails`
|
||||
when the token carries no `email` claim — so it is the email claim itself (an
|
||||
IdP fix, above) that makes the login land, not any Grafana toggle.
|
||||
- **What a new login becomes:** a Viewer in org 1 (`ALLOW_ASSIGN_ORG=true`,
|
||||
`ORG_ID=1`, no role-attribute mapping), with `login = <sub hex>` but the real
|
||||
`email`. The local admin account stays the only Admin. The landing that the
|
||||
edge-only 13-sso spec never checked is now asserted by
|
||||
`tests/e2e/specs/20-grafana-sso.spec.ts` (RP round-trip → `/api/user` 200 with
|
||||
an email).
|
||||
- **Never point this unit's `EnvironmentFile` at the repo-root `.env`.** That
|
||||
file is a gluetun env file and it carries `HOME=/root`; systemd imports it
|
||||
for the service, podman then looks for its configuration under `/root`
|
||||
|
||||
@@ -3,7 +3,7 @@ type: reference
|
||||
title: "Observability on Dankclaw"
|
||||
description: "The observability stack: what runs, how it is deployed, how it is exposed, and what is still unconfigured."
|
||||
tags: [observability, prometheus, grafana, loki, alloy, node-exporter, quadlet, dankclaw]
|
||||
timestamp: 2026-09-11
|
||||
timestamp: 2026-10-20
|
||||
---
|
||||
|
||||
# Observability on Dankclaw
|
||||
@@ -83,27 +83,37 @@ health endpoint (`/prometheus`: `/-/healthy`, `/alertmanager`: `/-/healthy`
|
||||
[[Observability Provisioning]] for what is provisioned as code and how to
|
||||
verify it through the APIs.
|
||||
|
||||
## Current state (2026-10-18)
|
||||
## Current state (2026-10-20)
|
||||
|
||||
* **Metrics: working.** `curl -s http://127.0.0.1:9090/api/v1/targets` reports
|
||||
eight up targets; the four downs are structural (the `comfyui` and `services`
|
||||
HTML probes, and Gitea's disabled metrics endpoint), not outages. Rule set:
|
||||
4 alert rules + 2 recording rules (`/api/v1/rules`, matching
|
||||
`shared/prometheus/rules/llm-fleet.yml`).
|
||||
eight up targets — the three LLM endpoints (frank-archon on `vllm:*`,
|
||||
frank-dragoon and frank-zealot on `llamacpp:*`), grafana, loki, prometheus
|
||||
and both node exporters. The four downs are structural (the `comfyui` and
|
||||
`services` HTML probes, and Gitea's disabled metrics endpoint), not outages.
|
||||
Two rule groups in `shared/prometheus/rules/llm-fleet.yml`: `llm-fleet`
|
||||
(2 recording + 4 alert, `llamacpp:*`) and `vllm-fleet` (1 recording + 4
|
||||
alert, `vllm:*` — archon queueing, prefix collapse, preemption, p95
|
||||
time-to-first-token), each keyed only to the namespace its runtime exports.
|
||||
* **Delivery: working.** Alertmanager's single `signal` receiver webhooks the
|
||||
relay, and `journalctl --user -u signal-relay` records successful
|
||||
`sent to +12062476336:` lines for every rule in the set.
|
||||
* **Dashboards: working.** One provisioned Prometheus datasource (uid
|
||||
`prometheus`) and the `llm-fleet` dashboard, both installed from
|
||||
`shared/grafana/provisioning/`.
|
||||
* **Logs: configured, empty.** Alloy ships zero lines (journal counter 0, both
|
||||
file globs refused) and Loki therefore holds zero streams — `/opt/loki/data`
|
||||
is a few hundred KB of WAL/compactor bookkeeping with **no chunk files**
|
||||
(8 files total by `find`, 2026-10-18) and `GET /loki/api/v1/labels` returns
|
||||
no label names.
|
||||
See [[Alloy]] → Delivery state and [[Loki]] → Contents.
|
||||
* **Not configured:** no Loki datasource in Grafana, no blackbox exporter (the
|
||||
structural downs have no real uptime signal), no Alertmanager vhost.
|
||||
* **Dashboards: working.** Provisioned datasources — Prometheus (uid
|
||||
`prometheus`) and Loki (uid `loki`) — and the `llm-fleet` dashboard, all
|
||||
installed from `shared/grafana/provisioning/`. The Loki datasource is how
|
||||
you read logs: Loki has no web UI of its own, so Grafana Explore is the
|
||||
human surface (the guarded `loki.franksdank.cloud` vhost serves only the API).
|
||||
* **Logs: fixed in code, awaiting deploy.** The pipeline shipped nothing because
|
||||
a rootless Alloy cannot read the `0640` journal files, and the container-log
|
||||
globs point at a root-storage layout that does not exist here. The fix is built
|
||||
and tested in-repo (2026-10-20): Alloy is now a **root system service on
|
||||
`Network=host`**, reading `system.journal` + `user-*.journal` and pushing to
|
||||
Loki's `127.0.0.1:3100` loopback publish — a throwaway root probe read 31,825
|
||||
lines and pushed 26,966 with zero drops. The **live** instance is still the old
|
||||
rootless unit until `deploy.sh dankclaw --service alloy` runs. Detail in
|
||||
[[Alloy]] → Why root; Loki's store state in [[Loki]] → Contents.
|
||||
* **Not configured:** no blackbox exporter (the structural downs have no real
|
||||
uptime signal) and no Alertmanager vhost. The log pipeline's fix is in code but
|
||||
not yet deployed (see *Logs* above).
|
||||
|
||||
## Citations
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ description: "Pi-hole is the LAN DNS server for the homelab."
|
||||
tags:
|
||||
- infrastructure
|
||||
- pi-hole
|
||||
timestamp: 2026-08-28
|
||||
timestamp: 2026-10-19
|
||||
---
|
||||
|
||||
# Pi-hole
|
||||
@@ -46,6 +46,8 @@ The file format is hosts-style (`<IP> <name> [<name> ...]`), one A record per Ca
|
||||
|
||||
**Pitfall (2026-08-24):** the first quadlet mounted the file into `/etc/dnsmasq.d/`. FTL never read it; names only resolved from the query cache. A fresh name returned NXDOMAIN. Mount into `/etc/pihole/hosts/`, not `/etc/dnsmasq.d/`.
|
||||
|
||||
**Pitfall (2026-10-19): FTL can come up with the record file mounted and still serve none of it.** A plain `systemctl --user restart pihole` (16:25, inside the open-notebook deploy window) left every `franksdank.cloud` name NXDOMAIN for LAN clients while **external names kept answering** — from FTL's cache, which is exactly what makes the failure read like a client or an upstream problem. **The trigger is a container restart, not a boot**, and that matters for where the check belongs: `uptime -s` reads 2026-10-18 01:24 (two days of uptime) while `journalctl --user -u pihole.service` shows `Stopping pihole.service` at 16:25:23 and the container's `StartedAt` at 16:25:27 — so verify after **any** pihole restart, deploy-triggered or manual, not only after a reboot. The instrument is on the box: `getent hosts grafana.franksdank.cloud` (or `dig +short @192.168.0.143 <name>`) answering empty means FTL is holding no local records at all; here the file was present and readable inside the container (`-rw-rw-r-- … /etc/pihole/hosts/addn-hosts`, 44 records) and `dns.hosts` in `pihole.toml` was intact. `podman exec pihole pihole restartdns reload` had every name answering within seconds. The mechanism is **not established** — the same mounts and the same config read fine before and after the reload — so treat it as a load failure of the record sources at FTL start, not as a lost or unmounted file. What it cost: the whole Playwright gate, 18 specs across every vhost failing `ERR_NAME_NOT_RESOLVED` while the fleet itself was healthy ([[E2E Suite]]).
|
||||
|
||||
## Upstream
|
||||
|
||||
The slirp netns has no useful `resolv.conf`. The env file pins the upstreams: `DNS1=1.1.1.1`, `DNS2=1.0.0.1` (mirrors the retired `dnsmasq.conf` `server=` lines). Every query that is not a local record goes to those resolvers. The zone apex and unknown subdomains fall through to Porkbun.
|
||||
@@ -74,6 +76,8 @@ Both prerequisites survive the dnsmasq → Pi-hole swap. Do not remove them.
|
||||
| Upstream queries fail (local names still answer) | `DNS1`/`DNS2` in the env file are missing or wrong |
|
||||
| Port 53 `bind: permission denied` crash loop after a reboot | sysctl conflict: the value must be 53 at boot, and remember `/etc/sysctl.conf` is applied **after** all of `sysctl.d/`. Redeploy pihole — `deploy_pihole` reinstalls the drop-in, neutralizes conflicting drop-ins, and comments out a stale `/etc/sysctl.conf` line |
|
||||
| Single-label name (for example `dankclaw`) answers `127.0.1.1` | The rootless container `/etc/hosts` carries the host entry (`127.0.1.1 <host>`), and FTL serves it. Use the FQDN, or add an `/etc/hosts` entry on the client |
|
||||
| Every `franksdank.cloud` name is NXDOMAIN for LAN clients while external names still answer | FTL holds no local records: `getent hosts <name>` on Dankclaw comes back empty → `podman exec pihole pihole restartdns reload` (the 2026-10-19 pitfall under **Records**) |
|
||||
| One client gets nothing while every other client is fine | FTL rate-limiting that client: `podman exec pihole grep Rate-limiting /var/log/pihole/FTL.log`. A `10.0.2.x` client address is a pasta container's DNS forwarder, not a machine |
|
||||
|
||||
## See Also
|
||||
|
||||
|
||||
@@ -239,45 +239,92 @@ prefill is over. `preemptions` separates the third shape: a preempted-and-recomp
|
||||
sequence stops producing entirely and then re-prefills from token 0, which is a
|
||||
stall, not sludge.
|
||||
|
||||
Knobs, in the order worth trying them (radiance GUIDE §4.5):
|
||||
**Split the step ledger before choosing a knob.** Three counters decide which mechanism you are
|
||||
looking at, and none is a rate: `engine_step_seconds_decode_total / engine_steps_decode_total` is
|
||||
the mean **decode-only** step, and `(step_seconds_total − step_seconds_decode_total) /
|
||||
(steps_total − steps_decode_total)` is the mean **chunk step** (a chunk alone, or a chunk with
|
||||
decode rows beside it). On frank-archon's live 1.8.1 shape that pair is **15.5 ms against 166 ms**,
|
||||
with 4.7 % of the steps holding 34 % of the step time. Decode-only steps themselves move only
|
||||
14.1 → 16.4 ms between a prefill-free minute and a heavy one, and the lagged buckets stay flat: so
|
||||
a decode-rate collapse on that box is **step time sharing**, and the eviction/residency story is
|
||||
worth ~10 % of one step — not a lever.
|
||||
|
||||
- `--prefill-interleave N` — after a chunk step taken beside decoders, the next N
|
||||
steps carry decode rows only and admit nothing. Decoders keep their cadence; the
|
||||
prompt pays that in TTFT. Counts as `engine_steps_held_total`.
|
||||
- `--busy-prefill-chunk MODE` — the chunk's share of a step that has other work:
|
||||
`auto` (default) reserves a row plus the draft window per decode row behind it and
|
||||
an eighth of the budget per waiting prefill/request, `off` lets the chunk take the
|
||||
whole step (admission never runs while it does), `N` pins the share. Never under
|
||||
the prefill stager's smallest pass — the startup line names it, and below it every
|
||||
routed layer reads its pooled experts synchronously, which costs the prefill far
|
||||
more than sharing saves. Counts as `engine_prefill_chunks_capped_total`.
|
||||
- `--max-num-batched-tokens` — the blunt rung, and the only one older pins have. The vendor's own
|
||||
curve prices **halving the shared chunk at roughly a fifth of prefill throughput** (a 48-layer MoE
|
||||
at 200k on two cards — knob pricing, not this endpoint's number, so treat it as the order of
|
||||
magnitude a chunk decision costs). Raising it past the checkpoint interval buys nothing and loses
|
||||
expert residency to the arena.
|
||||
- `--prefill-span N` — the inverse trade: a *lone* prefill carries N intervals a step while every
|
||||
shared step is planned at a smaller shared budget, which on an expert-tiered box lends that arena
|
||||
back to the expert slab. The vendor's own measurement says the win is **long-prompt-only**: nothing
|
||||
at short prompts, a large TTFT gain at the far end of the window.
|
||||
**Four traps in the PromQL above.** Learn them before trusting a bucket:
|
||||
- **Latency is `vllm:`, cadence is `radiance:`.** The TTFT / inter-token / per-output-token / e2e
|
||||
histograms and the running/waiting gauges exist only under the vLLM compatibility prefix; preemption
|
||||
is `vllm:num_preemptions_total` (the counter) while `radiance:num_requests_preempted` is the
|
||||
*instant* depth. A `radiance:`-only query finds no latency and no preemption, and reads as "the
|
||||
endpoint never stalls anyone".
|
||||
- **The tps gauges are 1 s decayed windows, batch-pooled.** Per-stream rate is
|
||||
`decode_tokens_per_second / vllm:num_requests_running`. With a 15 s scrape, a 120 s step samples one
|
||||
scrape in eight — use step ≤ 30 s for a bucket table.
|
||||
- **Every counter resets at a restart and the metric set changes with the build.** Prove the window is
|
||||
one shape (`up`, or the first sample of a counter that only the current build exports) before
|
||||
ranking buckets; frank-archon's 48 h window straddled two containers, and one bucket's "collapse"
|
||||
turned out to belong entirely to the older one.
|
||||
- **A 2-minute mean step time dilutes the chunk.** The sharp variable is the phase ratio in the query
|
||||
above, never `step_seconds/steps` — 4.7 % of steps cannot move a mean that is 95 % decode.
|
||||
|
||||
Two traps. **The arena/slab coupling makes a shared step doubly expensive on an
|
||||
expert-tiered shape**: the full-plan arena takes VRAM the resident expert slab would have had, so the
|
||||
decode rows inside that step also read experts over the link — a smaller shared budget helps twice.
|
||||
And **re-check
|
||||
`prefix_hit_rate` / `prefix_evictions` after any chunk change**: the arena and the KV
|
||||
pool are the same memory, and the failure looks like turn two taking as long as turn
|
||||
one.
|
||||
Three instruments the PromQL above cannot reach:
|
||||
|
||||
Availability is a version question, not a config question: `--busy-prefill-chunk` and
|
||||
`--prefill-interleave` are absent from older pins — prove it on the running container
|
||||
(`podman exec <c> <binary> --help | grep -iE "prefill|chunk"`), never from the vendored
|
||||
tree, and check the source clone's depth before dating a flag from `git log -S`
|
||||
(a `--depth 1` clone reports its own tip for every search).
|
||||
- **The engine's own step-detail histograms (`prefill_steps_duration`, `prefill_step_tokens`,
|
||||
`token_queue_wait`) are exported but reach no target**, and the arena / expert slab / heat engine /
|
||||
prefill stager have **no metric at all**. Per-request forensics mean `curl localhost:7770/stats` on
|
||||
the endpoint's own host — same phase counters plus `preemptions` (the count), the chunk geometry,
|
||||
and the expert/link byte totals. `/diagnostics` answers "is the *box* the limit" in one call:
|
||||
configuration drift, the VRAM split, each card's link chain and clocks, findings with fixes.
|
||||
- **`prefix_hit_cut_tokens_total` measures prefill load you created for yourself.** On a hybrid
|
||||
(linear + paged) model a prefix hit reuses only as far as the last **linear-state** checkpoint, so
|
||||
cached tokens are replayed as prefill and stall decoders for it; frank-archon ran 38 % of all its
|
||||
prefill tokens that way, every `prefix_hit_cuts_total` labelled `reason="no_snapshot"`. A high
|
||||
prefill rate is not always a client that sends new tokens. Retention is `--checkpoint-policy` -
|
||||
exactly three names, `geometric-backoff` (default), `tip-only`, `keep-all`, one owner:
|
||||
[[frank-archon]].
|
||||
- **Bucket the arriving request too** — `histogram_quantile` over
|
||||
`vllm:time_to_first_token_seconds_bucket` by the same prefill series. The decode gauge only sees
|
||||
sequences that are already running; the one waiting for its first token pays the same steps and
|
||||
shows in no decode number.
|
||||
|
||||
Run on: [[frank-archon]] — the census was taken on its 3-slot radiance 1.0.13 era (budget 2048 =
|
||||
interval 2048); the live 1.8.1 shape carries the two chunk knobs. The measured census and the verdict
|
||||
for this fleet's shape are that page's.
|
||||
Knobs, in the order worth trying them (radiance GUIDE §4.5; the pinned source is the referee):
|
||||
|
||||
- `--prefill-interleave N` — after a chunk step taken beside decoders, the next N steps carry decode
|
||||
rows only and admit nothing. **The knob that acts on the measured mechanism.** Decoders keep their
|
||||
cadence; the prefilling prompt pays in TTFT and an arriving request waits out the held steps.
|
||||
Counts as `engine_steps_held_total` — if that counter is zero, nothing is protecting anyone.
|
||||
- `--busy-prefill-chunk N` — pins a shared chunk's share of the step. **`auto` is a prefill-speed
|
||||
setting, not decoder protection:** it reserves `1 + spec` rows per decoding sequence plus an eighth
|
||||
of the budget per *waiting* request or prefill, and the source refuses to halve for "decode rows
|
||||
alone", so a shape can run `auto` and still lose every chunk step. Prove engagement from
|
||||
`engine_prefill_chunks_capped_total` against the chunk-step count. The floor is the prefill
|
||||
stager's smallest pass — the largest arena level that lends, plus one, named on the startup line
|
||||
`prefill staging: … a step of more than N tokens copies the pooled experts of …` — and under it
|
||||
every routed layer reads its pooled experts synchronously: the vendor prices the whole job (four
|
||||
decoders plus a long prompt) at **29.5 s with a 2048 chunk and 31.9 s at 512**, so shrinking the
|
||||
chunk blindly buys latency in both directions.
|
||||
- `--prefill-span N` — needs `--max-num-batched-tokens` of N intervals. A *lone* prefill then
|
||||
carries N intervals a step (on an expert-tiered box only from 8 intervals, where the vendor's own
|
||||
measurement says a span starts to win), while **every other step is planned at the shared
|
||||
budget** — a lending arena level that also keeps the stager's ring, the one combination the chunk
|
||||
knob cannot express. Priced by the vendor at 320 MiB a card of expert residency, ~3 % off the KV
|
||||
pool, and TTFT-neutral on one of his two 2×R9700 boxes: measure your own prompt lengths first.
|
||||
- `--max-num-batched-tokens` alone — the blunt rung, and the only one older pins have. It raises the
|
||||
arena the expert slab loses, and on this payload its resonance is violent (MNBT 2560 collapses
|
||||
decode — [[frank-archon]]).
|
||||
|
||||
Two traps. **The arena/slab coupling makes a shared step expensive twice:** the full-plan arena is
|
||||
reclaimed from the expert slab *synchronously before the step*, and the loan regrows only after a
|
||||
quiet interval, so every step of a prefill run — the decode-only ones included — runs on the smaller
|
||||
slab. And **re-check `prefix_hit_rate` / `prefix_evictions` after any chunk or span change**: the
|
||||
arena, the staging ring and the KV pool are the same memory, and the failure looks like turn two
|
||||
taking exactly as long as turn one.
|
||||
|
||||
Availability is a version question, not a config question: `--busy-prefill-chunk`,
|
||||
`--prefill-interleave` and `--prefill-span` are absent from older pins — prove it on the running
|
||||
container (`podman exec <c> <binary> --help | grep -iE "prefill|chunk"`) or from `/diagnostics`'s
|
||||
`configuration` block, never from a vendored tree, and check a source clone's depth before dating a
|
||||
flag from `git log -S` (a `--depth 1` clone reports its own tip for every search).
|
||||
|
||||
Run on: [[frank-archon]] — owner of the live census, the measured step pair, and the arms.
|
||||
|
||||
## Related
|
||||
|
||||
|
||||
@@ -215,9 +215,10 @@ is in `render`, so a quadlet needs `AddDevice=/dev/accel/accel0` +
|
||||
`GroupAdd=keep-groups`. **No NPU endpoint is deployed** — stays the
|
||||
history of the retired endpoint; what changed is only its precondition.
|
||||
|
||||
**Deliberately deferred:** the `pp_dpm_fclk` pin (level 7, 2000 MHz) from the issue's
|
||||
co-run step. Pinning the fabric clock changes iGPU timing for every workload, and
|
||||
nothing co-runs on the NPU yet; it goes in when an NPU endpoint actually exists.
|
||||
**The `pp_dpm_fclk` pin (level 7, 2000 MHz) went in on 2026-10-19**, with the first thing that needed it: the
|
||||
vendor's root host unit, installed and enabled by `frank-dragoon`'s deployer while its NPU lane is named. Pinning
|
||||
the fabric clock changes iGPU timing for every workload, which is exactly why it waited for a co-run to exist; the
|
||||
law, the hold and the serve-side re-check live in [[frank-dragoon]].
|
||||
PerfOpt still needs Linux 7.4 — `7.0.0-38-generic` has zero `perfopt` symbols and the
|
||||
26.04 archive caps at `7.0.0-38.38`.
|
||||
|
||||
|
||||
@@ -88,25 +88,30 @@ on stop. Boot budget 1800 s (the publisher's compose allows a 15-minute `start_p
|
||||
| Knob (env name, `FRANK_ARCHON_` dropped) | Live value |
|
||||
|---|---|
|
||||
| TP / wire | `TP=2`, `TP_WIRE=wht6` — libr4d one-shot all-reduce, not RCCL; `min-kb` stays the engine default |
|
||||
| Slots / window | `MAX_SEQS=2`, `MAX_MODEL_LEN=200000` (native 262,144; longer is refused, never truncated) — this pair is the **cache bound**, not a KV allocation |
|
||||
| Slots / window | `MAX_SEQS=4`, `MAX_MODEL_LEN=200000` (native 262,144; longer is refused, never truncated) — this pair is the **cache bound**: the resolver sizes the KV carve against `slots × window`, and no slot count funds itself (Laws) |
|
||||
| KV dtype | `KV_DTYPE=fp8` — the only dtype that funds the slot count (Laws) |
|
||||
| VRAM split | **derived at boot** from `GPU_HEADROOM_MIB=96` + the cache bound + `HOST_POOL_MIB`; naming `VRAM_WEIGHTS_MIB`/`VRAM_KV_MIB` is a boot refusal (removed at 1.3.0) |
|
||||
| Expert tier | `PLACEMENT=expert_tiered`, `HOST_POOL_MIB=44928` **per rank** |
|
||||
| Expert tier | `PLACEMENT=expert_tiered`, `HOST_POOL_MIB=49152` **per rank** — the pinned host pool is also the **KV lever**: the expert floor is `plane + 2 units − host room`, so every MiB pinned there lowers what the experts claim on the card one-for-one and the cache takes it (Laws) |
|
||||
| PLE | `NGRAM_PLACEMENT=ram` (47.7 GiB pinned; refuses boot on a short memlock) |
|
||||
| Spec / batch | `SPEC_TOKENS=2` — MTP depth 2, the `.rad` carries the model's own `mtp.*` head (`mtp.fc_hidden`/`fc_embedding`/`draft_head`); `MAX_BATCHED_TOKENS=2048` (MNBT — 2560 collapses decode, Laws) |
|
||||
| Prefix cache | `PREFIX_CACHE_MIB=8192`, pinned **host tier only** — the 64 GiB disk tier is gone, so an evicted prefix is re-prefilled; `STATE_DIR=/home/dank/radiance-state` now exists only to hold the rad-tune cache |
|
||||
| Template | `TEMPLATE=scripts/qwen38-froggeric-v22.5.jinja` (28,234 B) via `--override-chat-template` — froggeric's fixed Qwen3.8 template (the repo root's `chat_template.jinja`, vendored); the checkpoint's own template is baked into the `.rad` by rad-convert and this overrides it, and radiance derives the reasoning and tool-call reply format from it |
|
||||
| Step sharing | `BUSY_PREFILL_CHUNK=auto` (a shared chunk reserves a row plus the draft window per waiting decoder) · `PREFILL_INTERLEAVE=0` — held at zero until the decode-vs-prefill buckets ask for it |
|
||||
| TTM gate | `TTM_MIN_MIB=150000`, `TTM_TARGET_MIB=204800` |
|
||||
| Step sharing | `BUSY_PREFILL_CHUNK=auto` — prefill-speed first: it reserves 3 tokens per waiting decoder and ⅛ of the budget per waiting request, and does **not** shrink the chunk for decoders alone · `PREFILL_INTERLEAVE=1` — the decoder dial, **armed 2026-10-20**: one decode-only step after each chunk step taken beside decoders; meter `engine_steps_held_total`, price is the prefilling prompt's TTFT |
|
||||
| TTM gate | `TTM_MIN_MIB=162000`, `TTM_TARGET_MIB=204800` |
|
||||
| Boot | `RAD_MIN_MIB=110000` deployer floor; the listener binds only after the full load (health ~100 s, budget 1800 s) |
|
||||
|
||||
Boot ledger for the live shape (2026-10-19 11:10, pin 1.8.1, `MAX_SEQS=2`, `HOST_POOL_MIB=44928`):
|
||||
the resolver prints `elastic 26.73 GiB` → **experts 23.73 GiB (89 %) / kv cache 2.76 GiB (10 %)**,
|
||||
`vram_weights 27.31 GiB`, `host_pool 44.47 GiB`. `kv_attn` (13 layers, 4-token blocks) pages
|
||||
**26.00 KiB** → 91,301 blocks = **365,204 tokens = 1.8 full-length sessions = 91 % of the
|
||||
2 × 200,000 worst case** (the resolver never funds 100 %, see Laws); `mover` lends 2,146 slots
|
||||
(2.54 GiB) out of the cache to the expert slab; sampler **9 rows**, template 28,234 B, vision live
|
||||
(≤ 4,194,304 px, 16,384 patches/pass).
|
||||
Boot ledger for the live shape (2026-10-20, pin 1.8.1, `MAX_SEQS=4`, `HOST_POOL_MIB=49152`):
|
||||
the resolver prints `elastic 26.73 GiB` → **experts 20.72 GiB (78 %) / kv cache 5.64 GiB (21 %)**,
|
||||
`prefill staging 373.65 MiB` out of the cache, `vram_weights 24.30 GiB`, `host_pool 48.59 GiB`.
|
||||
The referee line: `kv: the cache can grow to 5.64 GiB a card — 747,492 tokens of context, 3.7
|
||||
sessions at 200,000 — with every block it lends the experts taken back; 440.04 MiB of that is
|
||||
per-sequence state, and up to 5.20 GiB is lent to the experts while no request holds it`.
|
||||
`--checkpoint-slots 40` is auto (`max_num_seqs × 10` slots of 111.98 MiB = 4,479 MiB of pinned
|
||||
host), `/health` reports `queue_cap 32` (`max(8, seqs × 8)`), sampler **12 rows**
|
||||
(= `seqs × (1 + speculative depth)`), 4 hardware queues a card, template 28,234 B, vision live
|
||||
(≤ 4,194,304 px, 16,384 patches/pass), P2P confirmed (`the 2 cards reach each other's memory
|
||||
directly`). Experts living in the pinned host pool: **8.99 GiB a rank** (was 5.99) — the price of
|
||||
the bigger cache, paid over PCIe when those experts are hit.
|
||||
|
||||
The boot **ledger**, not this arithmetic, is the referee: radiance states every budget before
|
||||
anything is allocated (weights vs KV, the host pool's coverage of the worst case, the PLE pin,
|
||||
@@ -145,6 +150,7 @@ marked "re-fetch" is gone from `/home/dank/models/`, which today holds only `Sti
|
||||
|
||||
| Window | Payload | Runtime | Shape | Replaced because | Rollback |
|
||||
|---|---|---|---|---|---|
|
||||
| 2026-10-19 → 10-20 | same `qwen3.8-next-flash-fp8-iq4r-moe.rad` (payload unchanged) | radiance **1.8.1**, `sha256:d590e300…` (same digest) | 2 × 200,000 fp8 KV — resolver cache 2.76 GiB = 365,204 tokens (91 % backing); host pool 44,928 MiB a rank, `TTM_MIN_MIB=150000`, `queue_cap 16`, sampler 9 rows; `--prefill-interleave 1` armed on 10-20 | Frank ordered **four** concurrent conversations over a ≥ 700,000-token pool; the resolver funds that only by lowering the expert floor with more pinned host room (Laws) | revert `1bb6399` and redeploy `scripts/deploy.sh dankserver --service frank-archon` — trio back to `MAX_SEQS=2` / `HOST_POOL_MIB=44928` / `TTM_MIN_MIB=150000` |
|
||||
| 2026-10-15 → 10-19 | same `qwen3.8-next-flash-fp8-iq4r-moe.rad` (payload unchanged) | radiance **1.0.13**, `sha256:4c3f4845…` | 3 × 200,000 fp8 KV bought with the **explicit** pair `--vram-weights-mib 25600 --vram-kv-mib 5120`; host pool 44,928 → 45,568 MiB a rank; prefix cache 8 GiB host + 64 GiB disk | decode turned to sludge beside any long prompt — step contention, 18.7 ms decode-only steps against 317 ms steps carrying a prefill chunk (Laws) — and the only two knobs that fix it, `--busy-prefill-chunk` and `--prefill-interleave`, exist only after this pin | `FRANK_ARCHON_IMAGE` back to `sha256:4c3f4845…` **plus a trio rewrite**: restore the pair and the host+disk prefix knobs, delete `BUSY_PREFILL_CHUNK`/`PREFILL_INTERLEAVE`. A digest swap alone is a boot refusal (Runtime) |
|
||||
| 2026-10-08 → 10-15 | in-house `qwen4exp-swift15-w4dn8-i8-hc8m.rad`, 116,416 MiB (Swift1.5 abliterated BF16, GPTQ w4 experts + 24 bf16 protected, KLD 0.0131 / top-1 97.92 %) | radiance 1.0.13, same digest | 3 × 200,000 fp8 KV, slab 25,600 / pool 5,120 — the shape the live file still carries | the engine author's own container is the supported path and his quality ceiling; the finetune's refusal-bending left the endpoint | `.rad` purged 2026-10-17 (trash-staged then emptied, Frank's approval). Re-download `frank8465/Swift-1.5-Qwen3.8-Flash-Next-GPTQ-W4A8-Radiance-AMD-R9700` → `qwen4exp-w4nl64-i8-hc8m.rad`, 122,071,973,888 B, LFS sha256 `09dd9306…`, via the iq4r fetch script's hfget/verify pattern retargeted at that repo; point `MODEL_FILE` at it, redeploy. Re-**convert** is dead: the BF16 source was `rm`'d 2026-10-15 (`kld-ref`, grams, corpora and `/home/dank/logs/swift15_*.log` survive) |
|
||||
| 2026-10-05 → 10-08 | `qwen4exp-fp8-native.rad`, 177,109 MiB (173.0 GiB) — native modelopt block-FP8, no requantisation | radiance 1.0.7 line, same digest | — | native FP8 experts (56.4 GiB/rank) ate a 40 GiB pool and left KV at 3,543 MiB = 386,904 tokens, **1.93 sessions** — context economics, not quality; and the host pool could not be trimmed to pay for it (36,864 refused the boot) | rebuild-only: `scripts/download-frank-archon.sh` (official `Qwen/Qwen3.8-Flash-Next-FP8`, 131 shards, 172.8 GB, ungated) + `scripts/build-frank-archon-rad.sh` (no-recipe convert under the descriptor law); raw **and** `.rad` were `rm`'d 2026-10-15 (Frank's call, to free the volume for the iq4r download) |
|
||||
@@ -211,27 +217,44 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
|
||||
never `--kv-cache-size-mib`; with the disk tier engaged the engine also forced ~9.6 GiB of weight
|
||||
buffers + arena onto the weights side (~29.6 GiB/card effective). **Neither flag exists at 1.8.1:**
|
||||
the same ceiling is enforced by the resolver, and passing the pair is a boot refusal.
|
||||
- **At 1.8.1 the split belongs to the resolver, and it funds ~90 % of the worst case, never 100 %.**
|
||||
The engine takes `--gpu-headroom-mib`, the cache bound (`--max-model-len` × `--max-num-seqs`) and
|
||||
`--host-pool-mib`, prints `claimable → elastic → experts / kv cache` before allocating anything, and
|
||||
sizes the expert plane from the model's own weight footprint plus slack — not from a number typed by
|
||||
hand. Three boots of evidence:
|
||||
- **At 1.8.1 the split belongs to the resolver, and the source fixes its order.**
|
||||
`core/mem/vram_budget.cpp` claims the card bottom-up (`capacity → already held → drift → arena →
|
||||
staging → queues → prefetch → headroom = claimable`), takes the **static weights off the top as a
|
||||
floor, not a share**, and divides the `elastic` remainder in three lines (146-156):
|
||||
|
||||
| shape | demand | cache given | backing |
|
||||
```cpp
|
||||
exp = std::min(b.expert_floor, b.elastic);
|
||||
kv = std::min(kv_cap, std::max(b.elastic - exp, kv_one));
|
||||
exp = std::min(std::max(b.expert_ceiling, b.expert_floor), b.elastic - kv);
|
||||
```
|
||||
|
||||
`kv_cap` is `KVManager::ceiling()` = `state + Σ_paged max_seqs × ceil(max_ctx / block) × bytes`
|
||||
(`core/mem/kv.cpp:266`), so `--max-num-seqs` and `--max-model-len` set the cache's **demand**, never
|
||||
its funding. The funding is `elastic − expert_floor`, and the floor is
|
||||
`min(plane + spare, plane + spare + 2·unit_max − host_room)` (`core/place/planner.cpp:100-104`, with
|
||||
`host_room = host_pool − file_stage_reserve` at `core/engine_bringup.cpp:1418-1422`). **That is the
|
||||
cache's lever: every MiB added to `--host-pool-mib` drops the expert floor one-for-one and the cache
|
||||
takes it** — minus `stage`, the prefill stager's ring (two layers of the experts left off the card),
|
||||
which is charged back out of the cache whenever the cache lends (`vram_budget.cpp:193-203`). Boots of
|
||||
evidence, all on 1.8.1:
|
||||
|
||||
| shape | demand | cache given | engine's `backs %` |
|
||||
|---|---|---|---|
|
||||
| 3 × 200,000, host pool 44,928 | 600,000 tok | 4.20 GiB | 93 % (2.8 sessions) |
|
||||
| 3 × 200,000, host pool 44,928 | 600,000 tok | 4.20 GiB | 93 % |
|
||||
| 3 × 200,000, host pool 45,568 | 600,000 tok | **4.20 GiB — byte-identical** | 93 % |
|
||||
| 2 × 200,000, host pool 44,928 (live) | 400,000 tok | 2.76 GiB = 365,204 tok | 91 % (1.8 sessions) |
|
||||
| 2 × 200,000, host pool 44,928 | 400,000 tok | 2.76 GiB | 91 % |
|
||||
| 4 × 200,000, host pool 49,152 (live) | 800,000 tok | **5.64 GiB = 747,492 tok** | 93 % |
|
||||
|
||||
So `--max-num-seqs` is a **demand signal** for the resolver and an admission cap for the scheduler —
|
||||
never a KV allocation — and no slot count fully funds the worst case. Growing host RAM by less than a
|
||||
slab step (640 MiB) moved nothing: the expert plane's claim decides what is left for KV, so cache is
|
||||
grown through slots × window or the dtype, and experts are grown by not leaving elastic pool idle.
|
||||
`--host-pool-mib` is still load-bearing the other way — it is where the expert rows the cards cannot
|
||||
hold get pinned, and under-sizing it has always been a boot failure (36,864 refused a boot in the fp8
|
||||
era). The two-slot boot's own behavioural check: three concurrent conversations all answer `200` —
|
||||
the third **queues** (`queue_cap: 16`), it is not refused; the 400 class is a single request over
|
||||
`--max-model-len`, or three concurrent full-height conversations against a 1.8-session cache.
|
||||
The 45,568 row is the ceiling case, not a mystery: at 3 slots the cache already held its whole
|
||||
`ceiling`, so extra host room could not grow it and shows up as `unclaimed`. **The referee is the
|
||||
ledger line `kv: the pool backs N % … the cache can grow to … N tokens`, never arithmetic.** To buy
|
||||
window: raise `--host-pool-mib` (cost: pinned host RAM, and experts evicted to host cross PCIe —
|
||||
`experts-from-host` was 5.99 GiB a rank at 44,928 and is 8.99 at 49,152) or cut the per-token cost
|
||||
(`fp8` KV). Raising slots raises demand **and** costs 10 × 111.98 MiB of pinned checkpoint-store RAM
|
||||
per slot. Under-sizing the pool the other way has always been a boot failure (36,864 refused a boot in
|
||||
the fp8 era: the host cannot hold what the cards will not). Behavioural check on the live shape:
|
||||
`/health` reports `queue_cap 32`; a request beyond the slot count queues rather than being refused,
|
||||
and the 400 class is a single request over `--max-model-len`.
|
||||
- **Idle KV VRAM is the expert cache's spillover — the price is paid twice.** `mover: 2275
|
||||
slots can be lent out of the KV cache (2.69 GiB)`, `arena: lends 740 MiB while steps are <=
|
||||
256 tokens`; the lend grows with the pool (2.69 GiB from 3.46, 6.08 from 6.912, 4.22 in the pair era)
|
||||
@@ -280,41 +303,104 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
|
||||
- **MNBT 2048.** 2560 collapses decode to 2.6-7.7 t/s (MTP resonance); the w4 file tier streams
|
||||
in place only at MNBT ≤ 2048 and the activation arena scales with the chunk (32768 → 33.3
|
||||
GiB, refused).
|
||||
- **Decode pays for the chunk it shares a step with.** Census of the live engine taken at the
|
||||
3-slot shape on 1.0.13 (`/stats` + `/metrics`, cumulative over the 2026-10-15 boot):
|
||||
1,435,585 decode-only steps at **18.7 ms** each against 127,991 prefill+mixed steps
|
||||
at **317 ms** — 60 % of engine wall-clock spent inside 8 % of the steps, because the
|
||||
chunk is `min(remaining, checkpoint interval 2048, MNBT 2048)` = 2048 rows and every
|
||||
decode row in that step waits the whole chunk. Day of Prometheus (120 s step, 721
|
||||
samples): `decode_tokens_per_second` median **13.3 t/s** in the 56 % of samples where
|
||||
`prefill_tokens_per_second` ≥ 1000, against 106-240 t/s in the light-prefill buckets;
|
||||
bucketed by mean step time it is 128.7 t/s at 0.02-0.05 s steps and 12.0 t/s at
|
||||
0.15-0.4 s. The stall is **during**, not after — decode in windows 2-14 min after a
|
||||
busy-prefill window sits at the idle baseline (58-64 vs 77-85 t/s, the residual
|
||||
confounded by co-scheduling), so no lingering expert-residency penalty at 2-minute
|
||||
resolution, and `preemptions: 0` rules out preempt-and-re-prefill. Method:
|
||||
[[LLM Performance Diagnosis]] §9. The law is shape-independent; the fix lives at 1.8.1 (next law),
|
||||
and nobody has re-run the census since the migration — so the fix is ordered, **not verified**.
|
||||
- **The knobs for it arrived with 1.8.1, and only one of them is on.** `--busy-prefill-chunk` caps the
|
||||
shared chunk so a step with waiting decoders takes a smaller piece (live: `auto`, the engine picks;
|
||||
the explicit value is 512), and `--prefill-interleave N` inserts N decode-only steps after a chunk
|
||||
step taken beside decoders. `interleave` is deliberately **0** — the env comment holds it there until
|
||||
the decode-vs-prefill buckets ask for it, and those buckets have not been read since the migration.
|
||||
The vendor prices a 1024 chunk at about a fifth off prefill throughput to halve the stall; that is a
|
||||
knob for choosing tomorrow's chunk, not a measurement of this shape. MNBT remains an arm and not an
|
||||
edit: 2560 collapses decode through MTP resonance. Any chunk change re-scores draft acceptance before
|
||||
it is trusted.
|
||||
- **Slots cost KV, rows, and window — never speed.** `--max-num-seqs` carries no radiance constraint: it
|
||||
adds ≈110 MiB of recurrent state per slot (the live `kv_full_backing` arithmetic prices them exactly),
|
||||
declares the fused TP all-reduce over `seqs × (1 + spec)` rows, and `wht6` is bit-exact up to 25 rows
|
||||
(single-sequence decode and any step of ≤ 25 rows is exact; only busier steps are lossy) — 2 × 3 = 6
|
||||
today. Measured 2026-10-06: seqs 2 and seqs 3 are statistically tied (30.9-33.1 t/s, inside the ±3 %
|
||||
floor). Measured 2026-10-19 at 1.8.1: the third slot's real price is **KV, not rows** — dropping it
|
||||
moved the resolver's cache from 4.20 GiB to 2.76 GiB and the backing from 2.8 to 1.8 sessions, while
|
||||
the ≈4 GiB of host RAM it freed came back unbudgeted, landing in elastic pool that turned into a bigger
|
||||
mover lend instead of cache (resolver law above). The old "seqs 2 cannot boot" law belonged to the
|
||||
retired tcclaviger fork's `torch.compile` shape specialization, and radiance booted seqs 2 on its first
|
||||
fp8-native start.
|
||||
- **Decode pays for the chunk it shares a step with, and `--busy-prefill-chunk auto` does not
|
||||
protect it.** Re-measured on the live 1.8.1 shape (34.8 h boot, Prometheus at 60 s,
|
||||
`engine_step_seconds{,_decode}_total` / `engine_steps{,_decode,_mixed,_prefill}_total` /
|
||||
`prefill_tokens_total`): decode-only steps run at **15.5 ms**, chunk steps (a chunk alone, or a
|
||||
chunk with decode rows beside it) at **166 ms** — 4.7 % of the steps hold **34 % of the engine's
|
||||
step time**, so a session decoding while another prompts advances one token per 166 ms. Bucket the
|
||||
harness's `radiance:decode_tokens_per_second` by `radiance:prefill_tokens_per_second` (15 s
|
||||
scrapes; per-stream rate = the gauge ÷ `vllm:num_requests_running`): median **166 t/s** under light
|
||||
prefill, 81 t/s at 1-5k, **13 t/s above 5k** — a **12.7×** collapse, and at 13 t/s with two
|
||||
sessions running each stream sits at 6.7 t/s = 149 ms a token, one token per shared step. Nothing
|
||||
else moves: decode-*only* steps stay 14-18 ms in every bucket (14.1 ms in a prefill-free minute,
|
||||
17.9 ms in the deep one) and the relationship is lag-0 only (r = −0.74 at lag 0, |r| ≤ 0.19 from
|
||||
30 s on, decode back at baseline by 5 min) — residency is not the mechanism either. The second
|
||||
victim is the **arriving** request: through a high-prefill window TTFT p90 goes 2.4 → **18.0 s**
|
||||
and 15.5 % of requests wait over 10 s for their first token, with `http_queue_depth` at 2-3, while
|
||||
median TTFT (~1.9 s) and per-token latency barely move. Preemption is real but burst-shaped: 264
|
||||
`vllm:num_preemptions_total` in the boot, 108 of them inside 105 s of a 6.5k-tok/s prefill. The
|
||||
chunk is `min(remaining, checkpoint interval, MNBT, busy_cap)`; the first three read 2048 here,
|
||||
and `busy_cap` is the fourth law.
|
||||
- **`auto` is a prefill-speed dial, not a decoder-protection dial.** `Scheduler::busy_cap` reserves
|
||||
`1 + spec` rows per decoding sequence behind the chunk (3 tokens at MTP 2) plus one eighth of the
|
||||
budget per *waiting request or waiting prefill*, and the code refuses to do more for decoders:
|
||||
"not for decode rows alone: their reserve is a few tokens, so the chunk ends a quantum short and
|
||||
the next step finishes the interval… Halving would hold a long prompt to half-interval chunks while
|
||||
anything decodes behind it, which costs more" (`core/sched/scheduler.cpp:874-878`, 1.8.1). Live
|
||||
proof: `prefill_chunks_capped` 5,018 against 62,386 chunk steps — 8 %, and those are the minutes
|
||||
with a waiter, not the minutes with a decoder. An explicit `N` is the only value that applies
|
||||
"whenever the chunk shares the step".
|
||||
- **At MNBT 2048 no chunk value keeps both the arena loan and the staged copies.** The arena is a
|
||||
ladder of levels (256 and 1024 rows here) and a step's rows are its **whole** token count —
|
||||
`max(n_tok, n_seq × (1 + spec))`, `core/engine.cpp:1142` — so a 1024-token chunk with two decode
|
||||
rows is a 1030-row step: it falls back to the full plan, which "lends nothing", and it arms the
|
||||
prefill stager. The stager's floor is the largest lending level plus one, 1025 rows here — named on
|
||||
the boot line `prefill staging: … a step of more than 1024 tokens copies the pooled experts of 48
|
||||
routed layers onto the card up to 3 layers ahead` — and below it "every routed layer reads its
|
||||
pooled experts synchronously and the prefill slows far more than sharing saves" (GUIDE §4.5). The
|
||||
vendor's own number for naive shrinking, on this architecture: four decoders with a long prompt
|
||||
beside them finished in **29.5 s at a 2048 chunk and 31.9 s at 512**.
|
||||
- **A big step takes the arena from the expert slab and keeps it for the whole prefill.** The arena
|
||||
lends 987.77 MiB (steps ≤ 256 rows) or 670.26 MiB (≤ 1024) to the slab; a full-plan step reclaims
|
||||
it **synchronously before the step** — units revert to their host copies (an address change, not a
|
||||
copy) and both read streams drain — and `arena_tick` regrows the loan only **2 s after the last big
|
||||
step** (`core/engine.cpp:1129-1202`), so every step of the prefill run, decode-only included, runs
|
||||
with a slab ~780 expert units smaller over a link also carrying the staged copies. `/diagnostics`
|
||||
says the binding constraint in one line: "a prefill step streams every routed layer's pooled
|
||||
experts onto the card, so its speed is bounded by the link; decode reads the ones it routes to".
|
||||
Measured size of the whole effect: the ~10 % above. Real, small, not worth chasing.
|
||||
- **`--prefill-interleave` is the dial that buys decode its cadence; armed at 1 on 2026-10-20.**
|
||||
N decode-only steps follow any chunk step taken beside decoders; held steps admit nothing, so it
|
||||
costs the prefilling prompt its TTFT and an arriving request its first step. Rate arithmetic at the
|
||||
two measured step times and 0.657 acceptance (≈2.3 tokens a step): N=0 → ~14 t/s, **N=1 (armed) →
|
||||
~23**, N=2 → ~33, N=4 → ~48, N=8 → ~69, with the prefill slowed by (166 + 15.5N)/166. **Arithmetic, not
|
||||
measurement** — the arm is chosen by the §9 bucket pair plus `steps_held`, and `auto` is not a
|
||||
substitute for it.
|
||||
- **`--prefill-span` is the shape that keeps the loan, and it is priced.** It needs MNBT = N
|
||||
intervals, so `--prefill-span 2 --max-num-batched-tokens 4096`: a *lone* prefill carries 2
|
||||
intervals a step (and on an expert-tiered box only from 8 intervals / 16k tokens, where the span
|
||||
starts to win), while **every other step is planned at the shared budget of 2048** — which is a
|
||||
lending level *and* keeps the stager's ring, the one combination `--busy-prefill-chunk` cannot
|
||||
express. Costs per GUIDE §4.5: 320 MiB a card of expert residency, ~3 % off the KV pool (§4.3's
|
||||
cliff is sharp), a decode step uploads twice the per-step arrays at 4096, and checkpoints inside a
|
||||
span are not written, so a partially-shared prompt resumes one interval earlier. On the vendor's
|
||||
second box (2×R9700, w4nl64, TP2) it was TTFT-neutral — 2.61 vs 2.56 s at 18k, 10.72 vs 10.55 at
|
||||
75k — hence their own rule: measure your own prompts at two ranks before turning it on. MNBT 4096
|
||||
does not boot at 3 slots; this shape is 2.
|
||||
- **Most of this endpoint's prefill work is replay of tokens the prefix cache already had.** Over
|
||||
the same window `prefill_tokens_total` 65.2 M against `prefix_hit_cut_tokens_total` 24.9 M = **38 %
|
||||
replayed**, and it is not an eviction: a hit on a hybrid model reuses only as far as the last
|
||||
**linear-state** checkpoint (`reuse = min(h.n_attn_tokens, h.n_linear_tokens)`,
|
||||
`core/sched/scheduler.cpp:953-978`), and the retention default keeps a **geometric ladder** - about
|
||||
`log2(depth/interval) + 2` snapshots - so a turn that diverges `d` behind the tip can resume as
|
||||
deep as `tip − 3d`. The live ladder for a 200k session at interval 2048 is 2048 · 131072 · 163840
|
||||
· … · 198656: diverge at 120k and 21.7k tokens replay, diverge at 50k and 48k. Every
|
||||
`prefix_hit_cuts_total` carries `reason="no_snapshot"` (the other two reasons stay dead), so the
|
||||
cause object agrees and `hit_cut_tokens` is the cost. `--checkpoint-policy` admits exactly
|
||||
**`geometric-backoff` | `tip-only` | `keep-all`** and refuses anything else at boot. It governs
|
||||
retention only - the writes happen at every interval whatever the policy says (only `--prefill-span`
|
||||
skips them) - so `keep-all` buys a replay bounded by one interval at any depth for **pinned host RAM
|
||||
alone**: 97 snapshots a 200k session, `--checkpoint-slots auto` asking 196 × 111.98 MiB = 21.4 GiB
|
||||
against a store budget of an eighth of host RAM (31.5 GiB here, so `share` admits it) with `room` -
|
||||
the pinning room left beside the expert pool at boot - as the hard bound the ledger line names.
|
||||
Retention also spares a snapshot a later turn resumed from (`core/mem/prefix.cpp:709-723`), so a
|
||||
chain that keeps resuming at one deep point stops paying for it. Shortening
|
||||
`--checkpoint-interval` does **not** buy the deep case: the coarsest ladder scale is the largest
|
||||
`interval × 2^k` under the depth, so the deepest gap stays ≈ half the prompt (129,024 tokens at
|
||||
interval 2048 against 130,816 at 256) while every cut multiplies the snapshot writes.
|
||||
- **Slots cost KV, host RAM, rows, and pin — never speed.** `--max-num-seqs` carries no radiance
|
||||
constraint: it adds ≈110 MiB of recurrent state per slot, 10 × 111.98 MiB of pinned checkpoint-store
|
||||
RAM per slot (`--checkpoint-slots auto`), and declares the fused TP all-reduce over
|
||||
`seqs × (1 + spec)` rows — `wht6` is bit-exact up to 25 rows, and the live shape is 4 × 3 = **12**, so
|
||||
single-sequence decode stays exact and so does any step up to that many rows. Raising it also raises
|
||||
the cache's *demand* (`ceiling = slots × window`), which the resolver funds from `elastic −
|
||||
expert_floor` — so a slot count without matching host room is paid for by expert residency. Measured
|
||||
2026-10-06: seqs 2 and 3 statistically tied (30.9-33.1 t/s, inside the ±3 % floor). Measured
|
||||
2026-10-19 → 10-20 at 1.8.1: 2 → 4 slots cost **+4,224 MiB of pinned host pool** and **−3.01 GiB of
|
||||
expert residency a card** and bought 2.76 → 5.64 GiB of cache (365,204 → 747,492 tokens). The old
|
||||
"seqs 2 cannot boot" law belonged to the retired tcclaviger fork's `torch.compile` shape
|
||||
specialization, and radiance booted seqs 2 on its first fp8-native start.
|
||||
- **A dequantize-to-compute path bills host RAM twice.** On the EMULATION backend an offload request
|
||||
of **118 GiB** landed as **~238 GiB** of real host RAM — the dequantized copy rides on top of the
|
||||
pinned weights, and the doubling is invisible in the weight count. Read the engine log for
|
||||
@@ -396,12 +482,35 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
|
||||
4600 MT/s ended it, with zero `gdn_hip` faults either side; the prefix-caching / GDN re-seed
|
||||
theory was confounded on both arms. Consumer non-ECC raises no MCE for a silent flip and a
|
||||
60 s / 8 GiB pattern check covers 0.5 % of 246 GiB — silent RAM corruption outranks every
|
||||
software theory.
|
||||
software theory. **That mitigation is a BIOS setting and it is not in effect now:** `sudo
|
||||
dmidecode -t memory | grep -i configured` answers 4800 MT/s on 2026-10-20, and the only event
|
||||
between the fix and today is the 2026-10-12 power failure that also reset TTM. Check the speed
|
||||
before crediting any software revert with a fix — and before believing a clean run of the arm
|
||||
you just tested, because the confounder is still in the slot.
|
||||
- **TP=2 here** is legal since the 2026-09-20 ProArt B850 swap (x8/x8 on CPU root ports,
|
||||
`Routing+`/`ReqEn+` end to end) only because radiance's wire is `libr4d`'s own one-shot
|
||||
all-reduce; RCCL needs atomics on the whole chain and fails `hipErrorIllegalState` without
|
||||
them (). `wht6` buys +5.6-6.3 % prefill for ~1.3 % of 8-stream
|
||||
decode, `exact` is the bit-exact alternative.
|
||||
- **The link is already at the board's ceiling — do not chase the `link-below-max` warning.**
|
||||
Both root ports (`00:01.1`, `00:01.3`) are **x8 max** at 32 GT/s, so PCIe 5.0 x8 ≈ 31.5 GB/s is
|
||||
the whole budget per card and the cards' own `pcie_dpm` tables expose only x8 states.
|
||||
`/diagnostics` still warns that the on-card switch hop (`01:00.0`, `05:00.0`) runs "x8 of its
|
||||
x16 while the card is 100 % busy" and suggests a reboot or a reseat — pointless when the host
|
||||
side of the chain is x8 max, which the same endpoint reports two findings later as
|
||||
`link-capped`. The width that matters is the minimum of the chain, and it is x8 either way, so
|
||||
the only lever on a link-bound prefill is **less traffic over the link** (more expert
|
||||
residency, fewer pooled experts), never a wider link. P2P resolves **on** (`auto` asks the
|
||||
driver, the two cards reach each other's memory), so the collectives do not hairpin through host
|
||||
RAM, and the engine's copies ride one DMA queue that "give[s] way to the collectives" — the
|
||||
all-reduce wins the link, the staged experts wait.
|
||||
- **Eight cores is the whole host, media stack included.** Dankserver is a Ryzen 7 7700X (8
|
||||
cores / 8 threads, one NUMA node, `nodes: -1` for both cards) running Jellyfin, Plex, Calibre
|
||||
and RomM beside the engine, and `--http-threads` defaults to one thread per core. Measured
|
||||
during a heavy prefill: load average ≈ 1.0, `jellyfin` 11 % CPU, `radiance` 7 % — **CPU
|
||||
contention is not the current decode culprit**, so do not spend an experiment window on it;
|
||||
re-read `uptime` and `ps -eo pcpu,comm --sort=-pcpu | head` first if a future collapse looks
|
||||
different from the step-sharing one.
|
||||
- `--prefix-cache-dir` creation is **not recursive** — the serve script `mkdir -p`s the path or
|
||||
the disk tier silently never opens. The tier does survive a payload swap when arch and
|
||||
tokenizer match (it recovered 900,176 blocks on the 2026-10-15 swap; `cache_n` proves a hit).
|
||||
@@ -445,11 +554,19 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
|
||||
## Contract (the wire)
|
||||
|
||||
- **Live surface:** `/health` (== `/ping`), `/v1/models`, `/v1/chat/completions`
|
||||
`/v1/completions`, `/metrics`, `/stats`, `/sessions`, `/dashboard`. `/health` answers 200
|
||||
JSON (model id, `in_flight`, `queue_depth`) while serving and **503 only while stopping**
|
||||
(`core/server/admin.cpp:385`), and the listener binds *after* the full load (`main.cpp`:
|
||||
`engine.start()` then `engine.serve()`) — during the load the port refuses connections, so a
|
||||
bare `curl -fsS …/health` gate is honest and needs no body grep.
|
||||
`/v1/completions`, `/metrics`, `/stats`, `/sessions`, `/dashboard`, **`/diagnostics`**. `/health`
|
||||
answers 200 JSON (model id, `in_flight`, `queue_depth`) while serving and **503 only while
|
||||
stopping** (`core/server/admin.cpp:385`), and the listener binds *after* the full load
|
||||
(`main.cpp`: `engine.start()` then `engine.serve()`) — during the load the port refuses
|
||||
connections, so a bare `curl -fsS …/health` gate is honest and needs no body grep.
|
||||
- **`/diagnostics` is the instrument for "is the box, not the config, the limit"** — one call
|
||||
returns `configuration` (every flag, so drift from the env file is checkable without the host),
|
||||
`environment`, `derived` (the VRAM split as the ledger prints it), `hardware` (host, per-card
|
||||
`link_chain` + `pcie_dpm` + clocks + power cap + DMA engine choice + P2P answer) and `findings`
|
||||
(level / what / why / fix). On this box it reports `experts-from-host` (5.99 GiB a rank crosses
|
||||
PCIe when used), `link-capped` (root ports are x8 max — the board), `swap-in-use`,
|
||||
`tp-wire-lossy`, and **`link-below-max` as a `warn` at 100 % card busy on both cards** — which is
|
||||
structural on this board; see the x8-ceiling law under *This host, and the deploy path*.
|
||||
- **Served id.** `FRANK_ARCHON_SERVED_NAME=frank-archon` (the wire rename of 2026-10-09 moved
|
||||
only the alias; container, unit, scripts and env keep the service name). The env's contract
|
||||
line is that **radiance validates the requested model name, so every client — harness, bench
|
||||
@@ -476,12 +593,28 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10).
|
||||
and 16,384 patches per pass. Probe with a ≥ 256 px solid-colour PNG and a ≥ 256 token budget
|
||||
— at 64 tokens you get empty `content` and the colour only inside `reasoning_content`.
|
||||
- **Capacity behaviour:** a prompt over `--max-model-len` is refused, never truncated; a request
|
||||
longer than the KV pool's token count is accepted then refused as too large to fit; under the
|
||||
3-slot contract a **fourth** full-height conversation is refused, not paged or squeezed.
|
||||
longer than the KV pool's token count is accepted then refused as too large to fit; a request
|
||||
beyond `--max-num-seqs` is held by the HTTP port (`queue_cap` = `max(8, 8 × slots)` — **32** here)
|
||||
and answered 503 only past that, never paged or squeezed. The cache funds **3.7** full-height
|
||||
sessions, so four conversations can be resident but a fourth near-full-height one can still
|
||||
**preempt a running sequence from the back** to free its blocks
|
||||
(`core/sched/scheduler.cpp:1291-1322`); the victim is recomputed, so its session halts and then
|
||||
re-prefills. Meter: `vllm:num_preemptions_total` (cumulative for the boot, `/stats preemptions`);
|
||||
`radiance:num_requests_preempted` is only the instant depth.
|
||||
- **Observability:** `/metrics` carries vLLM-compatible names plus `radiance:*`, scraped by the
|
||||
Dankclaw Prometheus job at `192.168.0.235:7770` over plain LAN routing (the netavark
|
||||
published-port quirk is same-host only). The boot ledger — the referee for every shape number
|
||||
above — is in the container log (`podman logs frank-archon`)
|
||||
Dankclaw Prometheus job at `192.168.0.235:7770` (15 s interval) over plain LAN routing (the
|
||||
netavark published-port quirk is same-host only). **Latency lives under `vllm:`, cadence under
|
||||
`radiance:`** — TTFT / inter-token / per-output-token / e2e / decode-time histograms and the
|
||||
running/waiting gauges are `vllm:*` with no radiance-native equivalent, so a `radiance:`-only
|
||||
query misses every latency answer. The step-phase counters are all scraped:
|
||||
`engine_steps{,_decode,_mixed,_prefill,_held}_total`, `engine_step_seconds{,_decode}_total`,
|
||||
`engine_prefill_chunks_capped_total`, `prefill_tokens_total`, `decode_tokens_per_second`,
|
||||
`prefill_tokens_per_second`, `prefix_hit_cut_tokens_total`, `http_queue_depth`. Two holes: the
|
||||
engine's own step-detail histograms (`prefill_steps_duration`, `prefill_step_tokens`,
|
||||
`token_queue_wait`) are exported but reach no target, and the **arena / expert slab / heat engine /
|
||||
prefill stager have no metric at all** — that half of every law above is read from the boot ledger,
|
||||
`/stats` and `/diagnostics`, never from a graph. The boot ledger — the referee for every shape
|
||||
number here — is in the container log (`podman logs frank-archon`);
|
||||
`/home/dank/logs/frank_archon.log` is where the serve script redirects `podman run`.
|
||||
|
||||
## Files
|
||||
|
||||
@@ -10,7 +10,7 @@ tags:
|
||||
- gfx1151
|
||||
- halogen
|
||||
- endpoint
|
||||
timestamp: 2026-10-18
|
||||
timestamp: 2026-10-19
|
||||
---
|
||||
|
||||
# frank-dragoon — halogen-flash-server 0.16.2, native v2 abliterated (Dankclaw iGPU)
|
||||
@@ -219,7 +219,7 @@ created it — earlier topologies in .
|
||||
- Deploy waits for :7770 to be free: a rootless container binds it at CREATE and exits 126, and a
|
||||
leftover Quadlet beside the root unit guarantees the race.
|
||||
|
||||
## The NPU lane (present in the image, not enabled)
|
||||
## The NPU lane (XDNA2, armed by one env var)
|
||||
|
||||
XDNA2 executes only vendor-precompiled AIE partition programs — that is the wall the TTS attempt died at
|
||||
([[Frank TTS]]) — and flash-server ships those programs. `HALOGEN_NPU_MODELS=<names>` starts a second engine **in the
|
||||
@@ -235,11 +235,19 @@ Embedding contract on that lane: `POST /v1/embeddings`, `model: "qwen3-embedding
|
||||
(`Instruct: …\nQuery:…`), `base64` accepted, and LangChain's `OpenAIEmbeddings` needs
|
||||
`check_embedding_ctx_length=False`.
|
||||
|
||||
Four prerequisites the trio does not satisfy today, from the vendor's own host list — and this box is one of the two
|
||||
configurations they test (Ubuntu 26.04's XRT 2.21 `libxrt2`+`libxrt-npu2`, the kernel's `amdxdna`, `iommu=pt`):
|
||||
Four host prerequisites, each refused **by name** in the serve script and mirrored in `deploy_llm_dragoon` — a
|
||||
precondition that surfaces as an engine crash-loop is a gate that was never written. All four are satisfied and the
|
||||
lane is **armed and serving** since 2026-10-19 (`HALOGEN_NPU_MODELS=qwen3-embedding-0.6b`):
|
||||
|
||||
1. `--device /dev/accel/accel0` plus `--group-add keep-groups` (the pattern ComfyUI uses for `/dev/kfd`; `dank` is
|
||||
already in `render`).
|
||||
1. `--device /dev/accel/accel0` **and** a node the container's namespace can open. `/dev/accel/accel0` is
|
||||
`root:render 0660` and a rootless namespace maps exactly one uid/gid, so the engine's own first words are
|
||||
*"this process cannot open it (crw-rw---- nobody:nogroup)"*. `--group-add keep-groups` — the vendor's advice and
|
||||
the ComfyUI pattern for `/dev/kfd` — is **dead in this shape**: measured 2026-10-19 from a `dank` shell already
|
||||
in `render`, with and without the flag the node stayed `nobody:nogroup` and the open was refused; an unmapped
|
||||
gid cannot be granted across a namespace that does not map it. The door is the udev mode
|
||||
(`dankclaw/udev/98-halogen-npu-access.rules`, installed and re-triggered by the deployer, `0666` like the GPU
|
||||
compute nodes beside it); the serve script refuses any node that is not other-accessible, so the fault cannot
|
||||
return as a crash-loop.
|
||||
2. The host's XRT reachable inside: there is **no `/opt/xilinx/xrt`** here — distro packages put the three files the
|
||||
container needs (`libxrt_coreutil.so.2`, `libxrt_core.so.2`, `libxrt_driver_xdna.so.2`) in
|
||||
`/usr/lib/x86_64-linux-gnu`, where all three exist.
|
||||
@@ -248,17 +256,27 @@ configurations they test (Ubuntu 26.04's XRT 2.21 `libxrt2`+`libxrt-npu2`, the k
|
||||
engine holds the clock itself only when it runs as root with `/sys` mounted; `frank-dragoon` is a **rootless**
|
||||
`podman run` under `User=dank`, so the path is the vendor's root host unit (`deploy/host/halogen-fabric-clock`) —
|
||||
which is exactly the `pp_dpm_fclk` level-7 pin deferred in [[dankclaw-iommu-translation-off]] with the words "it
|
||||
goes in when an NPU endpoint actually exists". Held: about a watt. Unheld: the hang. Measured 2026-10-19 nothing
|
||||
is held (fabric DPM lists 400–2000 MHz with no forced level, graphics clock sitting at 857 of 2900 MHz).
|
||||
4. `HALOGEN_DOWNLOAD=peonist-ai/halogen-qwen3.8-flash-next` on a boot with a **writable** `/models/npu` — the payload
|
||||
mount is `:ro` by law, so the programs land via a throwaway writable run or a dedicated read-write bind for that
|
||||
one subdirectory; the vendor notes a read-only volume works but re-runs the conversion at every start.
|
||||
goes in when an NPU endpoint actually exists". Held: about a watt. Unheld: the hang. The deployer installs and
|
||||
enables that unit whenever the lane is named, and `halogen-fabric-clock status` reports `held (manual)` here as of
|
||||
2026-10-19; the serve script re-reads the same two sysfs files at every start and refuses by name if the top level
|
||||
is not the starred one, because a hold done by hand is a countdown to the next reboot.
|
||||
4. The programs are staged under `$FRANK_DRAGOON_NPU_DIR`, which **must sit inside the payload dir**
|
||||
(`/home/dank/models/halogen-v2-abliterated/npu`): the payload's read-only mount is the container's only handle on
|
||||
`/models`. A nested bind at `/models/npu` cannot work — the image ships no `/models`, so runc has to `mkdir` the
|
||||
mountpoint through a read-only parent and dies at container init (measured 2026-10-19: 52 crash-loop restarts with
|
||||
chat down before the mount shape was fixed). `HALOGEN_NPU_DIR` is no escape; the shipped entrypoint rejects it by
|
||||
name ("our NPU models live under `/models/npu/<id>/`"). Nothing downloads at boot — `HALOGEN_DOWNLOAD` stays
|
||||
unset, so the container opens no outbound connection; `scripts/download-frank-dragoon-npu.sh` pins every file by
|
||||
size and sha256 against the catalog and the payload mount carries the tree in.
|
||||
|
||||
Price: the programs and their weights sit in host RAM beside the 62.9 GiB of locked weights ("a few GB in all" for
|
||||
the default set; ~1.1 GiB for `embeddinggemma-2`), and **the Flash model decodes slower while the NPU works** — same
|
||||
memory bus, same power budget. Under this page's RAM law the question is pool versus n-gram table, so read the boot
|
||||
ledger after the first NPU boot instead of modelling it. If the lane is opened, prefer **0.18.8 or newer**: that
|
||||
release budgets NPU memory before asking for it, fixing kernel errors from several NPU models under a concurrent mix
|
||||
Armed, the boot ledger reads (`/home/dank/logs/frank_dragoon.log`, 2026-10-19): the GPU fabric clock is held at its top level (2000 MHz) **before** the NPU starts; `/dev/accel/accel0` is `crw-rw-rw- nobody:nogroup` under driver `amdxdna` with the distro XRT and its NPU plugin; the program arrives as `15 files checked against this image's record`; then `npu_api: qwen3-embedding-0.6b (embeddings) on the NPU engine at 127.0.0.1:8740 (buckets u0…u9; inputs up to 4096 tokens)` and `up to 64 passes queued … up to 7200 s`. Callers never see `:8740` — the lane answers on the same `:8731` as the Flash model and a request picks the engine by its `model` field, so `/v1/models` lists `frank-dragoon` **and** `qwen3-embedding-0.6b`. The fleet's consumer is open-notebook's embedding default ([[Open Notebook]]); that page owns the app-side shapes (the URL guard, defaults-by-id, the `/api/embed` 500 that still writes the vector).
|
||||
|
||||
Price: the programs and their weights sit in host RAM beside the locked weights ("a few GB in all" for the default
|
||||
set; ~1.1 GiB for `embeddinggemma-2`), and **the Flash model decodes slower while the NPU works** — same memory bus,
|
||||
same power budget. Measured 2026-10-19 with one program armed and the endpoint idle, the whole container holds
|
||||
**1.55 GiB** of host RSS (`podman stats`), so this lane is not what the RAM law above is negotiating against; a
|
||||
multi-model set is where that question returns. If the lane is widened, prefer **0.18.8 or newer**: that release
|
||||
budgets NPU memory before asking for it, fixing kernel errors from several NPU models under a concurrent mix
|
||||
**on the stock kernel driver** — the driver this box runs.
|
||||
|
||||
## Files
|
||||
|
||||
@@ -130,7 +130,10 @@ is a locator: one row, one clause, no numbers.
|
||||
|---|---|
|
||||
| [[Agent Docs and Edits]] | Two self-inflicted doc damage classes from parallel agent edits, with the audit checks and recovery order for each. |
|
||||
| [[Command Center Dotfiles]] | How the dotfiles repo deploys: stow-equivalent packages, full-replica rsync, the pre-overwrite audit, transport gotchas. |
|
||||
| [[Domain Docs]] | Consumer rules for GLOSSARY.md and docs/adr/, and where this vault out-ranks them on operational facts. |
|
||||
| [[Issue Tracker]] | How agent skills read and write Gitea issues here: credential-safe curl, label IDs, and wayfinding without sub-issues. |
|
||||
| [[OMP Harness]] | The omp harness as configured here: compaction chain arithmetic, per-model sampler routing, and the context-lean audits. |
|
||||
| [[Triage Labels]] | The five canonical triage roles mapped to this repo's Gitea label strings, plus the numeric-ID quirk. |
|
||||
|
||||
## Research and methodology
|
||||
|
||||
|
||||
+18
-15
@@ -1,22 +1,34 @@
|
||||
// Alloy configuration — ships host and container logs to Loki.
|
||||
// One instance per host. Deployed to /opt/alloy/config/config.alloy.
|
||||
//
|
||||
// Runs as a ROOT system service on Network=host (see
|
||||
// dankclaw/observability/quadlet/alloy.container), so it reaches Loki on the
|
||||
// host's loopback and reads every journal — host services and rootless-quadlet
|
||||
// containers alike.
|
||||
|
||||
logging {
|
||||
level = "info"
|
||||
}
|
||||
|
||||
// The alloy UI service listens on 127.0.0.1:12345 by default; the
|
||||
// in-container healthcheck probes it there. alloy 1.x refuses to
|
||||
// configure the ui service from a config block, so there is no block.
|
||||
// The alloy UI listens on 127.0.0.1:12345; the in-container healthcheck probes
|
||||
// it there. alloy 1.x refuses to configure the ui service from a config block,
|
||||
// so there is no ui block.
|
||||
|
||||
// Ship logs to Loki (container name on the podman network).
|
||||
// Loki is published on the host's loopback (127.0.0.1:3100). Alloy shares the
|
||||
// host network, so it pushes by loopback, never by container name — a root
|
||||
// container is not on the rootless dankclaw_default network.
|
||||
loki.write "default" {
|
||||
endpoint {
|
||||
url = "http://loki:3100/loki/api/v1/push"
|
||||
url = "http://127.0.0.1:3100/loki/api/v1/push"
|
||||
}
|
||||
}
|
||||
|
||||
// Systemd journal from the host socket.
|
||||
// The systemd journal. As root with /etc/machine-id + the journal dirs mounted,
|
||||
// this one source reads system.journal AND user-*.journal, so it carries both
|
||||
// host services and the rootless Quadlet containers' logs. There is deliberately
|
||||
// no container-file source: podman container stdout lands in the journal, and
|
||||
// the old /var/lib/containers/.../ctr.log and /var/log/containers globs pointed
|
||||
// at a root-storage layout that does not exist on this host.
|
||||
loki.source.journal "journal" {
|
||||
format_as_json = true
|
||||
labels = {
|
||||
@@ -24,12 +36,3 @@ loki.source.journal "journal" {
|
||||
}
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
|
||||
// Container logs (podman storage path; adjust for docker).
|
||||
loki.source.file "containers" {
|
||||
targets = [
|
||||
{__path__ = "/var/lib/containers/storage/overlay-containers/*/userdata/ctr.log", job = "podman"},
|
||||
{__path__ = "/var/log/containers/*.log", job = "containers"},
|
||||
]
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
{
|
||||
"uid": "host-system",
|
||||
"title": "Hosts \u2014 Dankclaw & Dankserver",
|
||||
"description": "node_exporter for both servers. Starter set \u2014 throw-at-the-wall breadth; refine the windows and drop the noise. The picker selects the host; CPU/load/memory/filesystem/network are per-node.",
|
||||
"tags": ["host", "system"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "node",
|
||||
"label": "host",
|
||||
"type": "query",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"query": { "query": "label_values(up{job=~\"node-.*\"}, job)", "refId": "A" },
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"current": { "text": "All", "value": "$__all" }
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1, "type": "stat", "title": "Uptime",
|
||||
"description": "Time since boot. A drop means a reboot.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
|
||||
"targets": [{ "expr": "time() - node_boot_time_seconds{job=~\"$node\"}", "legendFormat": "{{job}}", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 2, "type": "stat", "title": "Memory used",
|
||||
"description": "1 - MemAvailable/MemTotal. The honest \"in use\" figure (excludes reclaimable cache).",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.8 }, { "color": "red", "value": 0.92 }] } }, "overrides": [] },
|
||||
"targets": [{ "expr": "1 - node_memory_MemAvailable_bytes{job=~\"$node\"} / node_memory_MemTotal_bytes{job=~\"$node\"}", "legendFormat": "{{job}}", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 3, "type": "stat", "title": "Load 5m / core",
|
||||
"description": "Per-core load5. >1 sustained means runnable tasks are queuing behind cores.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 1 }] } }, "overrides": [] },
|
||||
"targets": [{ "expr": "node_load5{job=~\"$node\"} / count by (job) (node_cpu_seconds_total{job=~\"$node\",mode=\"idle\"})", "legendFormat": "{{job}}", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 4, "type": "stat", "title": "Open file handles",
|
||||
"description": "FDs allocated vs the kernel maximum. Rising toward the max precedes EMFILE across the host.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit", "max": 1, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 0.9 }] } }, "overrides": [] },
|
||||
"targets": [{ "expr": "node_filefd_allocated{job=~\"$node\"} / node_filefd_maximum{job=~\"$node\"}", "legendFormat": "{{job}}", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 5, "type": "timeseries", "title": "CPU busy %",
|
||||
"description": "CPU seconds/s in each mode = cores busy per mode; the stack sums to total cores in use (16/host here). A user or system spike points at what is loading the box; steal > 0 means the hypervisor is throttling.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "normal" } } }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "expr": "sum by (job,mode) (rate(node_cpu_seconds_total{job=~\"$node\",mode!=\"idle\",mode!=\"iowait\"}[5m]))", "legendFormat": "{{job}} {{mode}}", "refId": "A" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6, "type": "timeseries", "title": "Load average",
|
||||
"description": "load1/5/15 per host.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "expr": "node_load1{job=~\"$node\"}", "legendFormat": "{{job}} load1", "refId": "A" },
|
||||
{ "expr": "node_load5{job=~\"$node\"}", "legendFormat": "{{job}} load5", "refId": "B" },
|
||||
{ "expr": "node_load15{job=~\"$node\"}", "legendFormat": "{{job}} load15", "refId": "C" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7, "type": "timeseries", "title": "Memory breakdown",
|
||||
"description": "Total, free, buffers+cache. Reclaimable cache is not pressure; MemAvailable is the real headroom.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "expr": "node_memory_MemTotal_bytes{job=~\"$node\"} - node_memory_MemAvailable_bytes{job=~\"$node\"}", "legendFormat": "{{job}} used", "refId": "A" },
|
||||
{ "expr": "node_memory_MemAvailable_bytes{job=~\"$node\"}", "legendFormat": "{{job}} available", "refId": "B" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8, "type": "timeseries", "title": "Filesystem free %",
|
||||
"description": "Per mountpoint. The media/library stacks fill disks.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0.15 }] } }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "expr": "node_filesystem_avail_bytes{job=~\"$node\",fstype!~\"tmpfs|overlay|squashfs\"} / node_filesystem_size_bytes{job=~\"$node\",fstype!~\"tmpfs|overlay|squashfs\"}", "legendFormat": "{{job}} {{mountpoint}}", "refId": "A" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9, "type": "timeseries", "title": "Disk throughput",
|
||||
"description": "Read/write bytes/s per device.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "expr": "sum by (job,device) (rate(node_disk_read_bytes_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} R", "refId": "A" },
|
||||
{ "expr": "-sum by (job,device) (rate(node_disk_written_bytes_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} W", "refId": "B" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10, "type": "timeseries", "title": "Disk busy %",
|
||||
"description": "Fraction of time the device had I/O in flight. Near 1 = the device is the bottleneck.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1 }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "expr": "rate(node_disk_io_time_seconds_total{job=~\"$node\"}[5m])", "legendFormat": "{{job}} {{device}}", "refId": "A" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11, "type": "timeseries", "title": "Network throughput",
|
||||
"description": "Receive (up) / transmit (down) bytes/s per interface.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "expr": "sum by (job,device) (rate(node_network_receive_bytes_total{job=~\"$node\",device!~\"lo|veth.*|br.*|docker.*\"}[5m]))", "legendFormat": "{{job}} {{device}} RX", "refId": "A" },
|
||||
{ "expr": "-sum by (job,device) (rate(node_network_transmit_bytes_total{job=~\"$node\",device!~\"lo|veth.*|br.*|docker.*\"}[5m]))", "legendFormat": "{{job}} {{device}} TX", "refId": "B" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12, "type": "timeseries", "title": "Network errors / drops",
|
||||
"description": "Interface errors and drops \u2014 the signature of a saturated link or a bad NIC/cable.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "expr": "sum by (job,device) (rate(node_network_receive_errs_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} RX err", "refId": "A" },
|
||||
{ "expr": "sum by (job,device) (rate(node_network_transmit_errs_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} TX err", "refId": "B" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"uid": "logs-overview",
|
||||
"title": "Logs \u2014 Loki (systemd journal)",
|
||||
"description": "Everything in Loki arrives as the journal (job=systemd-journal); the container is a JSON field parsed at query time, not a stream label. Starter breadth \u2014 the volume/error panels group by the parsed container_name. Loki keeps ~7 days; wide ranges are cheap counts, not the raw log panel.",
|
||||
"tags": ["logs", "loki"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "container",
|
||||
"label": "container",
|
||||
"type": "query",
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"query": { "query": "{job=\"systemd-journal\"} | json", "label": "container_name", "refId": "A" },
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"refresh": 2,
|
||||
"current": { "text": "All", "value": "$__all" }
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1, "type": "timeseries", "title": "Log volume by container (lines/s)",
|
||||
"description": "Parse the journal JSON, count lines per container. Spikes = a chatty or crashing unit.",
|
||||
"gridPos": { "h": 9, "w": 24, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps", "custom": { "fillOpacity": 20, "stacking": { "mode": "normal" } } }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum by (container_name) (count_over_time({job=\"systemd-journal\"} | json [$__auto]))", "legendFormat": "{{container_name}}", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 2, "type": "timeseries", "title": "Total log rate",
|
||||
"description": "All journal lines per second.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} [$__auto]))", "legendFormat": "all lines/s", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 3, "type": "timeseries", "title": "Error-level line rate",
|
||||
"description": "Lines matching error/panic/fatal/fail. A blunt text match \u2014 the journal carries no structured level for all units.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "fixed", "fixedColor": "red" } }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal|fail)\" [$__auto]))", "legendFormat": "error lines/s", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 4, "type": "timeseries", "title": "Errors by container",
|
||||
"description": "Which unit is throwing. Rank by the top offenders.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum by (container_name) (count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" | json [$__auto]))", "legendFormat": "{{container_name}}", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 5, "type": "stat", "title": "Error lines (last 15m)",
|
||||
"description": "Absolute count over the window, not a rate.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 500 }] } }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" [15m]))", "instant": true, "legendFormat": "errors/15m", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 6, "type": "logs", "title": "Journal (filtered)",
|
||||
"description": "Raw lines for the selected containers. Set the container picker; keep the window tight.",
|
||||
"gridPos": { "h": 12, "w": 24, "x": 0, "y": 25 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true },
|
||||
"targets": [{ "expr": "{job=\"systemd-journal\"} | json | container_name=~\"$container\"", "refId": "A" }]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
{
|
||||
"uid": "observability",
|
||||
"title": "Observability \u2014 self-monitoring",
|
||||
"description": "The health of the stack that watches the fleet: Prometheus scrape/TSDB, Loki ingest, Grafana. If this is red the dashboards lie. Note: Alertmanager is running but NOT scraped by Prometheus (no job) \u2014 its metrics are absent by design gap, not fault.",
|
||||
"tags": ["observability", "meta"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"panels": [
|
||||
{
|
||||
"id": 1, "type": "stat", "title": "Targets up",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } }, "overrides": [] },
|
||||
"targets": [{ "expr": "count(up == 1)", "instant": true, "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 2, "type": "stat", "title": "Targets down",
|
||||
"description": "Known structural downs: comfyui (no metrics endpoint), gitea (metrics off), and the homepage 'services' probes. Real regressions ride up from here too \u2014 check the table before dismissing.",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] } }, "overrides": [] },
|
||||
"targets": [{ "expr": "count(up == 0)", "instant": true, "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 3, "type": "stat", "title": "TSDB head series",
|
||||
"description": "Active series in memory. A jump = a new exporter or a label-cardinality leak.",
|
||||
"gridPos": { "h": 4, "w": 5, "x": 8, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||
"targets": [{ "expr": "prometheus_tsdb_head_series", "instant": true, "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 4, "type": "stat", "title": "Discovered targets",
|
||||
"gridPos": { "h": 4, "w": 5, "x": 13, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum(prometheus_sd_discovered_targets)", "instant": true, "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 5, "type": "stat", "title": "Prometheus mem",
|
||||
"gridPos": { "h": 4, "w": 3, "x": 18, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] },
|
||||
"targets": [{ "expr": "process_resident_memory_bytes{job=\"prometheus\"}", "instant": true, "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 6, "type": "stat", "title": "Grafana up",
|
||||
"gridPos": { "h": 4, "w": 3, "x": 21, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "mappings": [{ "type": "value", "options": { "1": { "text": "UP", "color": "green" }, "0": { "text": "DOWN", "color": "red" } } }], "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } }, "overrides": [] },
|
||||
"targets": [{ "expr": "up{job=\"grafana\"}", "instant": true, "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 7, "type": "table", "title": "Target health",
|
||||
"description": "Every scrape target and its current up. Sort by Value to surface regressions.",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"options": { "sortBy": [{ "displayName": "Value", "sortOrder": "Asc" }] },
|
||||
"targets": [{ "expr": "up", "format": "table", "instant": true, "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 8, "type": "timeseries", "title": "Scrape interval drift (p99 actual)",
|
||||
"description": "Actual seconds between scrapes (summary quantile). Above the 15s scrape_interval = Prometheus is falling behind its own cadence (slow endpoints or overload).",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
|
||||
"targets": [{ "expr": "prometheus_target_interval_length_seconds{quantile=\"0.99\"}", "legendFormat": "{{instance}}", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 9, "type": "timeseries", "title": "Scrape samples/s",
|
||||
"description": "Total samples ingested across all jobs.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum(rate(scrape_samples_scraped[5m]))", "legendFormat": "samples/s", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 10, "type": "timeseries", "title": "Loki request rate",
|
||||
"description": "Loki served requests/s (push, query). A drop to zero means ingest or query has stopped.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum by (route) (rate(loki_request_duration_seconds_count[5m]))", "legendFormat": "{{route}}", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 11, "type": "timeseries", "title": "Exporter process CPU",
|
||||
"description": "CPU by monitored process \u2014 spot a runaway exporter or the Prometheus process itself.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 21 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum by (job) (rate(process_cpu_seconds_total[5m]))", "legendFormat": "{{job}}", "refId": "A" }]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
# Grafana datasource for Loki, declared in git rather than clicked into
|
||||
# grafana.db — same reason as the Prometheus datasource: a UI-configured
|
||||
# datasource is invisible to review, lost with the data volume, and unknown
|
||||
# to the repository. Loki OSS ships no web UI of its own, so Explore through
|
||||
# this datasource is the only way a human reads the log store.
|
||||
#
|
||||
# URL is the container name on dankclaw_default (the same address Caddy
|
||||
# proxies to), not the published loopback port.
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
@@ -30,9 +30,9 @@
|
||||
},
|
||||
{
|
||||
"name": "open-notebook",
|
||||
"category": "user",
|
||||
"category": "admin",
|
||||
"section": "AI",
|
||||
"description": "Podcast generator — notes in, spoken episode out: script written by frank-zealot, voices rendered by frank-tts. Guarded twice: Authentik at the edge, then its own login.",
|
||||
"description": "Podcast generator — notes in, spoken episode out: script written by frank-archon, voices rendered by frank-tts. Guarded twice: Authentik at the edge, then its own login. Single shared password, no per-user accounts — admin-only.",
|
||||
"url": "http://notebook.franksdank.cloud",
|
||||
"docs": "",
|
||||
"probe": "127.0.0.1:8502",
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
# Alerting and recording rules for the LLM fleet.
|
||||
#
|
||||
# One metric namespace: llamacpp:* — frank-zealot and frank-dragoon, both
|
||||
# llama.cpp with /metrics (the family verified from /metrics on 2026-09-16
|
||||
# and 2026-09-20). frank-archon LEFT the family 2026-10-05: FreeToken (the
|
||||
# PR587 ROCm engine) has no /metrics endpoint, so its scrape job was
|
||||
# deleted rather than left never-green.
|
||||
# Two metric namespaces, because two runtime families are scraped:
|
||||
#
|
||||
# * llamacpp:* — frank-zealot (llama.cpp) and frank-dragoon (halogen, which
|
||||
# re-exports the llama.cpp metric names). Verified from /metrics on
|
||||
# 2026-09-16 / 2026-09-20.
|
||||
# * vllm:* — frank-archon, the radiance engine on Dankserver since
|
||||
# 2026-10-05. It exports vLLM-compatible names, a DIFFERENT namespace, so
|
||||
# it is a separate scrape job in prometheus.yml and the rules below are a
|
||||
# separate group. (The old claim that its scrape job was deleted is false:
|
||||
# it re-joined the day it left the FreeToken engine and is `up` today.)
|
||||
# (The netavark published-port quirk that forces the Dankclaw endpoints
|
||||
# onto the gateway is same-host only.)
|
||||
#
|
||||
@@ -22,7 +27,8 @@
|
||||
# cached by cached + prompt.
|
||||
# * Speculative decoding: drafts and accepted tokens are both counters,
|
||||
# so LlmSpecDecodeOff watches the pair.
|
||||
# * No latency histograms exist, so there is no p95 alert.
|
||||
# * No latency histograms in llama.cpp, so no p95 alert there; radiance's
|
||||
# vllm:* namespace DOES export one, so the vllm-fleet group alerts on it.
|
||||
#
|
||||
# IMPORTANT: delivery. Since 2026-09-17 a firing alert reaches Frank on
|
||||
# Signal: Prometheus pushes to Alertmanager (dankclaw/observability/quadlet/
|
||||
@@ -112,6 +118,84 @@ groups:
|
||||
description: >-
|
||||
Prometheus cannot reach the LLM endpoint. frank-zealot and
|
||||
frank-dragoon are root units on Dankclaw: `systemctl is-active
|
||||
frank-zealot` / `frank-dragoon`. Both scraped endpoints export
|
||||
the llamacpp:* family. (frank-archon left the scrape set
|
||||
2026-10-05 — FreeToken exports no /metrics.)
|
||||
frank-zealot` / `frank-dragoon` (both export llamacpp:* —
|
||||
dragoon via halogen's llama.cpp-compatible names). frank-archon
|
||||
is a radiance endpoint on Dankserver (`systemctl show
|
||||
frank-archon` there) exporting the vllm:* family; the
|
||||
`frank-.*` selector covers all three regardless of namespace.
|
||||
|
||||
# frank-archon (radiance) exports the vLLM-compatible namespace, NOT
|
||||
# llamacpp:*. Every rule here is keyed to a series the exporter actually
|
||||
# emits (verified from /metrics 2026-10-20: num_requests_waiting,
|
||||
# gpu_prefix_cache_hit_rate, num_preemptions_total, prompt_tokens_total,
|
||||
# time_to_first_token_seconds). The llamacpp rules above evaluate nothing for
|
||||
# this job and these evaluate nothing for llama.cpp/halogen — two clean
|
||||
# namespaces, each keyed only to what its runtime exports.
|
||||
- name: vllm-fleet
|
||||
interval: 30s
|
||||
rules:
|
||||
# p95 time-to-first-token, shared with the dashboard. When the endpoint
|
||||
# is idle every bucket rate is zero, histogram_quantile yields NaN, and
|
||||
# neither this nor the alert below can fire on an idle engine.
|
||||
- record: job:vllm_ttft_p95:rate5m
|
||||
expr: |
|
||||
histogram_quantile(0.95, sum by (job, le)
|
||||
(rate(vllm:time_to_first_token_seconds_bucket[5m])))
|
||||
|
||||
# Radiance exports a real queue-depth gauge (llama.cpp only shows the
|
||||
# wall as requests_deferred). --max-num-seqs is small, so a sustained
|
||||
# wait is either real load or this endpoint carrying the fleet alone.
|
||||
- alert: VllmRequestsQueueing
|
||||
expr: vllm:num_requests_waiting > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $labels.job }}: requests waiting for 10 minutes"
|
||||
description: >-
|
||||
The radiance scheduler has requests queued. Either the fleet is
|
||||
genuinely busy or this endpoint is carrying it alone.
|
||||
|
||||
# Radiance reports prefix reuse directly as a hit-rate gauge — no
|
||||
# counter division needed. The token-rate floor keeps an idle engine
|
||||
# (hit rate 0 with no traffic) from firing.
|
||||
- alert: VllmPrefixCacheCollapse
|
||||
expr: >-
|
||||
vllm:gpu_prefix_cache_hit_rate < 0.20
|
||||
and rate(vllm:prompt_tokens_total[5m]) > 0.05
|
||||
for: 15m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: "{{ $labels.job }}: prefix hit rate below 20 % for 15 minutes"
|
||||
description: >-
|
||||
Nearly every request prefilling from scratch. With several large
|
||||
contexts in flight this is the capacity wall, not a routing fault.
|
||||
|
||||
# Preemption is the KV-thrash signature radiance reports that llama.cpp
|
||||
# cannot: the engine evicts a running request to admit another. Any
|
||||
# sustained rate means the KV cache cannot hold the concurrency in flight.
|
||||
- alert: VllmPreempting
|
||||
expr: rate(vllm:num_preemptions_total[5m]) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $labels.job }}: engine preempting requests"
|
||||
description: >-
|
||||
The engine is evicting running requests to admit new ones — the KV
|
||||
cache is undersized for the concurrency in flight.
|
||||
|
||||
# p95 first-token latency, kept HIGH on purpose: a large-prompt prefill
|
||||
# legitimately spends many seconds before token 1, so this fires only on
|
||||
# a wedged scheduler or a queue backed up behind one request.
|
||||
- alert: VllmTtftSlow
|
||||
expr: job:vllm_ttft_p95:rate5m > 120
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $labels.job }}: p95 time-to-first-token over 120 s"
|
||||
description: >-
|
||||
First-token latency is pathological, not an honest big prefill.
|
||||
Likely a wedged scheduler or one request stalling the queue.
|
||||
|
||||
@@ -240,14 +240,12 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
|
||||
}
|
||||
|
||||
@test "alloy's UI address comes from the unit, not the config file" {
|
||||
# alloy 1.x refuses to configure the ui service from a config block
|
||||
# ("service ui does not support being configured"), so the unit moves the
|
||||
# UI off the container's loopback with a command-line flag. Without that
|
||||
# flag the port cannot be published and the service has no reachable
|
||||
# health endpoint at all — which is why the homepage status card for
|
||||
# alloy read "down" while it ran under compose.
|
||||
# alloy 1.x refuses to configure the ui service from a config block, so the
|
||||
# unit sets the UI address on the command line. As a Network=host root unit
|
||||
# that address is the host loopback directly (no PublishPort) — reachable
|
||||
# from the host, from nothing off-host.
|
||||
! grep -qE '^(ui|server) \{' "$REPO_ROOT/shared/alloy/config.alloy"
|
||||
local unit="$REPO_ROOT/dankclaw/observability/quadlet/alloy.container"
|
||||
grep -q -- '--server.http.listen-addr=0.0.0.0:12345' "$unit"
|
||||
grep -q '^PublishPort=127.0.0.1:12345:12345$' "$unit"
|
||||
grep -q -- '--server.http.listen-addr=127.0.0.1:12345' "$unit"
|
||||
! grep -q '^PublishPort=' "$unit"
|
||||
}
|
||||
|
||||
@@ -247,3 +247,21 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
}
|
||||
done
|
||||
}
|
||||
|
||||
@test "the fabric-clock host unit re-dispatches frank-dragoon and stages nothing else" {
|
||||
# The hold is the NPU lane's boot precondition and the deployer installs and
|
||||
# gates it, so a change to it has to reach the endpoint. The tracked boot
|
||||
# line under the same directory is a hand-taken reboot decision and must
|
||||
# never plan a restart, and staging the NPU model files is host-side state
|
||||
# like every other download script.
|
||||
run python3 "$PLANNER" --target dankclaw --paths \
|
||||
dankclaw/host/halogen-fabric-clock dankclaw/host/halogen-fabric-clock.service
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$output" = "frank-dragoon" ]
|
||||
run python3 "$PLANNER" --target dankclaw --paths dankclaw/host/grub
|
||||
[ "$status" -eq 0 ]
|
||||
[ -z "$output" ]
|
||||
run python3 "$PLANNER" --target dankclaw --paths scripts/download-frank-dragoon-npu.sh
|
||||
[ "$status" -eq 0 ]
|
||||
[ -z "$output" ]
|
||||
}
|
||||
|
||||
@@ -29,8 +29,14 @@ export default defineConfig({
|
||||
// green suite with a test that does not exist. 17- joined on 2026-09-16
|
||||
// (the hermes vhost, whose upstream is a host address rather than a
|
||||
// container name, so it needs its own pass condition). 18- joined on
|
||||
// 2026-09-24 (OpenWebUI SSO through authentik).
|
||||
testMatch: /(10|12|13|16|17|18)-.*\.spec\.ts/,
|
||||
// 2026-09-24 (OpenWebUI SSO through authentik). 20- joined on 2026-10-21
|
||||
// (Grafana's OAuth2 relying-party login, which the edge-only 13- never
|
||||
// follows). 21- joined 2026-10-21 (the open-notebook vhost, deployed
|
||||
// 2026-10-19 - the newest guarded app, in neither GUARDED list nor any
|
||||
// spec until now: the 17-hermes "uncovered vhost broke silently" class).
|
||||
// 22- joins Prometheus's app-is-really-there proof (its own API), closing
|
||||
// the "not a login wall but maybe a 404" hole 20- closed for Grafana.
|
||||
testMatch: /(10|12|13|16|17|18|20|21|22)-.*\.spec\.ts/,
|
||||
use: { storageState: ".auth/qa-e2e.json" },
|
||||
dependencies: ["setup"],
|
||||
},
|
||||
|
||||
@@ -15,7 +15,7 @@ const BASE = "https://www.franksdank.cloud";
|
||||
// Every vhost that Caddy routes through anubis:8923 (see dankclaw/Caddyfile).
|
||||
// Keep this list in lockstep with the Caddyfile: a new guarded vhost must be
|
||||
// added here or the guard-chain audit is incomplete.
|
||||
const GUARDED = ["www", "search", "romm", "files", "seerr", "grafana", "prometheus", "loki"];
|
||||
const GUARDED = ["www", "search", "romm", "files", "seerr", "grafana", "prometheus", "loki", "notebook"];
|
||||
|
||||
// Markers that prove app content leaked past the guard to an anonymous user.
|
||||
const LEAK_MARKERS: Record<string, string[]> = {
|
||||
@@ -25,6 +25,7 @@ const LEAK_MARKERS: Record<string, string[]> = {
|
||||
"seerr.franksdank.cloud": ["Request Movie"],
|
||||
"grafana.franksdank.cloud": ["Dashboards"],
|
||||
"prometheus.franksdank.cloud": ["Expression Browser", "Graph"],
|
||||
"notebook.franksdank.cloud": ["Open Notebook"],
|
||||
};
|
||||
|
||||
test("homepage root (/) is guarded for unauthenticated users", async ({ page }) => {
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
|
||||
// Grafana (grafana.franksdank.cloud) is an OAuth2 RELYING PARTY, not an
|
||||
// outpost-fronted proxy: the guard chain (13-sso) only proves an authenticated
|
||||
// session reaches Grafana's login page. The login itself happens through
|
||||
// Grafana's own "Sign in with Authentik" button, which the edge test never
|
||||
// follows — that is exactly how the RP login could be dead (500 on authorize,
|
||||
// or an email-claim gap bouncing back to /login) while 13- still passed green.
|
||||
// This spec drives the real RP round-trip and asserts a live Grafana session.
|
||||
|
||||
const HOST = "grafana.franksdank.cloud";
|
||||
const CHALLENGE = "Making sure you're not a bot";
|
||||
|
||||
test("SSO: Grafana relying-party login lands a real session", async ({ page, request }) => {
|
||||
test.setTimeout(180_000);
|
||||
|
||||
// Warm the guard: the vhost runs its own Anubis proof-of-work on first visit.
|
||||
await page.goto(`https://${HOST}/`);
|
||||
await expect
|
||||
.poll(
|
||||
async () => !(await page.locator("body").innerText()).includes(CHALLENGE),
|
||||
{ timeout: 90_000 },
|
||||
)
|
||||
.toBe(true);
|
||||
|
||||
// The "Sign in with Authentik" button targets this RP start URL exactly.
|
||||
// Going straight there is the same code path without depending on the SPA's
|
||||
// button markup.
|
||||
await page.goto(`https://${HOST}/login/generic_oauth`);
|
||||
|
||||
// The chain: grafana -> auth (implicit consent, the session is already
|
||||
// there) -> back to grafana with the code -> grafana provisions the user and
|
||||
// redirects to "/". A dead provider/claims config bounces back to /login.
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const u = new URL(page.url());
|
||||
return u.hostname === HOST && u.pathname !== "/login";
|
||||
},
|
||||
{ timeout: 90_000 },
|
||||
)
|
||||
.toBe(true);
|
||||
|
||||
const body = await page.locator("body").innerText();
|
||||
expect(body).not.toContain("Login failed");
|
||||
expect(body.toLowerCase()).not.toContain("powered by authentik"); // IdP flow shell
|
||||
expect(body).not.toContain(CHALLENGE);
|
||||
|
||||
// A genuine logged-in session exists: /api/user answers 200 with an email,
|
||||
// not 401. This is the assertion the edge-only spec could never make.
|
||||
const resp = await page.request.get(`https://${HOST}/api/user`);
|
||||
expect(resp.status()).toBe(200);
|
||||
const user = await resp.json();
|
||||
expect(typeof user.email).toBe("string");
|
||||
expect(user.email.length).toBeGreaterThan(0);
|
||||
});
|
||||
@@ -0,0 +1,52 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
|
||||
// open-notebook (notebook.franksdank.cloud), deployed 2026-10-19, is a guarded
|
||||
// vhost whose outpost (notebook-proxy) fronts an app with its OWN password
|
||||
// login - the same shape as hermes (17-) and OpenWebUI (18-): the guard chain
|
||||
// (Anubis PoW -> authentik outpost) must hand off to the app, and the app's own
|
||||
// login page is the expected landing. Until now this host appeared in neither
|
||||
// the 13-/14- GUARDED lists nor any dedicated spec, so a broken upstream would
|
||||
// have stayed invisible - the exact 17-hermes "uncovered vhost" failure class.
|
||||
// 14- now covers the unauthenticated gate; this spec owns authenticated
|
||||
// reachability with a POSITIVE app marker, not just "not a login wall".
|
||||
|
||||
const VHOST = "https://notebook.franksdank.cloud/";
|
||||
const CHALLENGE = "Making sure you're not a bot";
|
||||
|
||||
test("notebook vhost reaches the open-notebook app through the guard chain", async ({ page }) => {
|
||||
test.setTimeout(180_000);
|
||||
await page.goto(VHOST);
|
||||
|
||||
// The Anubis proof-of-work clears in a real browser and the outpost hands off
|
||||
// to the app; the user must never be parked on an authentik flow.
|
||||
await expect
|
||||
.poll(async () => !(await page.locator("body").innerText()).includes(CHALLENGE), { timeout: 90_000 })
|
||||
.toBe(true);
|
||||
await expect
|
||||
.poll(() => new URL(page.url()).hostname, { timeout: 60_000 })
|
||||
.toBe("notebook.franksdank.cloud");
|
||||
expect(
|
||||
new URL(page.url()).pathname.startsWith("/if/"),
|
||||
"must not be parked on an authentik flow",
|
||||
).toBe(false);
|
||||
expect(
|
||||
await page.locator('input[name="uidField"]').count(),
|
||||
"must not show the authentik login form",
|
||||
).toBe(0);
|
||||
|
||||
// Positive identity: the page really is open-notebook's own login, not a
|
||||
// Caddy error or the IdP shell. The app answering its own login is the pass.
|
||||
await expect(page, "destination title must be Open Notebook").toHaveTitle(/open notebook/i, {
|
||||
timeout: 30_000,
|
||||
});
|
||||
await expect(page.locator("body"), "open-notebook login marker").toContainText(
|
||||
/enter your password/i,
|
||||
{ timeout: 30_000 },
|
||||
);
|
||||
const body = await page.locator("body").innerText();
|
||||
expect(body, "must not leak the IdP flow shell").not.toContain("Powered by authentik");
|
||||
expect(body, "must not be a Caddy upstream error").not.toContain("Bad Gateway");
|
||||
|
||||
const resp = await page.request.get(VHOST);
|
||||
expect(resp.status(), "open-notebook must answer, not error").toBeLessThan(500);
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
|
||||
// Prometheus (prometheus.franksdank.cloud) sits behind the same embedded outpost
|
||||
// as Grafana. 13-sso and 11-admin only ever assert it is "not a login wall and
|
||||
// under 500" - which a Prometheus serving a 404 or a blank page would ALSO pass.
|
||||
// This is the app-is-really-there proof 20- provides for Grafana: with the
|
||||
// standard session the outpost hands off silently to the Prometheus UI, and the
|
||||
// app's own API answers success. A guard-route or upstream regression that lets
|
||||
// any non-Prometheus answer through fails here.
|
||||
|
||||
test("SSO: Prometheus app answers its own API through authentik", async ({ page, request }) => {
|
||||
test.setTimeout(180_000);
|
||||
await page.goto("https://prometheus.franksdank.cloud/");
|
||||
|
||||
// Challenge clears, outpost hands off silently to the app (no credential prompt).
|
||||
await expect
|
||||
.poll(
|
||||
async () => !(await page.locator("body").innerText()).includes("Making sure you're not a bot"),
|
||||
{ timeout: 90_000 },
|
||||
)
|
||||
.toBe(true);
|
||||
await expect
|
||||
.poll(() => new URL(page.url()).hostname, { timeout: 60_000 })
|
||||
.toBe("prometheus.franksdank.cloud");
|
||||
expect(await page.locator('input[name="uidField"]').count(), "no authentik form").toBe(0);
|
||||
await expect(page, "must land on the Prometheus UI").toHaveTitle(/prometheus/i, { timeout: 30_000 });
|
||||
|
||||
// Load-bearing assertion: the Prometheus API itself answers, authenticated,
|
||||
// through the outpost. An IdP bounce, a 404, or a raw upstream error fails here.
|
||||
const api = await request.get("https://prometheus.franksdank.cloud/api/v1/status/buildinfo");
|
||||
expect(api.status(), "Prometheus API must answer 200 through the guard").toBe(200);
|
||||
expect((await api.json()).status, "Prometheus API must report success").toBe("success");
|
||||
});
|
||||
@@ -145,7 +145,7 @@ nosudo_bin() {
|
||||
# (torch.compile query_start_loc constraint, 2026-10-02); radiance booted
|
||||
# seqs 2 in the fp8-native era and the 2026-10-06 sweep had seqs 2 and 3
|
||||
# tied inside the +-3% noise floor — slots buy concurrency, not speed.
|
||||
grep -q '^FRANK_ARCHON_MAX_SEQS=2$' "$ENV_FILE"
|
||||
grep -q '^FRANK_ARCHON_MAX_SEQS=4$' "$ENV_FILE"
|
||||
grep -q '^FRANK_ARCHON_TP=2$' "$ENV_FILE"
|
||||
grep -q '^FRANK_ARCHON_TP_WIRE=wht6$' "$ENV_FILE"
|
||||
grep -q '^FRANK_ARCHON_SPEC_TOKENS=2$' "$ENV_FILE"
|
||||
@@ -158,7 +158,7 @@ nosudo_bin() {
|
||||
# so the pool's token count is a boot-ledger line — verified live by the
|
||||
# capacity leg of scripts/probe-frank-archon-kv.py at every cutover.
|
||||
grep -q '^FRANK_ARCHON_PLACEMENT=expert_tiered$' "$ENV_FILE"
|
||||
grep -q '^FRANK_ARCHON_HOST_POOL_MIB=44928$' "$ENV_FILE"
|
||||
grep -q '^FRANK_ARCHON_HOST_POOL_MIB=49152$' "$ENV_FILE"
|
||||
grep -q '^FRANK_ARCHON_KV_DTYPE=fp8$' "$ENV_FILE"
|
||||
grep -q '^FRANK_ARCHON_NGRAM_PLACEMENT=ram$' "$ENV_FILE"
|
||||
# Prefix cache: pinned host RAM only (the disk tier was removed), and the
|
||||
@@ -177,8 +177,11 @@ nosudo_bin() {
|
||||
[ "$ttmt" -gt "$ttmm" ]
|
||||
pool=$(awk -F= '/^FRANK_ARCHON_HOST_POOL_MIB=/{print $2}' "$ENV_FILE")
|
||||
ph=$(awk -F= '/^FRANK_ARCHON_PREFIX_CACHE_MIB=/{print $2}' "$ENV_FILE")
|
||||
# 2 pools + PLE 47.7 GiB (48,824 MiB) + prefix host, rounded up.
|
||||
ledger=$(( 2 * pool + 48824 + ph + 2048 ))
|
||||
# 2 pools + PLE 47.7 GiB (48,824 MiB) + prefix host + MNBT + the auto
|
||||
# checkpoint store (MAX_SEQS x 10 slots of 112 MiB -- the store is pinned host
|
||||
# memory too, and it grows with the slot count).
|
||||
seqs=$(awk -F= '/^FRANK_ARCHON_MAX_SEQS=/{print $2}' "$ENV_FILE")
|
||||
ledger=$(( 2 * pool + 48824 + ph + 2048 + seqs * 10 * 112 ))
|
||||
[ "$ttmm" -ge "$ledger" ]
|
||||
# The .rad size floor is positive (a truncated convert must fail fast).
|
||||
grep -q '^FRANK_ARCHON_RAD_MIN_MIB=' "$ENV_FILE"
|
||||
|
||||
@@ -27,6 +27,10 @@ setup() {
|
||||
ENV_FILE="$ROOT/scripts/frank-dragoon.env"
|
||||
UNIT="$ROOT/scripts/frank-dragoon.service"
|
||||
DEPLOYER="$ROOT/scripts/deploy-remote.sh"
|
||||
HOST_UNIT="$ROOT/dankclaw/host/halogen-fabric-clock"
|
||||
HOST_SERVICE="$ROOT/dankclaw/host/halogen-fabric-clock.service"
|
||||
NPU_FETCH="$ROOT/scripts/download-frank-dragoon-npu.sh"
|
||||
NPU_RULE="$ROOT/dankclaw/udev/98-halogen-npu-access.rules"
|
||||
BUILD="$ROOT/scripts/build-rocmfpx.sh"
|
||||
TEST_TMP="$(mktemp -d)"
|
||||
touch "$TEST_TMP/empty.env"
|
||||
@@ -167,6 +171,14 @@ good_env() {
|
||||
kv_gib=$(( pool * 28 * 1024 / 1073741824 ))
|
||||
arena_tenths=$(( 38 + 84 * mt / 16384 ))
|
||||
total_gib=$(( 62 + kv_gib + arena_tenths / 10 + 2 + reserve + 15 ))
|
||||
# The NPU lane bills extra pinned host RAM per named model (the vendor's
|
||||
# budget note: ~2 GiB for an embedding/decider/rerank-sized program, more
|
||||
# for FLUX-class ones), and the lane is named in the shipped env, so the
|
||||
# ledger must still close on a 121 GiB unified-memory host.
|
||||
npu_models=0
|
||||
npu="$(sed -nE 's/^HALOGEN_NPU_MODELS=(.*)$/\1/p' "$ENV_FILE")"
|
||||
[ -n "$npu" ] && npu_models="$(echo "${npu//,/ }" | wc -w)"
|
||||
total_gib=$(( total_gib + 2 * npu_models ))
|
||||
[ "$total_gib" -le 121 ]
|
||||
}
|
||||
|
||||
@@ -339,6 +351,18 @@ $(cat "$TEST_TMP/serve.log")"
|
||||
grep -qF "LLM_DRAGOON_TOK_FILES=\"\${LLM_DRAGOON_TOK_FILES:-$tokf}\"" "$DEPLOYER"
|
||||
bytes="$(sed -nE 's/^FRANK_DRAGOON_CHECKPOINT_BYTES=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
grep -qF "LLM_DRAGOON_CKPT_BYTES=\"\${LLM_DRAGOON_CKPT_BYTES:-$bytes}\"" "$DEPLOYER"
|
||||
# The NPU twins: the lane switches on the same value on both sides. A blank
|
||||
# env line with a named deployer default (or the reverse) is the 2026-10-01
|
||||
# disagreement again — one side gates a lane the other does not build.
|
||||
npu_dir="$(sed -nE 's/^FRANK_DRAGOON_NPU_DIR=(.*)$/\1/p' "$ENV_FILE")"
|
||||
npu_models="$(sed -nE 's/^HALOGEN_NPU_MODELS=(.*)$/\1/p' "$ENV_FILE")"
|
||||
grep -qF "LLM_DRAGOON_NPU_DIR=\"\${LLM_DRAGOON_NPU_DIR:-$npu_dir}\"" "$DEPLOYER"
|
||||
grep -qF "LLM_DRAGOON_NPU_MODELS=\"\${LLM_DRAGOON_NPU_MODELS:-$npu_models}\"" "$DEPLOYER"
|
||||
# The deployer's own copy of the payload dir is what the lane's containment
|
||||
# gate compares against — a stale twin would wave a bad staging path through.
|
||||
grep -qF "LLM_DRAGOON_MODEL_DIR=\"\${LLM_DRAGOON_MODEL_DIR:-$dir}\"" "$DEPLOYER"
|
||||
net="$(sed -nE 's/^FRANK_DRAGOON_NETWORK=(.*)$/\1/p' "$ENV_FILE")"
|
||||
grep -qF "LLM_DRAGOON_NETWORK=\"\${LLM_DRAGOON_NETWORK:-$net}\"" "$DEPLOYER"
|
||||
}
|
||||
|
||||
@test "the deployer points a missing artifact at the v2abl fetch script" {
|
||||
@@ -383,3 +407,257 @@ $(cat "$TEST_TMP/serve.log")"
|
||||
grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD"
|
||||
grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD"
|
||||
}
|
||||
|
||||
# --- the NPU lane (XDNA2) ----------------------------------------------------
|
||||
# Opt-in lane added 2026-10-19: halogen runs the vendor's precompiled AIE
|
||||
# programs for the models HALOGEN_NPU_MODELS names on the NPU inside the SAME
|
||||
# container (they answer /v1/embeddings on 7770). Two things make this seam
|
||||
# worth pinning: the lane must be genuinely OFF when the variable is blank (the
|
||||
# rollback is one line, so the render must show nothing lane-specific), and
|
||||
# every host precondition the engine needs — the accel node, the distro XRT's
|
||||
# three libraries, the staged model files, the held GPU fabric clock — must be
|
||||
# refused BY NAME instead of surfacing as an engine crash-loop.
|
||||
|
||||
npu_fixture() {
|
||||
# 0666 like the installed node: a rootless container's namespace cannot open
|
||||
# the stock root:render 0660 mode, and the serve gate checks exactly that.
|
||||
touch "$TEST_TMP/accel0"
|
||||
chmod 666 "$TEST_TMP/accel0"
|
||||
mkdir -p "$TEST_TMP/xrt"
|
||||
for l in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
|
||||
touch "$TEST_TMP/xrt/$l"
|
||||
done
|
||||
mkdir -p "$TEST_TMP/npu/qwen3-embedding-0.6b/devices"
|
||||
mkdir -p "$TEST_TMP/sys/class/drm/card1/device"
|
||||
printf '0: 400Mhz \n7: 2000Mhz *\n' > "$TEST_TMP/sys/class/drm/card1/device/pp_dpm_fclk"
|
||||
printf 'manual\n' > "$TEST_TMP/sys/class/drm/card1/device/power_dpm_force_performance_level"
|
||||
NPU_ENV=(
|
||||
"HALOGEN_NPU_MODELS=qwen3-embedding-0.6b"
|
||||
"FRANK_DRAGOON_NPU_DIR=$TEST_TMP/npu"
|
||||
"FRANK_DRAGOON_DEV_ACCEL=$TEST_TMP/accel0"
|
||||
"FRANK_DRAGOON_XRT_DIR=$TEST_TMP/xrt"
|
||||
"FRANK_DRAGOON_SYSFS=$TEST_TMP/sys"
|
||||
)
|
||||
}
|
||||
|
||||
# run_with_log <args...> — run the serve script and fold its own stderr plus
|
||||
# the podman stub's argv (which the script redirects into the log) into
|
||||
# $output, so both refusal text and render assertions read one variable.
|
||||
run_with_log() {
|
||||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" "${NPU_ENV[@]}" "$@" "$SERVE"
|
||||
if [ -f "$TEST_TMP/serve.log" ]; then
|
||||
output="$output
|
||||
$(cat "$TEST_TMP/serve.log")"
|
||||
fi
|
||||
}
|
||||
|
||||
@test "the lane is off when HALOGEN_NPU_MODELS is blank: no device, no mounts, no engine env" {
|
||||
# The rollback is blanking one env line, so "off" has to be observable in
|
||||
# the command podman would run — not just implied by an absent log line.
|
||||
podman_stub
|
||||
good_env
|
||||
run_with_log HALOGEN_NPU_MODELS=
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"--publish 7770:8731"* ]]
|
||||
! grep -q '/models/npu' "$TEST_TMP/serve.log"
|
||||
! grep -q 'HALOGEN_NPU_MODELS' "$TEST_TMP/serve.log"
|
||||
! grep -q 'xilinx' "$TEST_TMP/serve.log"
|
||||
! grep -q 'accel0' "$TEST_TMP/serve.log"
|
||||
}
|
||||
|
||||
@test "the lane arms the accel node, the staged model dir and each XRT library twice" {
|
||||
# Each library mounts at the image's XRT path AND at its own path: the
|
||||
# distro libraries find each other where they were installed, and an
|
||||
# /opt/xilinx/xrt of symlinks into the system dir dangles inside the
|
||||
# container (the vendor calls that out by name).
|
||||
podman_stub
|
||||
good_env
|
||||
npu_fixture
|
||||
run_with_log
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"--device $TEST_TMP/accel0"* ]]
|
||||
# The lane's files ride the payload mount: /models is a read-only bind and
|
||||
# runc cannot mkdir a nested /models/npu mountpoint through it (measured
|
||||
# 2026-10-19), so the staging dir lives INSIDE the payload dir and gets no
|
||||
# -v of its own.
|
||||
[[ "$output" == *"-v $TEST_TMP:/models:ro"* ]]
|
||||
[[ "$output" != *":/models/npu"* ]]
|
||||
[[ "$output" == *"-e HALOGEN_NPU_MODELS=qwen3-embedding-0.6b"* ]]
|
||||
for l in libxrt_coreutil.so.2 libxrt_core.so.2 libxrt_driver_xdna.so.2; do
|
||||
[[ "$output" == *"-v $TEST_TMP/xrt/$l:/opt/xilinx/xrt/lib/$l:ro"* ]]
|
||||
[[ "$output" == *"-v $TEST_TMP/xrt/$l:$TEST_TMP/xrt/$l:ro"* ]]
|
||||
done
|
||||
# The Flash payload still rides the same container: the lane is additive.
|
||||
[[ "$output" == *"-e HALOGEN_CHECKPOINT=/models/qwen.hgn"* ]]
|
||||
}
|
||||
|
||||
@test "the lane refuses a staging dir outside the payload dir and says why" {
|
||||
# A staging dir outside the payload dir needs a nested bind the read-only
|
||||
# /models mount cannot host: runc dies at container init, which is AFTER the
|
||||
# deployer has stopped the old unit — so the shape is refused here, before
|
||||
# anything is rendered.
|
||||
podman_stub
|
||||
good_env
|
||||
npu_fixture
|
||||
run_with_log FRANK_DRAGOON_NPU_DIR=/elsewhere/npu
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"/elsewhere/npu"*"outside the payload dir"* ]]
|
||||
[[ "$output" != *"--device"* ]]
|
||||
}
|
||||
|
||||
@test "the lane refuses a model that is not staged and names the fetch script" {
|
||||
# The engine cannot fetch these at boot here (/models is read-only and
|
||||
# HALOGEN_DOWNLOAD stays unset), so an un-staged name must stop the unit
|
||||
# before podman runs, with the command that fixes it.
|
||||
podman_stub
|
||||
good_env
|
||||
npu_fixture
|
||||
rm -rf "$TEST_TMP/npu/qwen3-embedding-0.6b"
|
||||
run_with_log
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"qwen3-embedding-0.6b"*"not staged"* ]]
|
||||
[[ "$output" == *"download-frank-dragoon-npu.sh"* ]]
|
||||
}
|
||||
|
||||
@test "the lane refuses without the accel node and says what to check" {
|
||||
podman_stub
|
||||
good_env
|
||||
npu_fixture
|
||||
rm "$TEST_TMP/accel0"
|
||||
run_with_log
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"$TEST_TMP/accel0 missing"* ]]
|
||||
[[ "$output" == *"iommu=pt"* ]]
|
||||
}
|
||||
|
||||
@test "the lane refuses an NPU node the container namespace could not open, and names the rule" {
|
||||
# Stock Debian mode is root:render 0660. The engine's own first words on such
|
||||
# a node are "this process cannot open it (crw-rw---- nobody:nogroup)" and no
|
||||
# flag fixes a mode an unmapped gid cannot cross (keep-groups measured dead
|
||||
# 2026-10-19), so the unit refuses instead of crash-looping 189 times.
|
||||
podman_stub
|
||||
good_env
|
||||
npu_fixture
|
||||
chmod 660 "$TEST_TMP/accel0"
|
||||
run_with_log
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"$TEST_TMP/accel0 is 660"* ]]
|
||||
[[ "$output" == *"98-halogen-npu-access.rules"* ]]
|
||||
[[ "$output" != *"--device"* ]]
|
||||
}
|
||||
|
||||
@test "the NPU access rule is the accel counterpart of the GPU compute rule" {
|
||||
[ -f "$NPU_RULE" ]
|
||||
grep -qE '^KERNEL=="accel\[0-9\]\*", SUBSYSTEM=="accel", MODE="0666"$' "$NPU_RULE"
|
||||
# The deployer installs the rule and re-triggers the node BEFORE it trusts
|
||||
# the mode, so a machine that never had it self-heals instead of refusing.
|
||||
grep -qE '98-halogen-npu-access\.rules' "$DEPLOYER"
|
||||
grep -qE 'udevadm control --reload-rules' "$DEPLOYER"
|
||||
grep -qE 'udevadm trigger --sysname-match=' "$DEPLOYER"
|
||||
}
|
||||
|
||||
@test "the lane refuses when an XRT NPU library is missing" {
|
||||
podman_stub
|
||||
good_env
|
||||
npu_fixture
|
||||
rm "$TEST_TMP/xrt/libxrt_driver_xdna.so.2"
|
||||
run_with_log
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"libxrt_driver_xdna.so.2 missing"* ]]
|
||||
}
|
||||
|
||||
@test "the lane refuses while the fabric clock is moving" {
|
||||
# The engine itself refuses to start in this state (GPU+NPU co-run hangs
|
||||
# Strix Halo while FCLK changes speed) but cannot see host sysfs from
|
||||
# inside the container — the proof has to happen on the host side.
|
||||
podman_stub
|
||||
good_env
|
||||
npu_fixture
|
||||
printf 'auto\n' > "$TEST_TMP/sys/class/drm/card1/device/power_dpm_force_performance_level"
|
||||
run_with_log
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"not held"* ]]
|
||||
[[ "$output" == *"sudo halogen-fabric-clock hold"* ]]
|
||||
}
|
||||
|
||||
@test "the lane refuses when no fabric-clock control can be read" {
|
||||
# A host whose sysfs exposes nothing cannot PROVE the hold, and a silent
|
||||
# pass here is a boot that dies in the engine instead.
|
||||
podman_stub
|
||||
good_env
|
||||
npu_fixture
|
||||
rm -rf "$TEST_TMP/sys/class/drm"
|
||||
run_with_log
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"cannot prove"* ]]
|
||||
}
|
||||
|
||||
@test "the lane also passes when the clock is held at the high level" {
|
||||
# The vendor's status accepts `high` as well as `manual` + the top fclk
|
||||
# level; refusing a correctly-held host would block a deploy for nothing.
|
||||
podman_stub
|
||||
good_env
|
||||
npu_fixture
|
||||
printf 'high\n' > "$TEST_TMP/sys/class/drm/card1/device/power_dpm_force_performance_level"
|
||||
run_with_log
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"-e HALOGEN_NPU_MODELS=qwen3-embedding-0.6b"* ]]
|
||||
}
|
||||
|
||||
@test "the deployer gates the same lane artifacts and installs the fabric-clock unit" {
|
||||
grep -qE 'LLM_DRAGOON_NPU_MODELS' "$DEPLOYER"
|
||||
grep -qE 'install -m 0755 dankclaw/host/halogen-fabric-clock' "$DEPLOYER"
|
||||
grep -qE 'install -m 0644 dankclaw/host/halogen-fabric-clock\.service' "$DEPLOYER"
|
||||
grep -qE 'halogen-fabric-clock status' "$DEPLOYER"
|
||||
grep -qE 'download-frank-dragoon-npu\.sh' "$DEPLOYER"
|
||||
# The lane's own health gate is a real embedding, not a 200 on /health.
|
||||
grep -qE '/v1/embeddings' "$DEPLOYER"
|
||||
# Parity with the serve gate: a staging dir outside the payload dir is
|
||||
# refused before the restart, not left to runc.
|
||||
grep -qE 'outside the payload dir' "$DEPLOYER"
|
||||
}
|
||||
|
||||
@test "the fabric-clock host unit is the vendor shape and boots on its own" {
|
||||
[ -x "$HOST_UNIT" ]
|
||||
grep -qE '^ (hold\)|release\)|status\))' "$HOST_UNIT"
|
||||
grep -qE 'pp_dpm_fclk' "$HOST_UNIT"
|
||||
grep -qE '^Type=oneshot$' "$HOST_SERVICE"
|
||||
grep -qE '^WantedBy=.*multi-user\.target' "$HOST_SERVICE"
|
||||
# A resume hands the clock back to the driver, so the unit must re-run then.
|
||||
grep -qE '^WantedBy=.*suspend\.target' "$HOST_SERVICE"
|
||||
grep -qE '^ExecStart=.*/usr/local/sbin/halogen-fabric-clock hold$' "$HOST_SERVICE"
|
||||
}
|
||||
|
||||
@test "the NPU fetch script pins the vendor catalog by size and sha256" {
|
||||
# XDNA2 runs only AIE-compiled programs, so the vendor's per-model repo is
|
||||
# the only source; the engine re-checks size+sha256 against its own catalog
|
||||
# at boot, so this manifest has to agree with it or the boot refuses.
|
||||
[ -x "$NPU_FETCH" ]
|
||||
grep -qE 'curl -fL' "$NPU_FETCH"
|
||||
grep -qE 'qwen3-embedding-0\.6b\.hnpw 813032248 [0-9a-f]{64}' "$NPU_FETCH"
|
||||
grep -qE 'sha256sum' "$NPU_FETCH"
|
||||
# curl -f alone is not enough on Hugging Face: the resolve URL redirects
|
||||
# and curl drops Range across it (the sibling 62 GiB fetch, 2026-10-18).
|
||||
grep -qE 'url_effective' "$NPU_FETCH"
|
||||
}
|
||||
|
||||
@test "the container network is podman's default when unnamed and the app bridge when named" {
|
||||
# The app-to-endpoint leg hangs on this: pasta is a private netns nothing
|
||||
# else can join (podman refuses a second network, the published port is
|
||||
# refused from app containers, and host.containers.internal is link-local,
|
||||
# which open-notebook refuses to store), so reaching the NPU endpoints by
|
||||
# name means the container lives on dankclaw_default. Both arms are
|
||||
# observable in the command podman would run.
|
||||
podman_stub
|
||||
good_env
|
||||
run_with_log FRANK_DRAGOON_NETWORK=
|
||||
[ "$status" -eq 0 ]
|
||||
! grep -q -- '--network' "$TEST_TMP/serve.log"
|
||||
run_with_log FRANK_DRAGOON_NETWORK=dankclaw_default
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"--network dankclaw_default"* ]]
|
||||
# The deployer refuses a network that does not exist rather than starting a
|
||||
# unit that restart-loops on it.
|
||||
grep -qE 'podman network exists' "$DEPLOYER"
|
||||
grep -qE '^FRANK_DRAGOON_NETWORK=dankclaw_default$' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@@ -41,13 +41,17 @@ service_section() {
|
||||
}
|
||||
|
||||
@test "every observability unit is boot-persistent" {
|
||||
# Generated units cannot be enabled on this host's podman 5.x v1 quadlet
|
||||
# generator; [Install] + linger is the entire boot-persistence story.
|
||||
# Generated units cannot be enabled on this host's podman quadlet generator;
|
||||
# [Install] + linger is the boot-persistence story for the rootless five.
|
||||
# Alloy is a ROOT system unit, so its [Install] targets multi-user.target.
|
||||
local svc
|
||||
for svc in "${UNITS[@]}"; do
|
||||
[[ "$svc" == alloy ]] && continue
|
||||
grep -q '^WantedBy=default.target$' "$UNIT_DIR/$svc.container" \
|
||||
|| { echo "$svc is not boot-persistent"; return 1; }
|
||||
done
|
||||
grep -q '^WantedBy=multi-user.target$' "$UNIT_DIR/alloy.container" \
|
||||
|| { echo "alloy (root unit) is not boot-persistent"; return 1; }
|
||||
}
|
||||
|
||||
@test "every observability unit is self-healing" {
|
||||
@@ -140,15 +144,20 @@ service_section() {
|
||||
grep -q 'reverse_proxy prometheus:9090' "$REPO_ROOT/dankclaw/Caddyfile"
|
||||
grep -q 'reverse_proxy loki:3100' "$REPO_ROOT/dankclaw/Caddyfile"
|
||||
local svc
|
||||
for svc in prometheus grafana loki alloy; do
|
||||
for svc in prometheus grafana loki; do
|
||||
grep -q "^ContainerName=$svc$" "$UNIT_DIR/$svc.container" \
|
||||
|| { echo "$svc unit renames its container"; return 1; }
|
||||
grep -q '^Network=dankclaw_default$' "$UNIT_DIR/$svc.container" \
|
||||
|| { echo "$svc is not on the network Caddy and its peers use"; return 1; }
|
||||
done
|
||||
# Alloy is not on that network: it is a root container on Network=host, the
|
||||
# only way to reach Loki's 127.0.0.1:3100 publish from outside a bridge.
|
||||
grep -q '^ContainerName=alloy$' "$UNIT_DIR/alloy.container"
|
||||
grep -q '^Network=host$' "$UNIT_DIR/alloy.container" \
|
||||
|| { echo "alloy must be Network=host to reach Loki on the host loopback"; return 1; }
|
||||
grep -q '"grafana:3000"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
|
||||
grep -q '"loki:3100"' "$REPO_ROOT/shared/prometheus/prometheus.yml"
|
||||
grep -q 'url = "http://loki:3100/loki/api/v1/push"' "$REPO_ROOT/shared/alloy/config.alloy"
|
||||
grep -q 'url = "http://127.0.0.1:3100/loki/api/v1/push"' "$REPO_ROOT/shared/alloy/config.alloy"
|
||||
}
|
||||
|
||||
@test "every observability unit has a health check" {
|
||||
@@ -239,12 +248,14 @@ service_section() {
|
||||
}
|
||||
|
||||
@test "startup order follows the data dependencies" {
|
||||
# Grafana is useless without Prometheus and Alloy drops logs without Loki;
|
||||
# both pairs are expressed as systemd ordering, not launch luck.
|
||||
# Grafana is useless without Prometheus (both rootless, same manager). Alloy
|
||||
# used to order on loki.service; as a ROOT unit it lives on the SYSTEM
|
||||
# manager and cannot order against a rootless user unit, so it depends only
|
||||
# on the network and retries its push until Loki answers.
|
||||
grep -q 'After=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
|
||||
grep -q 'Wants=network-online.target prometheus.service' "$UNIT_DIR/grafana.container"
|
||||
grep -q 'After=network-online.target loki.service' "$UNIT_DIR/alloy.container"
|
||||
grep -q 'Wants=network-online.target loki.service' "$UNIT_DIR/alloy.container"
|
||||
grep -q '^After=network-online.target$' "$UNIT_DIR/alloy.container"
|
||||
! grep -q 'loki.service' "$UNIT_DIR/alloy.container"
|
||||
}
|
||||
|
||||
@test "the deployer installs, restarts and health-checks each unit" {
|
||||
@@ -281,6 +292,24 @@ PAIRS
|
||||
grep -q 'build_compose_image alloy' "$REMOTE_DEPLOY"
|
||||
}
|
||||
|
||||
@test "alloy is a root host-network unit that reads every journal" {
|
||||
# The whole point of the root cutover: a rootless container cannot open the
|
||||
# 0640 journal files. Pin the properties that make the pipeline carry data.
|
||||
local u="$UNIT_DIR/alloy.container"
|
||||
grep -q '^Network=host$' "$u"
|
||||
grep -q '^Volume=/etc/machine-id:/etc/machine-id:ro$' "$u" \
|
||||
|| { echo "alloy missing the load-bearing machine-id mount"; return 1; }
|
||||
grep -q '^Volume=/var/log/journal:/var/log/journal:ro$' "$u"
|
||||
grep -q '^Volume=/run/log/journal:/run/log/journal:ro$' "$u"
|
||||
grep -q -- '--server.http.listen-addr=127.0.0.1:12345' "$u"
|
||||
# Deployer: system install + root image mirror, not the user quadlet loop.
|
||||
grep -q '^deploy_alloy() {' "$REMOTE_DEPLOY"
|
||||
grep -q 'install_system_quadlet dankclaw/observability/quadlet alloy' "$REMOTE_DEPLOY"
|
||||
grep -q '/etc/containers/systemd/' "$REMOTE_DEPLOY"
|
||||
grep -q 'mirror_image_to_root localhost/franksdank/alloy:latest' "$REMOTE_DEPLOY"
|
||||
grep -q 'alloy) deploy_alloy ;;' "$REMOTE_DEPLOY"
|
||||
}
|
||||
|
||||
@test "no dankclaw compose file competes with the quadlet units" {
|
||||
# Two owners for one container name is how a compose run resurrects a
|
||||
# stale duplicate next to the quadlet-managed one. This file was retired on
|
||||
@@ -313,33 +342,33 @@ PAIRS
|
||||
done
|
||||
}
|
||||
|
||||
@test "prometheus scrapes every LLM endpoint that exports metrics" {
|
||||
# The LLM endpoints are the services whose behaviour this fleet argues about
|
||||
# most (KV pressure, prefix-cache reuse, queue depth). Both Dankclaw
|
||||
# endpoints export the llamacpp:* family. frank-archon LEFT the set
|
||||
# 2026-10-05: the FreeToken PR587 ROCm engine has NO /metrics endpoint —
|
||||
# scraping it was a structural never-green (404), so the job was deleted
|
||||
# rather than left vacuous (the Lucebox DS4.1 precedent, 2026-09-26).
|
||||
# The two Dankclaw endpoints are HOST processes reached through
|
||||
# host.containers.internal: this Prometheus is a bridge-networked
|
||||
# container, and on this netavark host a container can only reach the
|
||||
# host's LAN IP at a PUBLISHED port — a host-process bind like
|
||||
# llama-server's 0.0.0.0:7770 is refused at 192.168.0.143:7770 while it
|
||||
# answers at 169.254.1.2:7770 (why frank-zealot stayed red by construction
|
||||
# for days until the 2026-09-20 /targets audit).
|
||||
@test "prometheus scrapes every LLM endpoint, in its real metric namespace" {
|
||||
# Every LLM endpoint that exports /metrics must have a scrape job. As of
|
||||
# 2026-10-05 there are TWO runtime namespaces scraped:
|
||||
# * llamacpp:* — frank-zealot (llama.cpp) and frank-dragoon (halogen,
|
||||
# re-exporting the llama.cpp names), both root host processes on
|
||||
# Dankclaw reached via host.containers.internal — this Prometheus is a
|
||||
# bridge container and a host-process bind is refused at the LAN IP,
|
||||
# which is why frank-zealot read red by construction for days.
|
||||
# * vllm:* — frank-archon (radiance) on Dankserver, a different machine,
|
||||
# so plain LAN routing reaches 192.168.0.235:7770. It RE-JOINED the
|
||||
# scrape set with radiance; the old "job deleted, exports no /metrics"
|
||||
# note belonged to the retired FreeToken engine and is false today, so
|
||||
# this seam now asserts archon IS scraped (a bare `! grep` here was
|
||||
# set -e-exempt and never enforced the opposite).
|
||||
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
||||
grep -q '"host.containers.internal:7771"' "$cfg" || { echo "frank-zealot not scraped"; return 1; }
|
||||
grep -q '"host.containers.internal:7770"' "$cfg" || { echo "frank-dragoon not scraped"; return 1; }
|
||||
! grep -q '"192.168.0.235:7770"' "$cfg"
|
||||
grep -q '"host.containers.internal:7770,192.168.0.235:7770"' "$cfg" \
|
||||
&& { echo "frank-archon folded into a Dankclaw gateway target"; return 1; }
|
||||
for job in frank-zealot frank-dragoon; do
|
||||
grep -q '"192.168.0.235:7770"' "$cfg" || { echo "frank-archon not scraped"; return 1; }
|
||||
for job in frank-zealot frank-dragoon frank-archon; do
|
||||
grep -q "job_name: \"$job\"" "$cfg" || { echo "no scrape job for $job"; return 1; }
|
||||
done
|
||||
! grep -q 'job_name: "frank-archon"' "$cfg"
|
||||
# None of them is reached through a guarded vhost: Prometheus cannot solve
|
||||
# an Anubis challenge, so a franksdank.cloud target is red by construction.
|
||||
! grep -qE '^\s*-\s*"?https?://[^"]*franksdank' "$cfg"
|
||||
# No LLM endpoint is reached through a guarded vhost: Prometheus cannot
|
||||
# solve an Anubis challenge, so a franksdank.cloud target is red by
|
||||
# construction.
|
||||
grep -qE '^[[:space:]]*-[[:space:]]*"?https?://[^"]*franksdank' "$cfg" \
|
||||
&& { echo "a scrape target points at a guarded vhost"; return 1; }
|
||||
true
|
||||
}
|
||||
|
||||
@test "grafana provisioning is declared in git and installed by the deployer" {
|
||||
@@ -378,28 +407,44 @@ names = [r.get('alert') or r.get('record') for g in d['groups'] for r in g['rule
|
||||
for want in ('job:llm_prefix_cache_reuse:rate5m',
|
||||
'job:llm_spec_accept_length:rate5m',
|
||||
'LlmRequestsQueueing', 'LlmPrefixCacheCollapse',
|
||||
'LlmSpecDecodeOff', 'LlmEndpointDown'):
|
||||
'LlmSpecDecodeOff', 'LlmEndpointDown',
|
||||
'job:vllm_ttft_p95:rate5m', 'VllmRequestsQueueing',
|
||||
'VllmPrefixCacheCollapse', 'VllmPreempting', 'VllmTtftSlow'):
|
||||
assert want in names, want
|
||||
# Every rule must be keyed to a series that actually exists. A rule naming a
|
||||
# series nobody exports evaluates nothing forever and errors never — the
|
||||
# failure this seam exists for. Since the 2026-09-26 cutover the only scraped
|
||||
# LLM runtime is llama.cpp (frank-zealot, frank-dragoon); the vllm: family left
|
||||
# with frank-medium's radiance runtime and must not survive in any expr.
|
||||
# failure this seam exists for. Two runtimes are scraped in two namespaces:
|
||||
# llama.cpp/halogen (llamacpp:*) for frank-zealot and frank-dragoon, and
|
||||
# radiance (vllm:*) for frank-archon. Both families were read straight off the
|
||||
# endpoints' /metrics; anything else is a guess.
|
||||
exprs = [r.get('expr', '') for g in d['groups'] for r in g['rules']]
|
||||
# llama.cpp families measured from /metrics on 2026-09-16 and 2026-09-20
|
||||
# (frank-zealot, frank-dragoon). Anything else is a guess.
|
||||
live = {'llamacpp:prompt_tokens_total', 'llamacpp:prompt_tokens_cached_total',
|
||||
'llamacpp:requests_deferred', 'llamacpp:requests_processing',
|
||||
'llamacpp:spec_decode_num_drafts_total',
|
||||
'llamacpp:spec_decode_num_accepted_tokens_total',
|
||||
'llamacpp:spec_decode_num_draft_tokens_total',
|
||||
'llamacpp:tokens_predicted_total',
|
||||
'llamacpp:predicted_tokens_seconds', 'llamacpp:prompt_tokens_seconds',
|
||||
'llamacpp:n_decode_total', 'llamacpp:n_busy_slots_per_decode'}
|
||||
live = {
|
||||
# llama.cpp / halogen, measured from /metrics 2026-09-16 and 2026-09-20.
|
||||
'llamacpp:prompt_tokens_total', 'llamacpp:prompt_tokens_cached_total',
|
||||
'llamacpp:requests_deferred', 'llamacpp:requests_processing',
|
||||
'llamacpp:spec_decode_num_drafts_total',
|
||||
'llamacpp:spec_decode_num_accepted_tokens_total',
|
||||
'llamacpp:spec_decode_num_draft_tokens_total',
|
||||
'llamacpp:tokens_predicted_total',
|
||||
'llamacpp:predicted_tokens_seconds', 'llamacpp:prompt_tokens_seconds',
|
||||
'llamacpp:n_decode_total', 'llamacpp:n_busy_slots_per_decode',
|
||||
# radiance (frank-archon), measured from /metrics 2026-10-20.
|
||||
'vllm:num_requests_waiting', 'vllm:gpu_prefix_cache_hit_rate',
|
||||
'vllm:num_preemptions_total', 'vllm:prompt_tokens_total',
|
||||
'vllm:time_to_first_token_seconds_bucket',
|
||||
}
|
||||
used = {m for e in exprs
|
||||
for m in re.findall(r'(?:llamacpp|vllm):[a-z_0-9]+', e)}
|
||||
assert used <= live, used - live
|
||||
assert not any(':vllm' in e for e in exprs), 'vllm: series in rules but no vllm endpoint is scraped'
|
||||
# The two namespaces must not bleed across groups: a llamacpp: selector
|
||||
# evaluates nothing for the radiance job and vice versa, so each group may
|
||||
# reference only its own family.
|
||||
per = {g['name']: {m.split(':')[0] for r in g['rules']
|
||||
for m in re.findall(r'(?:llamacpp|vllm):[a-z_0-9]+', r.get('expr', ''))}
|
||||
for g in d['groups']}
|
||||
for name, fams in per.items():
|
||||
allowed = {'llamacpp'} if name == 'llm-fleet' else {'vllm'}
|
||||
assert fams <= allowed, (name, fams)
|
||||
print('rules ok:', len(names))
|
||||
"
|
||||
# A rule file that nothing loads evaluates nothing, and Prometheus reports
|
||||
|
||||
@@ -290,7 +290,9 @@ vhost = sys.argv[2]
|
||||
cards = [s for s in data if s["name"] == "open-notebook"]
|
||||
assert len(cards) == 1, f"expected exactly one open-notebook card, got {len(cards)}"
|
||||
card = cards[0]
|
||||
assert card["category"] == "user" and card["section"] == "AI", card
|
||||
# category moved to "admin": single shared password, no per-user accounts, so it
|
||||
# is positioned as an admin-side tool, not a user-facing app (see the vault page).
|
||||
assert card["category"] == "admin" and card["section"] == "AI", card
|
||||
assert card["probe"] == "127.0.0.1:8502", card["probe"]
|
||||
assert card["method"] == "http"
|
||||
# Deployed and guarded: the card is live and its link is the vhost, so the
|
||||
|
||||
Reference in New Issue
Block a user