From c96741794a0a5c0ec15a2f9995ee48def4160683 Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 20 Oct 2026 11:04:41 -0700 Subject: [PATCH] live: heartbeat sync 2026-10-20T11:04:41-07:00 (Refs #11) --- .../fleet-status-daily-report/SKILL.md | 56 +++++++++++++++++++ .../state.json | 2 +- .../state.json | 2 +- .../.local/state/noctalia/settings.toml | 8 +-- 4 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 agent/managed-skills/fleet-status-daily-report/SKILL.md diff --git a/agent/managed-skills/fleet-status-daily-report/SKILL.md b/agent/managed-skills/fleet-status-daily-report/SKILL.md new file mode 100644 index 0000000..dba2c4d --- /dev/null +++ b/agent/managed-skills/fleet-status-daily-report/SKILL.md @@ -0,0 +1,56 @@ +--- +name: fleet-status-daily-report +description: "Answer \"status update / what happened yesterday\" for the dank fleet: git-window record on origin/main plus a live per-host census (endpoints, units, RAM, DNS), with the reboot-attribution check that stops false incident causes." +--- + +# Fleet status report (Dankclaw + Dankserver) + +Use for "what's the state of the fleet", "what happened yesterday", or a post-incident sweep. Read-only. +For one service's live state, prefer `fleet-service-runtime-census`; this is the fleet-wide sweep. + +## 1. The day's record (repo is the ledger) + +```bash +git fetch origin --quiet +git log --no-merges --since="YYYY-MM-DD 00:00" --until="YYYY-MM-DD+1 00:00" \ + --date=format:'%H:%M' --pretty='%ad %h %s' origin/main +git log --merges --since=... --pretty='%ad %h %s' --date=format:'%H:%M' origin/main # PR numbers +``` +Merge lines carry the PR number — that is the review trail; the non-merge log is what actually changed. +Group commits into streams (endpoint names lead the subject) rather than listing 20 subjects. + +## 2. Live census (parallel: one call per host + one endpoint sweep) + +Endpoint sweep from any LAN box — `/health` is honest on all four (archon binds only after full load): + +```bash +for u in 192.168.0.143:7770 192.168.0.143:7771 192.168.0.235:7770 192.168.0.143:7772; do + printf "%s %s\n" "$u" "$(curl -s -m 12 -w ' [%{http_code}]' http://$u/health | head -c 160)"; done +curl -s http://192.168.0.143:7770/v1/models # dragoon lists EVERY engine it serves, incl. NPU-lane models +``` + +Per host: `uptime -s; uptime -p`; `systemctl show -p ActiveEnterTimestamp -p NRestarts -p DropInPaths` +(`sudo` for the root LLM units, `--user` for the quadlets); `list-units --failed`; `free -h`; +`journalctl -k --since "" -p err`; the host's own engine log tail. + +## 3. Attribution rules (where reports go wrong) + +- **Never call a host reboot from a container's start timestamp.** Check `uptime -s` first; if uptime spans the + event, the trigger was a **container restart** — find it with + `journalctl --user -u .service --since … --until …` + `podman inspect --format '{{.State.StartedAt}}'`. + (This exact mistake mislabelled a 2026-10-19 Pi-hole incident as boot-caused; the law's remedy belongs after + *any* restart, which is a different action item.) +- `podman stats` is a 28-container wall — sort/limit it; RSS of the LLM tenants is what matters. +- This podman's inspect template has **no `.Config.PodName`, no `.Config.GroupAdd`, and `.HostConfig.Devices` + can read `[]` even when a quadlet `AddDevice=` worked**. Prove GPU use from the engine's own log + (`ggml_vulkan: Found 1 Vulkan devices: … Radeon 8060S (RADV GFX1151)`), not from inspect. +- Kernel lines: **count** occurrences (`journalctl -k --since '7 days' | grep -c "NETDEV WATCHDOG"`) before + calling a one-off a trend. +- Failed units: `systemctl show -p ExecMainExitTimestamp -p ExecMainStatus --value` separates + "broken since boot" from "broke yesterday" — and a nightly failure (e.g. logrotate) is a live risk, not noise. + +## 4. Report shape + +Stream table (what landed) → live-state table (what is true now) → **corrections to my own earlier record** → +open items that need Frank's decision, each with the evidence line and the one action. Flag anything that would +touch host config outside the repo as a decision, never a silent fix. diff --git a/agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/state.json b/agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/state.json index daf0441..b388373 100644 --- a/agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/state.json +++ b/agent/memories/sharpshooter/wt-20261019-020309-6d1fc1f-8knxmfcrfa1k/state.json @@ -1,6 +1,6 @@ { "v": 1, - "lastConsolidatedAt": 1792515715452, + "lastConsolidatedAt": 1792519315496, "lastResult": { "at": 1792433353188, "sessions": 1, diff --git a/agent/memories/sharpshooter/wt-20261019-125509-7c8a99d-me8scwe1t2nf/state.json b/agent/memories/sharpshooter/wt-20261019-125509-7c8a99d-me8scwe1t2nf/state.json index ddc688a..1c75ffb 100644 --- a/agent/memories/sharpshooter/wt-20261019-125509-7c8a99d-me8scwe1t2nf/state.json +++ b/agent/memories/sharpshooter/wt-20261019-125509-7c8a99d-me8scwe1t2nf/state.json @@ -1,6 +1,6 @@ { "v": 1, - "lastConsolidatedAt": 1792515641458, + "lastConsolidatedAt": 1792519241504, "lastResult": { "at": 1792452243238, "sessions": 1, diff --git a/dotfiles/noctalia/.local/state/noctalia/settings.toml b/dotfiles/noctalia/.local/state/noctalia/settings.toml index 06b223b..b7b08a5 100644 --- a/dotfiles/noctalia/.local/state/noctalia/settings.toml +++ b/dotfiles/noctalia/.local/state/noctalia/settings.toml @@ -275,13 +275,13 @@ transition = [ "fade" ] interval_seconds = 1200 [wallpaper.default] - path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_73qev3.jpg" + path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_v9w8r5.jpg" [wallpaper.last] - path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_73qev3.jpg" + path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_v9w8r5.jpg" [wallpaper.monitors.DP-1] - path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_9m7vvx.jpg" + path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_v9w8r5.jpg" [wallpaper.monitors.DP-2] path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_oxex69.jpg" @@ -290,7 +290,7 @@ transition = [ "fade" ] path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_v9e2w3.jpg" [wallpaper.monitors.eDP-1] - path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_73qev3.jpg" + path = "/home/blank/.local/share/noctalia/wallpapers/wallhaven_v9w8r5.jpg" [weather] unit = "imperial"