fix(fleet): every LLM root unit restarts on a voluntary exit; parity seam gates it

frank-zealot shipped Restart=on-failure while its two siblings shipped
always. exit 0 is a CLEAN stop to systemd: on-failure leaves the endpoint
dark and the unit GREEN until the next boot — and frank-zealot is the
compaction hub for all three endpoints, so its quiet death degrades every
conversation in the fleet.

tests/llm_units.bats gains the fleet parity seam (Restart=always,
RestartSec 1..20, WantedBy=multi-user.target across scripts/frank-*.service);
the per-unit seams drop their weaker Restart greps so the policy lives in
one place. Runbook unit line updated.
This commit is contained in:
Frank committed 2026-10-13 22:04:48 -07:00
1 parent ab80c1366d
commit 156ea71879
5 files changed
+50 -8

No files matched your search

+35
View File
@@ -80,3 +80,38 @@ unit_repo_path() {
}
done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
}
@test "every LLM root unit is self-healing against a voluntary exit" {
# Restart=always, not on-failure. Every endpoint here is a container
# runner (podman run) or a server whose front end can exit 0 on a
# control-plane event; systemd treats exit 0 as a CLEAN stop and
# on-failure leaves the endpoint dark until the next boot, with the unit
# reporting success. frank-archon learned this on 2026-10-04; the fleet
# law is that no LLM unit may ship the weaker policy.
local unit
while IFS= read -r unit; do
grep -qx 'Restart=always' "$unit" || {
echo "$(basename "$unit"): $(grep '^Restart=' "$unit") (must be always)"
return 1
}
done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
}
@test "every LLM root unit restarts promptly and starts at boot" {
# RestartSec=10 is the fleet's cadence: fast enough that a transient
# fault is invisible to traffic, slow enough that a hard fault does not
# spin the engine (each radiance boot re-pins ~147 GiB).
# WantedBy=multi-user.target is boot persistence for a ROOT unit —
# default.target is the USER-manager's target and means nothing here.
local unit
while IFS= read -r unit; do
grep -qE '^RestartSec=([1-9]|1[0-9]|20)$' "$unit" || {
echo "$(basename "$unit"): no RestartSec in 1..20"
return 1
}
grep -qx 'WantedBy=multi-user.target' "$unit" || {
echo "$(basename "$unit"): $(grep '^WantedBy=' "$unit")"
return 1
}
done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
}