fix(fleet): every LLM root unit restarts on a voluntary exit; parity seam gates it
frank-zealot shipped Restart=on-failure while its two siblings shipped always. exit 0 is a CLEAN stop to systemd: on-failure leaves the endpoint dark and the unit GREEN until the next boot — and frank-zealot is the compaction hub for all three endpoints, so its quiet death degrades every conversation in the fleet. tests/llm_units.bats gains the fleet parity seam (Restart=always, RestartSec 1..20, WantedBy=multi-user.target across scripts/frank-*.service); the per-unit seams drop their weaker Restart greps so the policy lives in one place. Runbook unit line updated.
This commit is contained in:
1 parent
ab80c1366d
commit
156ea71879
5 files changed
+50
-8
No files matched your search
@@ -80,3 +80,38 @@ unit_repo_path() {
|
||||
}
|
||||
done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
|
||||
}
|
||||
|
||||
@test "every LLM root unit is self-healing against a voluntary exit" {
|
||||
# Restart=always, not on-failure. Every endpoint here is a container
|
||||
# runner (podman run) or a server whose front end can exit 0 on a
|
||||
# control-plane event; systemd treats exit 0 as a CLEAN stop and
|
||||
# on-failure leaves the endpoint dark until the next boot, with the unit
|
||||
# reporting success. frank-archon learned this on 2026-10-04; the fleet
|
||||
# law is that no LLM unit may ship the weaker policy.
|
||||
local unit
|
||||
while IFS= read -r unit; do
|
||||
grep -qx 'Restart=always' "$unit" || {
|
||||
echo "$(basename "$unit"): $(grep '^Restart=' "$unit") (must be always)"
|
||||
return 1
|
||||
}
|
||||
done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
|
||||
}
|
||||
|
||||
@test "every LLM root unit restarts promptly and starts at boot" {
|
||||
# RestartSec=10 is the fleet's cadence: fast enough that a transient
|
||||
# fault is invisible to traffic, slow enough that a hard fault does not
|
||||
# spin the engine (each radiance boot re-pins ~147 GiB).
|
||||
# WantedBy=multi-user.target is boot persistence for a ROOT unit —
|
||||
# default.target is the USER-manager's target and means nothing here.
|
||||
local unit
|
||||
while IFS= read -r unit; do
|
||||
grep -qE '^RestartSec=([1-9]|1[0-9]|20)$' "$unit" || {
|
||||
echo "$(basename "$unit"): no RestartSec in 1..20"
|
||||
return 1
|
||||
}
|
||||
grep -qx 'WantedBy=multi-user.target' "$unit" || {
|
||||
echo "$(basename "$unit"): $(grep '^WantedBy=' "$unit")"
|
||||
return 1
|
||||
}
|
||||
done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
|
||||
}
|
||||
Reference in new issue
Block a user