fix(fleet): every LLM root unit restarts on a voluntary exit; parity seam gates it
frank-zealot shipped Restart=on-failure while its two siblings shipped always. exit 0 is a CLEAN stop to systemd: on-failure leaves the endpoint dark and the unit GREEN until the next boot — and frank-zealot is the compaction hub for all three endpoints, so its quiet death degrades every conversation in the fleet. tests/llm_units.bats gains the fleet parity seam (Restart=always, RestartSec 1..20, WantedBy=multi-user.target across scripts/frank-*.service); the per-unit seams drop their weaker Restart greps so the policy lives in one place. Runbook unit line updated.
This commit is contained in:
1 parent
ab80c1366d
commit
156ea71879
5 files changed
+50
-8
No files matched your search
@@ -346,8 +346,9 @@ $(cat "$TEST_TMP/serve.log")"
|
||||
}
|
||||
|
||||
@test "the deployed unit keeps the fleet's restart and boot-persistence shape" {
|
||||
grep -qE '^Restart=(always|on-failure)$' "$UNIT"
|
||||
grep -qE '^WantedBy=' "$UNIT"
|
||||
# Restart=always and the WantedBy value are the FLEET parity seam
|
||||
# (tests/llm_units.bats); what is endpoint-specific is which script the
|
||||
# unit runs, under whom, and what a stop must reap.
|
||||
grep -qE '^ExecStart=.*/serve-frank-dragoon\.sh$' "$UNIT"
|
||||
grep -qE '^User=dank$' "$UNIT"
|
||||
grep -qE '^LimitMEMLOCK=infinity$' "$UNIT"
|
||||
|
||||
@@ -85,10 +85,12 @@ run_stage() {
|
||||
[ -f "$ENV_FILE" ]
|
||||
}
|
||||
|
||||
@test "frank-zealot is boot-persistent, self-healing and unprivileged" {
|
||||
grep -q '^WantedBy=multi-user.target$' "$UNIT"
|
||||
grep -q '^Restart=' "$UNIT"
|
||||
grep -q '^RestartSec=' "$UNIT"
|
||||
@test "frank-zealot runs unprivileged with the memlock budget and its env file" {
|
||||
# Restart policy, RestartSec and WantedBy are the FLEET parity seam
|
||||
# (tests/llm_units.bats). What is endpoint-specific: CUDA pins its
|
||||
# staging buffers, so the memlock budget is load-bearing even though the
|
||||
# weights are copied into VRAM, and the unit must name the serve script
|
||||
# and the env file that sizes it.
|
||||
grep -q '^User=dank$' "$UNIT"
|
||||
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
|
||||
grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-zealot.sh$' "$UNIT"
|
||||
|
||||
@@ -80,3 +80,38 @@ unit_repo_path() {
|
||||
}
|
||||
done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
|
||||
}
|
||||
|
||||
@test "every LLM root unit is self-healing against a voluntary exit" {
|
||||
# Restart=always, not on-failure. Every endpoint here is a container
|
||||
# runner (podman run) or a server whose front end can exit 0 on a
|
||||
# control-plane event; systemd treats exit 0 as a CLEAN stop and
|
||||
# on-failure leaves the endpoint dark until the next boot, with the unit
|
||||
# reporting success. frank-archon learned this on 2026-10-04; the fleet
|
||||
# law is that no LLM unit may ship the weaker policy.
|
||||
local unit
|
||||
while IFS= read -r unit; do
|
||||
grep -qx 'Restart=always' "$unit" || {
|
||||
echo "$(basename "$unit"): $(grep '^Restart=' "$unit") (must be always)"
|
||||
return 1
|
||||
}
|
||||
done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
|
||||
}
|
||||
|
||||
@test "every LLM root unit restarts promptly and starts at boot" {
|
||||
# RestartSec=10 is the fleet's cadence: fast enough that a transient
|
||||
# fault is invisible to traffic, slow enough that a hard fault does not
|
||||
# spin the engine (each radiance boot re-pins ~147 GiB).
|
||||
# WantedBy=multi-user.target is boot persistence for a ROOT unit —
|
||||
# default.target is the USER-manager's target and means nothing here.
|
||||
local unit
|
||||
while IFS= read -r unit; do
|
||||
grep -qE '^RestartSec=([1-9]|1[0-9]|20)$' "$unit" || {
|
||||
echo "$(basename "$unit"): no RestartSec in 1..20"
|
||||
return 1
|
||||
}
|
||||
grep -qx 'WantedBy=multi-user.target' "$unit" || {
|
||||
echo "$(basename "$unit"): $(grep '^WantedBy=' "$unit")"
|
||||
return 1
|
||||
}
|
||||
done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
|
||||
}
|
||||
Reference in new issue
Block a user