diff --git a/scripts/frank-primary.service b/scripts/frank-primary.service index c1f4b1d..10b139c 100644 --- a/scripts/frank-primary.service +++ b/scripts/frank-primary.service @@ -53,8 +53,11 @@ TimeoutStartSec=1800 # stopped), so bench windows and deploys are unaffected. Restart=always RestartSec=10 -# Do not let the start-rate limiter give up after a crash loop. -StartLimitIntervalSec=0 +# ExecStop allows the engine 120 s to drain (`podman stop -t 120`); without an +# explicit TimeoutStopSec systemd's 90 s default cuts the stop short, kills it, +# and records 'Failed with result timeout' (measured 2026-10-01 13:27:17). +# Must exceed the podman grace with margin for the engine's own shutdown. +TimeoutStopSec=180 # The engine pins tens of GiB of host RAM (expert host slabs); it is the # last process on this host that should be chosen by the kernel OOM killer. OOMScoreAdjust=-500 diff --git a/tests/frank_primary_unit.bats b/tests/frank_primary_unit.bats index f0fc669..9f07fdd 100755 --- a/tests/frank_primary_unit.bats +++ b/tests/frank_primary_unit.bats @@ -47,6 +47,17 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" # (measured 2026-10-01 — the endpoint stayed down until a human noticed). grep -q '^Restart=always$' "$UNIT" ! grep -q '^Restart=on-failure$' "$UNIT" + # Exiting the engine takes as long as vLLM's drain: ExecStop runs + # `podman stop -t 120`, so the stop timeout MUST exceed 120 s or systemd + # kills the stop and records 'Failed with result timeout' (measured + # 2026-10-01). Guard the relationship, not just the literal. + stop_t=$(awk -F= '/^TimeoutStopSec=/{print $2}' "$UNIT") + [ -n "$stop_t" ] && [ "$stop_t" -gt 120 ] + grep -q '^ExecStop=.*podman stop -t 120' "$UNIT" + # StartLimitIntervalSec belongs to [Unit]; a copy in [Service] is an + # 'Unknown key' warning systemd logs on every load. + [ "$(grep -c '^StartLimitIntervalSec=' "$UNIT")" = "1" ] + awk '/^\[Service\]/{in_svc=1} /^StartLimitIntervalSec=/{if(in_svc) exit 1}' "$UNIT" grep -q '^RestartSec=' "$UNIT" grep -q '^User=dank$' "$UNIT" grep -q '^LimitMEMLOCK=infinity$' "$UNIT"