diff --git a/scripts/frank-archon.service b/scripts/frank-archon.service index 5a5553c..47811ee 100644 --- a/scripts/frank-archon.service +++ b/scripts/frank-archon.service @@ -35,6 +35,13 @@ Group=dank LimitMEMLOCK=infinity Environment="FRANK_ARCHON_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-archon.env" ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-archon.sh +# The `+` prefix runs this as root even though the service is User=dank — the +# ttm pages cap is a root-owned knob and the pinned-RAM ledger (~147 GiB) is +# unusable below it. After a reboot the knob is back at its default (half of +# RAM), so the raise must be part of the start, not a note in a runbook: the +# 2026-10-12 power failure reset it and the endpoint refused to boot 6,919 +# times. The serve script re-verifies the cap after this hook. +ExecStartPre=+/home/dank/serverAppConfigs/scripts/ttm-preflight.sh --raise # systemd stop: wait alone cannot stop the container child; stop it # explicitly. No graceful drain contract is claimed (the OpenAI front end # holds no queue to drain — in-flight decodes are lost, which a stop diff --git a/scripts/ttm-preflight.sh b/scripts/ttm-preflight.sh index 2b15ce2..4567837 100755 --- a/scripts/ttm-preflight.sh +++ b/scripts/ttm-preflight.sh @@ -42,15 +42,24 @@ ttm_cap_mib() { echo "${cur:-0}" } -# Writes $1 MiB (as 4 KiB pages) into every writable knob present. Failures are +# Writes $1 MiB (as 4 KiB pages) into every knob present. Failures are # swallowed: the caller re-reads the cap, and a write the kernel refused shows # up there (ttm refuses a limit below the pages it has already pinned). +# +# PRIVILEGE, measured the hard way: the knob is root-owned 0644 and the service +# process runs as User=dank, so `-w` is FALSE there and a plain echo fails. The +# canonical privileged path is the unit's ExecStartPre with the `+` prefix +# (systemd runs it as root even under User=); `sudo -n` is the fallback for a +# human running the gate from a shell (passwordless sudo on both hosts). ttm_raise_cap() { local pages=$(( $1 * 256 )) p for p in "$FRANK_ARCHON_TTM_SYS_DIR/pages_limit" \ "$FRANK_ARCHON_TTM_SYS_DIR/pages_max"; do + [ -e "$p" ] || continue if [ -w "$p" ]; then echo "$pages" > "$p" 2>/dev/null || true + elif command -v sudo >/dev/null 2>&1; then + sudo -n sh -c "echo $pages > '$p'" >/dev/null 2>&1 || true fi done } @@ -74,3 +83,22 @@ ttm_ensure_cap() { fi echo "frank-archon: ttm pages cap ${cap} MiB clears the pinned-ledger floor $FRANK_ARCHON_TTM_MIN_MIB" >&2 } + +# Executed (not sourced) it is the privileged pre-start hook: systemd runs this +# as root from the unit's `ExecStartPre=+…`, where the knob write is legal and +# the `Restart=always` loop can never outrun it. +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + set -euo pipefail + case "${1:-}" in + --raise) + TTM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + # shellcheck source=/dev/null + source "${FRANK_ARCHON_ENV_FILE:-$TTM_DIR/frank-archon.env}" + ttm_ensure_cap + ;; + *) + echo "usage: ${0##*/} --raise (raise the ttm pages cap, then verify)" >&2 + exit 2 + ;; + esac +fi diff --git a/tests/frank_archon_unit.bats b/tests/frank_archon_unit.bats index cd3d5d1..da76b05 100755 --- a/tests/frank_archon_unit.bats +++ b/tests/frank_archon_unit.bats @@ -33,6 +33,16 @@ BUILD="$REPO_ROOT/scripts/convert-swift15-w4dn8.sh" HFGET="$REPO_ROOT/scripts/hfget.sh" TEMPLATE="$REPO_ROOT/scripts/qwen38-froggeric-v22.5.jinja" +# A PATH with `cat` (the gate reads knobs with it) and NO `sudo`, so the +# privileged fallback cannot rescue a write the test means to fail. +nosudo_bin() { + local bin="$BATS_TEST_TMPDIR/nosudo-bin" + mkdir -p "$bin" + ln -sf "$(command -v cat)" "$bin/cat" + ln -sf "$(command -v bash)" "$bin/bash" + echo "$bin" +} + @test "frank-archon unit, serve script, env, fetch and build scripts and template exist" { [ -f "$UNIT" ] [ -x "$SERVE" ] @@ -298,8 +308,11 @@ TEMPLATE="$REPO_ROOT/scripts/qwen38-froggeric-v22.5.jinja" echo 32253039 > "$dir/pages_limit" chmod 444 "$dir/pages_limit" # the write refused: cap cannot move - run bash -c '. "$1"; ttm_ensure_cap' _ "$TTM_LIB" - [ "$status" -ne 0 ] + # The refusal case must not have `sudo` on PATH: the gate's privileged + # fallback runs `sudo -n`, and on a passwordless host root can write a 444 + # file, which would silently test nothing. + run env PATH="$(nosudo_bin)" bash -c '. "$1"; ttm_ensure_cap' _ "$TTM_LIB" + [ "$status" -eq 1 ] [[ "$output" == *"ERROR: ttm pages cap is 125988 MiB"* ]] [ "$(cat "$dir/pages_limit")" = "32253039" ] } @@ -323,12 +336,35 @@ TEMPLATE="$REPO_ROOT/scripts/qwen38-froggeric-v22.5.jinja" [ "$output" = "117187" ] chmod 444 "$dir/pages_limit" "$dir/pages_max" # both writes refused - run bash -c '. "$1"; ttm_ensure_cap' _ "$TTM_LIB" - [ "$status" -ne 0 ] + run env PATH="$(nosudo_bin)" bash -c '. "$1"; ttm_ensure_cap' _ "$TTM_LIB" + [ "$status" -eq 1 ] [[ "$output" == *"ERROR: ttm pages cap is 117187 MiB"* ]] [ "$(cat "$dir/pages_max")" = "30000000" ] } +@test "the gate runs as a root pre-start hook, where the knob write is legal" { + # The knob is root-owned 0644 and the service runs as User=dank, so the + # gate CANNOT raise it from the service process. systemd's `+` prefix on + # ExecStartPre runs as root even under User= — the canonical privileged + # path — and the gate's own --raise mode is what that line calls. + grep -q '^ExecStartPre=+/home/dank/serverAppConfigs/scripts/ttm-preflight.sh --raise$' "$UNIT" + + # Behaviour of that entry point: sourced env values, knob raised, exit 0. + local dir="$BATS_TEST_TMPDIR/ttm3" envf="$BATS_TEST_TMPDIR/ttm3.env" + mkdir -p "$dir" + printf 'FRANK_ARCHON_TTM_MIN_MIB=150000\nFRANK_ARCHON_TTM_TARGET_MIB=204800\n' > "$envf" + echo 32253039 > "$dir/pages_limit" + run env FRANK_ARCHON_ENV_FILE="$envf" FRANK_ARCHON_TTM_SYS_DIR="$dir" \ + bash "$TTM_LIB" --raise + [ "$status" -eq 0 ] + [ "$(cat "$dir/pages_limit")" = "52428800" ] + + # Usage is a named failure, not a silent no-op. + run bash "$TTM_LIB" + [ "$status" -eq 2 ] + [[ "$output" == *"--raise"* ]] +} + @test "the serve script clears stale containers and self-heals via podman wait" { # A crashed unit can leave the container name registered forever after; # the legacy identities hold the same GPUs and die here too. Then block