#!/usr/bin/env bash # ttm-preflight.sh — the TTM pages-cap gate for engines that PIN host RAM. # # Sourced, never executed. An engine that pins tens of GiB of host RAM (the # radiance expert pools + the PLE n-gram table) has its pinned pages charged # against the kernel's ttm page cap: /sys/module/ttm/parameters/pages_limit # (and pages_max on kernels that expose one). The cap function below takes the # MIN over the knobs the running kernel actually exposes — an absent knob is # skipped, never read as zero, because a zero would refuse a healthy cap. # # THE LAW THIS FILE EXISTS FOR: the cap is a RUNTIME knob. The kernel default is # half of RAM, so every raise is lost on reboot. On 2026-10-12 23:17 a power # failure on dankserver reset it 200 GiB -> 125,988 MiB, and frank-archon's # preflight refused the start — 6,919 crash-restarts over ~20 hours, with # `Restart=always` cheerfully re-running a refusal. A manual sysfs write is # therefore not an acceptable repair procedure: the gate raises the knob itself # (the units that need it run as root, and the knob is a 0644 root-owned module # parameter), re-reads, and only then refuses. # # Contract (set by the sourcing script's env file): # FRANK_ARCHON_TTM_MIN_MIB floor the pinned ledger needs; refuse below it # FRANK_ARCHON_TTM_TARGET_MIB value to raise the cap to when it is below the # floor (the recorded host ceiling — writing the # floor itself would leave the ledger no slack) # FRANK_ARCHON_TTM_SYS_DIR knob directory; defaults to the real sysfs # path, a test points it at a temp dir. # Pages are 4 KiB: pages = MiB x 256. : "${FRANK_ARCHON_TTM_SYS_DIR:=/sys/module/ttm/parameters}" # Prints the effective cap in MiB, 0 when no knob is readable. ttm_cap_mib() { local p v cur="" for p in "$FRANK_ARCHON_TTM_SYS_DIR/pages_limit" \ "$FRANK_ARCHON_TTM_SYS_DIR/pages_max"; do [ -r "$p" ] || continue v="$(cat "$p" 2>/dev/null)" || continue [ -n "$v" ] || continue v=$(( v * 4 / 1024 )) if [ "$v" -lt "${cur:-999999999}" ]; then cur=$v; fi done echo "${cur:-0}" } # Writes $1 MiB (as 4 KiB pages) into every knob present. Failures are # swallowed: the caller re-reads the cap, and a write the kernel refused shows # up there (ttm refuses a limit below the pages it has already pinned). # # PRIVILEGE, measured the hard way: the knob is root-owned 0644 and the service # process runs as User=dank, so `-w` is FALSE there and a plain echo fails. The # canonical privileged path is the unit's ExecStartPre with the `+` prefix # (systemd runs it as root even under User=); `sudo -n` is the fallback for a # human running the gate from a shell (passwordless sudo on both hosts). ttm_raise_cap() { local pages=$(( $1 * 256 )) p for p in "$FRANK_ARCHON_TTM_SYS_DIR/pages_limit" \ "$FRANK_ARCHON_TTM_SYS_DIR/pages_max"; do [ -e "$p" ] || continue if [ -w "$p" ]; then echo "$pages" > "$p" 2>/dev/null || true elif command -v sudo >/dev/null 2>&1; then sudo -n sh -c "echo $pages > '$p'" >/dev/null 2>&1 || true fi done } # Returns 0 when the cap clears FRANK_ARCHON_TTM_MIN_MIB, raising it first if # needed; returns 1 with a named refusal when it cannot be made to. ttm_ensure_cap() { local cap cap="$(ttm_cap_mib)" if [ "$cap" -lt "$FRANK_ARCHON_TTM_MIN_MIB" ]; then echo "frank-archon: ttm pages cap is ${cap} MiB, below the pinned-ledger" \ "floor $FRANK_ARCHON_TTM_MIN_MIB — raising the knob to" \ "$FRANK_ARCHON_TTM_TARGET_MIB MiB ($(( FRANK_ARCHON_TTM_TARGET_MIB * 256 )) pages)" >&2 ttm_raise_cap "$FRANK_ARCHON_TTM_TARGET_MIB" cap="$(ttm_cap_mib)" fi if [ "$cap" -lt "$FRANK_ARCHON_TTM_MIN_MIB" ]; then echo "ERROR: ttm pages cap is ${cap} MiB, the pinned ledger needs >= $FRANK_ARCHON_TTM_MIN_MIB" >&2 echo "frank-archon: raise $FRANK_ARCHON_TTM_SYS_DIR/pages_limit (and pages_max if the kernel exposes one) to >= $(( FRANK_ARCHON_TTM_MIN_MIB * 256 )) in 4 KiB pages; a read-only knob needs the module option (options ttm pages_limit=N) and a reboot" >&2 return 1 fi echo "frank-archon: ttm pages cap ${cap} MiB clears the pinned-ledger floor $FRANK_ARCHON_TTM_MIN_MIB" >&2 } # Executed (not sourced) it is the privileged pre-start hook: systemd runs this # as root from the unit's `ExecStartPre=+…`, where the knob write is legal and # the `Restart=always` loop can never outrun it. if [ "${BASH_SOURCE[0]}" = "${0}" ]; then set -euo pipefail case "${1:-}" in --raise) TTM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null source "${FRANK_ARCHON_ENV_FILE:-$TTM_DIR/frank-archon.env}" ttm_ensure_cap ;; *) echo "usage: ${0##*/} --raise (raise the ttm pages cap, then verify)" >&2 exit 2 ;; esac fi