Frank
a6f55f4cc6
a9da65e put the raise inside the serve script, which runs as `User=dank`: the knob is root-owned 0644, so `-w` is false and the write silently failed — the gate printed its refusal on a cap it had just failed to move. The privileged path is systemd's `+` prefix: `ExecStartPre=+scripts/ttm-preflight.sh --raise` runs as root even under `User=`, before ExecStart, so `Restart=always` can never outrun it. The lib is now also executable (`--raise` sources the env file and verifies), and keeps a `sudo -n` fallback for a human running the gate from a shell. Seam: the refusal cases run with a PATH that has `cat`+`bash` and NO `sudo` (root writes a 444 file, so a passwordless dev box would have tested nothing), and both now assert the gate's own exit 1, not merely non-zero. New seam covers the unit line and the `--raise` entry point end to end (writable knob -> raised, no args -> usage exit 2).
105 lines
5.0 KiB
Bash
Executable File
105 lines
5.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ttm-preflight.sh — the TTM pages-cap gate for engines that PIN host RAM.
|
|
#
|
|
# Sourced, never executed. An engine that pins tens of GiB of host RAM (the
|
|
# radiance expert pools + the PLE n-gram table) has its pinned pages charged
|
|
# against the kernel's ttm page cap: /sys/module/ttm/parameters/pages_limit
|
|
# (and pages_max on kernels that expose one). The cap function below takes the
|
|
# MIN over the knobs the running kernel actually exposes — an absent knob is
|
|
# skipped, never read as zero, because a zero would refuse a healthy cap.
|
|
#
|
|
# THE LAW THIS FILE EXISTS FOR: the cap is a RUNTIME knob. The kernel default is
|
|
# half of RAM, so every raise is lost on reboot. On 2026-10-12 23:17 a power
|
|
# failure on dankserver reset it 200 GiB -> 125,988 MiB, and frank-archon's
|
|
# preflight refused the start — 6,919 crash-restarts over ~20 hours, with
|
|
# `Restart=always` cheerfully re-running a refusal. A manual sysfs write is
|
|
# therefore not an acceptable repair procedure: the gate raises the knob itself
|
|
# (the units that need it run as root, and the knob is a 0644 root-owned module
|
|
# parameter), re-reads, and only then refuses.
|
|
#
|
|
# Contract (set by the sourcing script's env file):
|
|
# FRANK_ARCHON_TTM_MIN_MIB floor the pinned ledger needs; refuse below it
|
|
# FRANK_ARCHON_TTM_TARGET_MIB value to raise the cap to when it is below the
|
|
# floor (the recorded host ceiling — writing the
|
|
# floor itself would leave the ledger no slack)
|
|
# FRANK_ARCHON_TTM_SYS_DIR knob directory; defaults to the real sysfs
|
|
# path, a test points it at a temp dir.
|
|
# Pages are 4 KiB: pages = MiB x 256.
|
|
|
|
: "${FRANK_ARCHON_TTM_SYS_DIR:=/sys/module/ttm/parameters}"
|
|
|
|
# Prints the effective cap in MiB, 0 when no knob is readable.
|
|
ttm_cap_mib() {
|
|
local p v cur=""
|
|
for p in "$FRANK_ARCHON_TTM_SYS_DIR/pages_limit" \
|
|
"$FRANK_ARCHON_TTM_SYS_DIR/pages_max"; do
|
|
[ -r "$p" ] || continue
|
|
v="$(cat "$p" 2>/dev/null)" || continue
|
|
[ -n "$v" ] || continue
|
|
v=$(( v * 4 / 1024 ))
|
|
if [ "$v" -lt "${cur:-999999999}" ]; then cur=$v; fi
|
|
done
|
|
echo "${cur:-0}"
|
|
}
|
|
|
|
# Writes $1 MiB (as 4 KiB pages) into every knob present. Failures are
|
|
# swallowed: the caller re-reads the cap, and a write the kernel refused shows
|
|
# up there (ttm refuses a limit below the pages it has already pinned).
|
|
#
|
|
# PRIVILEGE, measured the hard way: the knob is root-owned 0644 and the service
|
|
# process runs as User=dank, so `-w` is FALSE there and a plain echo fails. The
|
|
# canonical privileged path is the unit's ExecStartPre with the `+` prefix
|
|
# (systemd runs it as root even under User=); `sudo -n` is the fallback for a
|
|
# human running the gate from a shell (passwordless sudo on both hosts).
|
|
ttm_raise_cap() {
|
|
local pages=$(( $1 * 256 )) p
|
|
for p in "$FRANK_ARCHON_TTM_SYS_DIR/pages_limit" \
|
|
"$FRANK_ARCHON_TTM_SYS_DIR/pages_max"; do
|
|
[ -e "$p" ] || continue
|
|
if [ -w "$p" ]; then
|
|
echo "$pages" > "$p" 2>/dev/null || true
|
|
elif command -v sudo >/dev/null 2>&1; then
|
|
sudo -n sh -c "echo $pages > '$p'" >/dev/null 2>&1 || true
|
|
fi
|
|
done
|
|
}
|
|
|
|
# Returns 0 when the cap clears FRANK_ARCHON_TTM_MIN_MIB, raising it first if
|
|
# needed; returns 1 with a named refusal when it cannot be made to.
|
|
ttm_ensure_cap() {
|
|
local cap
|
|
cap="$(ttm_cap_mib)"
|
|
if [ "$cap" -lt "$FRANK_ARCHON_TTM_MIN_MIB" ]; then
|
|
echo "frank-archon: ttm pages cap is ${cap} MiB, below the pinned-ledger" \
|
|
"floor $FRANK_ARCHON_TTM_MIN_MIB — raising the knob to" \
|
|
"$FRANK_ARCHON_TTM_TARGET_MIB MiB ($(( FRANK_ARCHON_TTM_TARGET_MIB * 256 )) pages)" >&2
|
|
ttm_raise_cap "$FRANK_ARCHON_TTM_TARGET_MIB"
|
|
cap="$(ttm_cap_mib)"
|
|
fi
|
|
if [ "$cap" -lt "$FRANK_ARCHON_TTM_MIN_MIB" ]; then
|
|
echo "ERROR: ttm pages cap is ${cap} MiB, the pinned ledger needs >= $FRANK_ARCHON_TTM_MIN_MIB" >&2
|
|
echo "frank-archon: raise $FRANK_ARCHON_TTM_SYS_DIR/pages_limit (and pages_max if the kernel exposes one) to >= $(( FRANK_ARCHON_TTM_MIN_MIB * 256 )) in 4 KiB pages; a read-only knob needs the module option (options ttm pages_limit=N) and a reboot" >&2
|
|
return 1
|
|
fi
|
|
echo "frank-archon: ttm pages cap ${cap} MiB clears the pinned-ledger floor $FRANK_ARCHON_TTM_MIN_MIB" >&2
|
|
}
|
|
|
|
# Executed (not sourced) it is the privileged pre-start hook: systemd runs this
|
|
# as root from the unit's `ExecStartPre=+…`, where the knob write is legal and
|
|
# the `Restart=always` loop can never outrun it.
|
|
if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
|
|
set -euo pipefail
|
|
case "${1:-}" in
|
|
--raise)
|
|
TTM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck source=/dev/null
|
|
source "${FRANK_ARCHON_ENV_FILE:-$TTM_DIR/frank-archon.env}"
|
|
ttm_ensure_cap
|
|
;;
|
|
*)
|
|
echo "usage: ${0##*/} --raise (raise the ttm pages cap, then verify)" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
fi
|