fix(frank-archon): raise the ttm knob from a root ExecStartPre — User=dank cannot write it
a9da65e put the raise inside the serve script, which runs as `User=dank`: the knob is root-owned 0644, so `-w` is false and the write silently failed — the gate printed its refusal on a cap it had just failed to move. The privileged path is systemd's `+` prefix: `ExecStartPre=+scripts/ttm-preflight.sh --raise` runs as root even under `User=`, before ExecStart, so `Restart=always` can never outrun it. The lib is now also executable (`--raise` sources the env file and verifies), and keeps a `sudo -n` fallback for a human running the gate from a shell. Seam: the refusal cases run with a PATH that has `cat`+`bash` and NO `sudo` (root writes a 444 file, so a passwordless dev box would have tested nothing), and both now assert the gate's own exit 1, not merely non-zero. New seam covers the unit line and the `--raise` entry point end to end (writable knob -> raised, no args -> usage exit 2).
This commit is contained in:
@@ -35,6 +35,13 @@ Group=dank
|
|||||||
LimitMEMLOCK=infinity
|
LimitMEMLOCK=infinity
|
||||||
Environment="FRANK_ARCHON_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-archon.env"
|
Environment="FRANK_ARCHON_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-archon.env"
|
||||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-archon.sh
|
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-archon.sh
|
||||||
|
# The `+` prefix runs this as root even though the service is User=dank — the
|
||||||
|
# ttm pages cap is a root-owned knob and the pinned-RAM ledger (~147 GiB) is
|
||||||
|
# unusable below it. After a reboot the knob is back at its default (half of
|
||||||
|
# RAM), so the raise must be part of the start, not a note in a runbook: the
|
||||||
|
# 2026-10-12 power failure reset it and the endpoint refused to boot 6,919
|
||||||
|
# times. The serve script re-verifies the cap after this hook.
|
||||||
|
ExecStartPre=+/home/dank/serverAppConfigs/scripts/ttm-preflight.sh --raise
|
||||||
# systemd stop: wait alone cannot stop the container child; stop it
|
# systemd stop: wait alone cannot stop the container child; stop it
|
||||||
# explicitly. No graceful drain contract is claimed (the OpenAI front end
|
# explicitly. No graceful drain contract is claimed (the OpenAI front end
|
||||||
# holds no queue to drain — in-flight decodes are lost, which a stop
|
# holds no queue to drain — in-flight decodes are lost, which a stop
|
||||||
|
|||||||
@@ -42,15 +42,24 @@ ttm_cap_mib() {
|
|||||||
echo "${cur:-0}"
|
echo "${cur:-0}"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Writes $1 MiB (as 4 KiB pages) into every writable knob present. Failures are
|
# Writes $1 MiB (as 4 KiB pages) into every knob present. Failures are
|
||||||
# swallowed: the caller re-reads the cap, and a write the kernel refused shows
|
# swallowed: the caller re-reads the cap, and a write the kernel refused shows
|
||||||
# up there (ttm refuses a limit below the pages it has already pinned).
|
# up there (ttm refuses a limit below the pages it has already pinned).
|
||||||
|
#
|
||||||
|
# PRIVILEGE, measured the hard way: the knob is root-owned 0644 and the service
|
||||||
|
# process runs as User=dank, so `-w` is FALSE there and a plain echo fails. The
|
||||||
|
# canonical privileged path is the unit's ExecStartPre with the `+` prefix
|
||||||
|
# (systemd runs it as root even under User=); `sudo -n` is the fallback for a
|
||||||
|
# human running the gate from a shell (passwordless sudo on both hosts).
|
||||||
ttm_raise_cap() {
|
ttm_raise_cap() {
|
||||||
local pages=$(( $1 * 256 )) p
|
local pages=$(( $1 * 256 )) p
|
||||||
for p in "$FRANK_ARCHON_TTM_SYS_DIR/pages_limit" \
|
for p in "$FRANK_ARCHON_TTM_SYS_DIR/pages_limit" \
|
||||||
"$FRANK_ARCHON_TTM_SYS_DIR/pages_max"; do
|
"$FRANK_ARCHON_TTM_SYS_DIR/pages_max"; do
|
||||||
|
[ -e "$p" ] || continue
|
||||||
if [ -w "$p" ]; then
|
if [ -w "$p" ]; then
|
||||||
echo "$pages" > "$p" 2>/dev/null || true
|
echo "$pages" > "$p" 2>/dev/null || true
|
||||||
|
elif command -v sudo >/dev/null 2>&1; then
|
||||||
|
sudo -n sh -c "echo $pages > '$p'" >/dev/null 2>&1 || true
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
@@ -74,3 +83,22 @@ ttm_ensure_cap() {
|
|||||||
fi
|
fi
|
||||||
echo "frank-archon: ttm pages cap ${cap} MiB clears the pinned-ledger floor $FRANK_ARCHON_TTM_MIN_MIB" >&2
|
echo "frank-archon: ttm pages cap ${cap} MiB clears the pinned-ledger floor $FRANK_ARCHON_TTM_MIN_MIB" >&2
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Executed (not sourced) it is the privileged pre-start hook: systemd runs this
|
||||||
|
# as root from the unit's `ExecStartPre=+…`, where the knob write is legal and
|
||||||
|
# the `Restart=always` loop can never outrun it.
|
||||||
|
if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
|
||||||
|
set -euo pipefail
|
||||||
|
case "${1:-}" in
|
||||||
|
--raise)
|
||||||
|
TTM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${FRANK_ARCHON_ENV_FILE:-$TTM_DIR/frank-archon.env}"
|
||||||
|
ttm_ensure_cap
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "usage: ${0##*/} --raise (raise the ttm pages cap, then verify)" >&2
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
|||||||
@@ -33,6 +33,16 @@ BUILD="$REPO_ROOT/scripts/convert-swift15-w4dn8.sh"
|
|||||||
HFGET="$REPO_ROOT/scripts/hfget.sh"
|
HFGET="$REPO_ROOT/scripts/hfget.sh"
|
||||||
TEMPLATE="$REPO_ROOT/scripts/qwen38-froggeric-v22.5.jinja"
|
TEMPLATE="$REPO_ROOT/scripts/qwen38-froggeric-v22.5.jinja"
|
||||||
|
|
||||||
|
# A PATH with `cat` (the gate reads knobs with it) and NO `sudo`, so the
|
||||||
|
# privileged fallback cannot rescue a write the test means to fail.
|
||||||
|
nosudo_bin() {
|
||||||
|
local bin="$BATS_TEST_TMPDIR/nosudo-bin"
|
||||||
|
mkdir -p "$bin"
|
||||||
|
ln -sf "$(command -v cat)" "$bin/cat"
|
||||||
|
ln -sf "$(command -v bash)" "$bin/bash"
|
||||||
|
echo "$bin"
|
||||||
|
}
|
||||||
|
|
||||||
@test "frank-archon unit, serve script, env, fetch and build scripts and template exist" {
|
@test "frank-archon unit, serve script, env, fetch and build scripts and template exist" {
|
||||||
[ -f "$UNIT" ]
|
[ -f "$UNIT" ]
|
||||||
[ -x "$SERVE" ]
|
[ -x "$SERVE" ]
|
||||||
@@ -298,8 +308,11 @@ TEMPLATE="$REPO_ROOT/scripts/qwen38-froggeric-v22.5.jinja"
|
|||||||
|
|
||||||
echo 32253039 > "$dir/pages_limit"
|
echo 32253039 > "$dir/pages_limit"
|
||||||
chmod 444 "$dir/pages_limit" # the write refused: cap cannot move
|
chmod 444 "$dir/pages_limit" # the write refused: cap cannot move
|
||||||
run bash -c '. "$1"; ttm_ensure_cap' _ "$TTM_LIB"
|
# The refusal case must not have `sudo` on PATH: the gate's privileged
|
||||||
[ "$status" -ne 0 ]
|
# fallback runs `sudo -n`, and on a passwordless host root can write a 444
|
||||||
|
# file, which would silently test nothing.
|
||||||
|
run env PATH="$(nosudo_bin)" bash -c '. "$1"; ttm_ensure_cap' _ "$TTM_LIB"
|
||||||
|
[ "$status" -eq 1 ]
|
||||||
[[ "$output" == *"ERROR: ttm pages cap is 125988 MiB"* ]]
|
[[ "$output" == *"ERROR: ttm pages cap is 125988 MiB"* ]]
|
||||||
[ "$(cat "$dir/pages_limit")" = "32253039" ]
|
[ "$(cat "$dir/pages_limit")" = "32253039" ]
|
||||||
}
|
}
|
||||||
@@ -323,12 +336,35 @@ TEMPLATE="$REPO_ROOT/scripts/qwen38-froggeric-v22.5.jinja"
|
|||||||
[ "$output" = "117187" ]
|
[ "$output" = "117187" ]
|
||||||
|
|
||||||
chmod 444 "$dir/pages_limit" "$dir/pages_max" # both writes refused
|
chmod 444 "$dir/pages_limit" "$dir/pages_max" # both writes refused
|
||||||
run bash -c '. "$1"; ttm_ensure_cap' _ "$TTM_LIB"
|
run env PATH="$(nosudo_bin)" bash -c '. "$1"; ttm_ensure_cap' _ "$TTM_LIB"
|
||||||
[ "$status" -ne 0 ]
|
[ "$status" -eq 1 ]
|
||||||
[[ "$output" == *"ERROR: ttm pages cap is 117187 MiB"* ]]
|
[[ "$output" == *"ERROR: ttm pages cap is 117187 MiB"* ]]
|
||||||
[ "$(cat "$dir/pages_max")" = "30000000" ]
|
[ "$(cat "$dir/pages_max")" = "30000000" ]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@test "the gate runs as a root pre-start hook, where the knob write is legal" {
|
||||||
|
# The knob is root-owned 0644 and the service runs as User=dank, so the
|
||||||
|
# gate CANNOT raise it from the service process. systemd's `+` prefix on
|
||||||
|
# ExecStartPre runs as root even under User= — the canonical privileged
|
||||||
|
# path — and the gate's own --raise mode is what that line calls.
|
||||||
|
grep -q '^ExecStartPre=+/home/dank/serverAppConfigs/scripts/ttm-preflight.sh --raise$' "$UNIT"
|
||||||
|
|
||||||
|
# Behaviour of that entry point: sourced env values, knob raised, exit 0.
|
||||||
|
local dir="$BATS_TEST_TMPDIR/ttm3" envf="$BATS_TEST_TMPDIR/ttm3.env"
|
||||||
|
mkdir -p "$dir"
|
||||||
|
printf 'FRANK_ARCHON_TTM_MIN_MIB=150000\nFRANK_ARCHON_TTM_TARGET_MIB=204800\n' > "$envf"
|
||||||
|
echo 32253039 > "$dir/pages_limit"
|
||||||
|
run env FRANK_ARCHON_ENV_FILE="$envf" FRANK_ARCHON_TTM_SYS_DIR="$dir" \
|
||||||
|
bash "$TTM_LIB" --raise
|
||||||
|
[ "$status" -eq 0 ]
|
||||||
|
[ "$(cat "$dir/pages_limit")" = "52428800" ]
|
||||||
|
|
||||||
|
# Usage is a named failure, not a silent no-op.
|
||||||
|
run bash "$TTM_LIB"
|
||||||
|
[ "$status" -eq 2 ]
|
||||||
|
[[ "$output" == *"--raise"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
@test "the serve script clears stale containers and self-heals via podman wait" {
|
@test "the serve script clears stale containers and self-heals via podman wait" {
|
||||||
# A crashed unit can leave the container name registered forever after;
|
# A crashed unit can leave the container name registered forever after;
|
||||||
# the legacy identities hold the same GPUs and die here too. Then block
|
# the legacy identities hold the same GPUs and die here too. Then block
|
||||||
|
|||||||
Reference in New Issue
Block a user