From ba67ca5c9ed7cb180d013dfa783e84d83002041b Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 6 Oct 2026 00:26:28 -0700 Subject: [PATCH] frank-large: pin the prompt cache (16 GB) and capture the engine ledger on disk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Auto cache-sizing is OFF on this box: the boot ledger refused it at 36.9 GB (2 x full-entry) floor vs 35.7 GB free, so every deep agent request re-prefills its whole 79-81k context (84.9-179.8 s measured on serve_api lines, 2026-10-06) for ~500-650 new tokens per turn. Pin HALOGEN_CACHE_MB=16384 (>= one full-slot entry at ~70 KB/token) — the documented override; the per-store reserve re-check keeps the host safe. Also: the serve script's 'podman run -d > LOG' captured only the container ID — the runbook's boot-ledger grep read a one-line hex file and the ledger existed nowhere on disk. Foreground 'podman run' makes the ledger land in the log file as documented; exit-code and ExecStop semantics are unchanged. Test seam updated for both. --- scripts/frank-large.env | 24 +++++++++++++++++++----- scripts/serve-frank-large.sh | 17 +++++++++++------ tests/frank_large_unit.bats | 27 +++++++++++++++++---------- 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/scripts/frank-large.env b/scripts/frank-large.env index 6f64197..2678555 100644 --- a/scripts/frank-large.env +++ b/scripts/frank-large.env @@ -106,8 +106,22 @@ HALOGEN_DRAFTER=2 # honor that. Do not lower the timeout alone. HALOGEN_MAX_TOKENS_CAP=65536 HALOGEN_QUEUE_TIMEOUT=7200 -# Engine prompt cache: EMPTY = auto (sized from available memory at -# startup). Pin an explicit MB only to argue with the auto figure — a -# full-context entry is ~18 GB at 262K, so a too-small pin yields a cache -# that reports enabled and never hits (the engine warns at startup). -#HALOGEN_CACHE_MB= +# Prompt cache — PINNED (2026-10-06). Auto is OFF on this box: the boot +# ledger refused it — "serve: prompt cache OFF - 35.7 GB free after the +# mapped checkpoint, 8.6 GB reserved for the system, which leaves less +# than the 36.9 GB floor (2 x a 18.4 GB full-context entry). Set +# HALOGEN_CACHE_MB to override." Auto demands TWO full-context entries; +# the documented override is an explicit byte budget. Measured cost of +# leaving it off: every deep agent request re-prefills its whole +# context — 84.9-179.8 s for 46-81k prompts that grow by only ~500-650 +# tokens per turn (serve_api lines, 2026-10-06) — while the README +# prices the cache at ~20x time-to-first-token on a warm turn. +# +# Sizing: an entry costs ~70 KB/token (engine figure), so ONE full-slot +# entry (200,700 tok) is ~13.7 GB; 16384 MB holds it with margin, and +# several ordinary 60-80k conversations under LRU-by-bytes eviction. +# The reserve (HALOGEN_CACHE_RESERVE_MB, default 8192) is re-checked +# against live MemAvailable on every store, so a pin cannot starve the +# host silently. HALOGEN_CACHE_ALIGN stays at its shipped 2048 — that +# value is what makes warm answers byte-identical to cold ones. +HALOGEN_CACHE_MB=16384 diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 06f43cd..d5a8c5a 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -119,7 +119,7 @@ mkdir -p "$(dirname "$LOG")" # either. No comment lines inside the backslash # chain — a mid-chain # terminates the continuation (the 2026-10-05 # exit-127 crash loop on the sibling serve script). -podman run -d --name "$FRANK_LARGE_CONTAINER" \ +podman run --name "$FRANK_LARGE_CONTAINER" \ --publish "$FRANK_LARGE_PUBLISH" \ --ipc=host \ --ulimit memlock=-1:-1 \ @@ -139,8 +139,13 @@ podman run -d --name "$FRANK_LARGE_CONTAINER" \ "$FRANK_LARGE_IMAGE" \ > "$LOG" 2>&1 -# This script's lifetime IS the container's: podman wait returns the -# container's exit code, so a crash is a nonzero ExecStart and the unit's -# Restart=always is real. systemd stop goes through the unit's ExecStop -# (podman wait alone cannot stop the child). -exec podman wait "$FRANK_LARGE_CONTAINER" +# FOREGROUND `podman run`: the engine's stdout/stderr are this script's +# output, so the boot ledger (kv pool, prompt cache, drafter state, any +# clamp) lands in $LOG as documented. The previous `-d` + `exec podman +# wait` shape captured only the container ID — the ledger existed +# nowhere on disk and the runbook's post-deploy grep read a one-line +# hex file (found 2026-10-06 while tuning). podman run returns the +# container's exit code, so a crash is a nonzero ExecStart and the +# unit's Restart=always stays real; systemd stop goes through the +# unit's ExecStop and the run client forwards SIGTERM to the engine, +# which drains on it. diff --git a/tests/frank_large_unit.bats b/tests/frank_large_unit.bats index bed8873..3b164ff 100755 --- a/tests/frank_large_unit.bats +++ b/tests/frank_large_unit.bats @@ -74,6 +74,7 @@ good_env() { "HALOGEN_SLOT_CTX=200700" "HALOGEN_KV_SLOTS=1" "HALOGEN_DRAFTER=2" + "HALOGEN_CACHE_MB=16384" "STUB_DIGESTS=localhost/x ghcr.io/peonist-ai/halogen@sha256:cafe" ) } @@ -115,17 +116,20 @@ good_env() { [[ "$output" == *"no gfx1151 to serve on"* ]] } -@test "the KV budget fits unified memory with margin" { +@test "the KV pool plus pinned cache fit unified memory with margin" { # Pool arithmetic is the engine's documented product: slots x - # slot_ctx x 64 KiB. Against it: 33.4 GiB of mapped weights, the - # engine working set + auto prompt cache (~25 GiB of headroom - # allowed), host baseline and frank-small co-tenancy. The box has - # 121 GiB unified; the 2026-09-16 halogen era ran the same engine - # class here on a LARGER pool with less co-tenancy slack. + # slot_ctx x 64 KiB. The prompt cache is PINNED (HALOGEN_CACHE_MB, + # ~70 KB/token per entry — 16384 MB >= one full-slot entry). Against + # both: 33.4 GiB of mapped weights, ~20 GiB engine working set, host + # baseline and frank-small co-tenancy. The box has 121 GiB unified; + # the 2026-09-16 halogen era ran the same engine class here on a + # LARGER pool with less co-tenancy slack. ctx="$(sed -nE 's/^HALOGEN_SLOT_CTX=([0-9]+)$/\1/p' "$ENV_FILE")" slots="$(sed -nE 's/^HALOGEN_KV_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")" + cache_mb="$(sed -nE 's/^HALOGEN_CACHE_MB=([0-9]+)$/\1/p' "$ENV_FILE")" pool_gib=$(( slots * ctx * 65536 / 1073741824 )) - total_gib=$(( 34 + pool_gib + 25 + 15 )) + cache_gib=$(( ${cache_mb:-0} / 1024 )) + total_gib=$(( 34 + pool_gib + cache_gib + 20 + 15 )) [ "$total_gib" -le 121 ] } @@ -192,9 +196,12 @@ $(cat "$TEST_TMP/serve.log")" [[ "$output" == *"--device $TEST_TMP/qwen.hgn"* ]] [[ "$output" == *"$TEST_TMP/tokenizer:/tokenizer:ro"* ]] [[ "$output" == *"ghcr.io/peonist-ai/halogen@sha256:cafe"* ]] - # The unit's restart policy is real because the script becomes - # `podman wait`: the container's exit code is the service's exit. - [[ "$output" == *"PODMAN wait frank-large-halogen"* ]] + # Foreground run, never -d: the engine ledger must land in the log + # file (`podman run -d` captured only the container ID and the + # runbook's ledger grep read a one-line hex file, found 2026-10-06). + ! grep -qE '(^|[[:space:]])-d([[:space:]]|$)' "$TEST_TMP/serve.log" + ! grep -q 'PODMAN wait' "$TEST_TMP/serve.log" + [[ "$output" == *"-e HALOGEN_CACHE_MB=16384"* ]] } @test "the shipped env pins the fleet's window and shape" {