diff --git a/scripts/serve-frank-primary.sh b/scripts/serve-frank-primary.sh index d09b6c8..459b211 100755 --- a/scripts/serve-frank-primary.sh +++ b/scripts/serve-frank-primary.sh @@ -104,6 +104,7 @@ podman run -d --name "$FRANK_PRIMARY_CONTAINER" \ --cap-add SYS_PTRACE \ --device /dev/kfd --device /dev/dri --device /dev/mem \ -e HIP_VISIBLE_DEVICES=0,1 -e VLLM_ROCM_USE_AITER=0 \ + -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ -v "$FRANK_PRIMARY_MODEL_DIR":/models:ro \ "$FRANK_PRIMARY_IMAGE" /models \ --served-model-name "$FRANK_PRIMARY_SERVED_NAME" --port 7770 \ diff --git a/tests/frank_primary_unit.bats b/tests/frank_primary_unit.bats index a14b942..21a9bfc 100755 --- a/tests/frank_primary_unit.bats +++ b/tests/frank_primary_unit.bats @@ -106,6 +106,15 @@ DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh" grep -q '^FRANK_PRIMARY_REASONINGPARSER=glm47$' "$ENV_FILE" } +@test "expandable segments is ON — the cache arena leaves no slack" { + # The expert-offload plan fills VRAM to the modelled headroom, so the + # profile/graph-capture phase has ~0.2 GiB to work with and the default + # allocator's fragmentation fails it: measured 2026-09-30 on the deployed + # unit, "Tried to allocate 662.00 MiB ... 240.00 MiB is free" from the + # engine core while the identical shape booted with this env set. + grep -q 'PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True' "$SERVE" +} + @test "expert offload is ON — the payload cannot fit VRAM without it" { # 170 GiB of int4 experts against 64 GiB of VRAM. Without the offload flag # the engine has no home for the expert tensors at all; the arena