diff --git a/docs/LLM/deployment.md b/docs/LLM/deployment.md index f9cf53f..9bf1fa5 100644 --- a/docs/LLM/deployment.md +++ b/docs/LLM/deployment.md @@ -145,8 +145,9 @@ The active service on Dankclaw is the charlie12345/ROCmFPX llama.cpp runtime against system ROCm 7.2.4 / amdclang 22) with a **strict Qwen MTP** draft (`--spec-type draft-mtp --spec-mtp-strict-qwen`) running on ROCm0 with all layers (`--spec-draft-ngl all`) — no separate draft file, because the MTP head -is embedded in the main GGUF. K/V are f16, full 262144 context (1 slot, -concurrent requests queue, the server never preempts a running generation), +is embedded in the main GGUF. K/V are f16, 196608 context (64K below the +native 262144 maximum; 1 slot, concurrent requests queue, the server never +preempts a running generation), `-t 16 -tb 32`, batch 512 / ubatch 256 (ubatch is swappable; sweep it on 8k context with `scripts/bench-ubatch-rocmi4.sh`), 16 context checkpoints, 8 GiB prompt cache, and reasoning in deepseek format. The unit file is diff --git a/docs/LLM/eemin-rocmi4-mtp.md b/docs/LLM/eemin-rocmi4-mtp.md index f2ca616..f46d384 100644 --- a/docs/LLM/eemin-rocmi4-mtp.md +++ b/docs/LLM/eemin-rocmi4-mtp.md @@ -112,9 +112,10 @@ path's greedy decisions; it does not remove the W4A4 activation approximation. ### Serving settings -- 262144 context (native max), 1 slot, f16 KV, context checkpoints, prompt - cache. Concurrent requests queue; the server never preempts a running - generation. +- 196608 context (64K below the native 262144 maximum), 1 slot, f16 KV, + context checkpoints, prompt cache. The lower limit leaves headroom for the + model plus KV cache in the host's unified-memory budget. Concurrent requests + queue; the server never preempts a running generation. - `-t 16 -tb 32`, batch 512 / ubatch 256 (the qualified W4A4 profile). - `--temp 0.7 --top-p 0.95`, reasoning on (`deepseek` puts `` into `message.reasoning_content`). diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 3115e7d..9a4bda6 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -36,11 +36,11 @@ ALIAS=frank-default-large BIND_ADDR=0.0.0.0 LISTEN_PORT=7770 -# Serving: full 262144 context, 1 slot. f16 KV, context checkpoints, prompt -# cache, strict-MTP embedded draft on ROCm0. Unified memory; rocm-smi reports -# the 64 GiB budget. Concurrent requests queue; the server never preempts a -# running generation. -CTX_SIZE=262144 +# Serving: 196608 context (64K below the model's 262144 maximum), 1 slot. f16 +# KV, context checkpoints, prompt cache, strict-MTP embedded draft on ROCm0. +# This leaves sufficient unified-memory headroom for the model plus KV cache. +# Concurrent requests queue; the server never preempts a running generation. +CTX_SIZE=196608 PARALLEL=1 # Batch/ubatch follow the qualified ROCmI4 W4A4 profile (see the bench in # scripts/bench-ubatch-rocmi4.sh for the ubatch sweep on 8k contexts). diff --git a/scripts/frank-default-large.service b/scripts/frank-default-large.service index c7a323e..96759b1 100644 --- a/scripts/frank-default-large.service +++ b/scripts/frank-default-large.service @@ -1,5 +1,5 @@ [Unit] -Description=CIRU llama.cpp Qwen3.8-27B ROCmFP4 STRIX MTP + Sharp template (dankclaw, frank-default-large) +Description=ROCmFPX llama.cpp Qwen3.8-27B ROCmI4 embedded MTP + Sharp template (dankclaw, frank-default-large) After=network-online.target Wants=network-online.target diff --git a/tests/frank_large_config.bats b/tests/frank_large_config.bats index f085fc9..c52822c 100644 --- a/tests/frank_large_config.bats +++ b/tests/frank_large_config.bats @@ -39,6 +39,11 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja" grep -q '^MTP_BACKEND_SAMPLING=on$' "$ENV_FILE" } +@test "frank-default-large fits the model and KV cache in unified memory" { + grep -q '^CTX_SIZE=196608$' "$ENV_FILE" + grep -q ': "${CTX_SIZE:=262144}"' "$SERVE_FILE" +} + @test "frank-default-large exposes a swappable ubatch" { grep -q '^UBATCH_SIZE=256$' "$ENV_FILE" grep -q ': "${UBATCH_SIZE:=256}"' "$SERVE_FILE"