diff --git a/docs/LLM/dealign-ornith-crack.md b/docs/LLM/dealign-ornith-crack.md index 2576a82..b3a30e3 100644 --- a/docs/LLM/dealign-ornith-crack.md +++ b/docs/LLM/dealign-ornith-crack.md @@ -77,7 +77,7 @@ llama-server \ --chat-template-file chat-template.jinja \ --spec-type draft-mtp \ --override-tensor "\.ffn_(up|down|gate|gate_up)_(ch|)exps=CPU" \ - -ngl all -c 786432 -np 3 \ + -ngl all -c 688128 -np 3 \ --cache-type-k f16 --cache-type-v f16 --no-mmap \ --temp 1.0 --top-p 0.95 --top-k 20 ``` diff --git a/docs/LLM/deployment.md b/docs/LLM/deployment.md index d628b47..5a4c1de 100644 --- a/docs/LLM/deployment.md +++ b/docs/LLM/deployment.md @@ -128,7 +128,8 @@ binary with the MoE expert weights pinned to CPU by an explicit `--override-tensor` regex (the expert `\.ffn_(up|down|gate|gate_up)_(ch|)exps` pattern, buffer type `CPU`) and 7 CPU threads, native `draft-mtp` speculative decoding (bundled MoE MTP head, n_max 4), f16 KV, context -786432 (3 slots of ~262K each), batch 4096 / ubatch 2048, `--no-mmap`. The +688128 (3 slots of ~229K each, 32k headroom per stream), batch 4096 / +ubatch 2048, `--no-mmap`. The unit is `scripts/frank-default-small.service`. It launches through `scripts/serve-frank-small.sh`, which reads tunables from `scripts/frank-default-small.env`. See [[dealign-ornith-crack]]. diff --git a/docs/LLM/index.md b/docs/LLM/index.md index 2270c0d..84df901 100644 --- a/docs/LLM/index.md +++ b/docs/LLM/index.md @@ -20,7 +20,7 @@ graph LR B[upstream llama.cpp b10121 frank-default-small, port 7770] C[Ornith-1.5-35B-A3B-CRACK Q8_0 -native MTP, MoE experts offloaded via override, 786432 ctx] +native MTP, MoE experts offloaded via override, 688128 ctx] A --> B --> C end diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index 4ba0843..c8b3920 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -25,10 +25,10 @@ ALIAS=frank-default-small BIND_ADDR=0.0.0.0 LISTEN_PORT=7770 -# 3 parallel slots x 262144 total context pool (max per-slot), unchanged -# from the prior Qwen3.6 service. Lower PARALLEL if VRAM is tight with the -# larger Q8_0 payload. -CTX_SIZE=786432 +# 3 parallel slots x 229376 ctx each (262144 lowered by 32k per stream for +# cache headroom) = 688128 total context pool. The per-slot 229376 matches +# the per-slot context advertised in models.json (229376). +CTX_SIZE=688128 PARALLEL=3 # Spec: MoE model with a bundled MTP head -> native draft-mtp, n_max 4