diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 227c9fa..b8b6ca7 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -33,5 +33,12 @@ CACHE_TYPE_V=q8_0 # User-specified ubatch 4096 / batch 12288 is not feasible here (see bench). BATCH_SIZE=2048 UBATCH_SIZE=512 +# Sampling settings — per HauhauCS Qwen3.6-27B model card. +# Thinking-general: temp=1.0, top_p=0.95, top_k=20, min_p=0. +# Clients may override per-request; these are the server-side defaults. +TEMP=1.0 +TOP_K=20 +TOP_P=0.95 +MIN_P=0.0 THREADS=12 THREADS_HTTP=1 diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index a4ce5f5..b2a8180 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -8,15 +8,15 @@ PORT=7770 # (Increased from 3×200k to add a 4th parallel stream.) CTX_SIZE=800000 PARALLEL=4 -# Spec: ngram-mod. n_min=48, n_max=64, n_match=24 (defaults). 8 expert -# layers back on GPU; rest on CPU. Batch 8192 / ubatch 2048. +# Spec: ngram-mod. n_min=48, n_max=64, n_match=24 (defaults). 6 expert +# layers back on GPU; 34 on CPU. Batch 8192 / ubatch 2048. SPEC_TYPE=ngram-mod NGRAM_MOD_N_MIN=48 NGRAM_MOD_N_MAX=64 NGRAM_MOD_N_MATCH=24 -# Expert offload: 33 of 40 layers keep MoE weights on CPU; remaining -# 7 layers have their expert weights resident on GPU for faster routing. -N_CPU_MOE=33 +# Expert offload: 34 of 40 layers keep MoE weights on CPU; remaining +# 6 layers have their expert weights resident on GPU for faster routing. +N_CPU_MOE=34 N_GPU_LAYERS=all # KV cache on GPU, K quantized to q8_0, V stays f16 (user-locked decision). CACHE_TYPE_K=q8_0 @@ -31,5 +31,12 @@ UBATCH_SIZE=2048 # CPU, so eager RAM load avoids page faults during decode (uses ~31GB RAM). NO_MMAP=1 # Compute threads lowered to 7 so other processes on the box keep a core. +# Sampling settings — per HauhauCS Qwen3.6-35B-A3B model card. +# Thinking-general: temp=1.0, top_p=0.95, top_k=20, min_p=0. +# Clients may override per-request; these are the server-side defaults. +TEMP=1.0 +TOP_K=20 +TOP_P=0.95 +MIN_P=0.0 THREADS=7 THREADS_HTTP=8 \ No newline at end of file diff --git a/scripts/serve-llama-frank-large.sh b/scripts/serve-llama-frank-large.sh index 359d8ec..30e4b20 100755 --- a/scripts/serve-llama-frank-large.sh +++ b/scripts/serve-llama-frank-large.sh @@ -37,6 +37,10 @@ fi : "${CACHE_TYPE_V:=q8_0}" : "${BATCH_SIZE:=2048}" : "${UBATCH_SIZE:=512}" +: "${TEMP:=1.0}" +: "${TOP_K:=20}" +: "${TOP_P:=0.95}" +: "${MIN_P:=0.0}" : "${THREADS:=12}" : "${THREADS_HTTP:=1}" @@ -68,6 +72,10 @@ exec "$LLAMA_SERVER" \ --parallel "$PARALLEL" \ --batch-size "$BATCH_SIZE" \ --ubatch-size "$UBATCH_SIZE" \ + --temp "$TEMP" \ + --top-k "$TOP_K" \ + --top-p "$TOP_P" \ + --min-p "$MIN_P" \ --threads "$THREADS" \ --threads-http "$THREADS_HTTP" \ --alias "$ALIAS" \ diff --git a/scripts/serve-llama-frank-small.sh b/scripts/serve-llama-frank-small.sh index 1a3c2b3..b052439 100755 --- a/scripts/serve-llama-frank-small.sh +++ b/scripts/serve-llama-frank-small.sh @@ -32,12 +32,16 @@ fi : "${NGRAM_MOD_N_MATCH:=24}" : "${CACHE_TYPE_K:=q8_0}" : "${CACHE_TYPE_V:=f16}" -: "${N_CPU_MOE:=40}" # all 40 layers: MoE expert weights stay on CPU +: "${N_CPU_MOE:=34}" # 34 of 40 layers: MoE expert weights stay on CPU; 6 on GPU : "${N_GPU_LAYERS:=all}" # every non-expert tensor goes to GPU : "${BATCH_SIZE:=2048}" : "${UBATCH_SIZE:=512}" : "${CTX_CHECKPOINTS:=0}" : "${NO_MMAP:=0}" +: "${TEMP:=1.0}" +: "${TOP_K:=20}" +: "${TOP_P:=0.95}" +: "${MIN_P:=0.0}" : "${THREADS:=8}" # 7700X physical cores : "${THREADS_HTTP:=8}" @@ -70,6 +74,10 @@ exec "$LLAMA_SERVER" \ --ubatch-size "$UBATCH_SIZE" \ --ctx-checkpoints "$CTX_CHECKPOINTS" \ $([ "$NO_MMAP" = "1" ] && printf -- "--no-mmap") \ + --temp "$TEMP" \ + --top-k "$TOP_K" \ + --top-p "$TOP_P" \ + --min-p "$MIN_P" \ --threads "$THREADS" \ --threads-http "$THREADS_HTTP" \ --alias "$ALIAS" \