diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index 0039554..7843057 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -39,9 +39,11 @@ ALIAS=frank-default-small BIND_ADDR=0.0.0.0 LISTEN_PORT=52625 -# Serving defaults from the model card: 64K context, single slot, f16 KV, -# context checkpoints, prompt cache. Native MTP depth-4. -CTX_SIZE=65536 +# Serving: 128K context (user request, 2026-08-18; model card default is +# 64K), single slot, f16 KV, context checkpoints, prompt cache. Native MTP +# depth-4. KV at 128K is 8 GiB (64 KiB/token); estimated total ~43 GiB of +# the 64 GiB SVM pool, leaving ~21 GiB headroom. +CTX_SIZE=131072 PARALLEL=1 BATCH_SIZE=2048 UBATCH_SIZE=2048 @@ -60,6 +62,10 @@ SPEC_DRAFT_P_SPLIT=0.10 TEMPERATURE=0.7 TOP_P=0.95 -REASONING=off +# Thinking enabled (user request, 2026-08-18). deepseek format moves the +# reasoning into message.reasoning_content (OpenAI-compatible); +# none would leave the thought tags unparsed in content. +REASONING=on +REASONING_FORMAT=deepseek VERBOSITY=4 diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 78ecd54..f56e1b2 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -37,7 +37,7 @@ fi : "${ALIAS:=frank-default-small}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=52625}" -: "${CTX_SIZE:=65536}" +: "${CTX_SIZE:=131072}" : "${PARALLEL:=1}" : "${BATCH_SIZE:=2048}" : "${UBATCH_SIZE:=2048}" @@ -51,7 +51,8 @@ fi : "${SPEC_DRAFT_P_SPLIT:=0.10}" : "${TEMPERATURE:=0.7}" : "${TOP_P:=0.95}" -: "${REASONING:=off}" +: "${REASONING:=on}" +: "${REASONING_FORMAT:=deepseek}" : "${VERBOSITY:=4}" MODEL="$MODEL_DIR/$MODEL_FILE" @@ -113,6 +114,6 @@ exec "$LLAMA_BIN" \ --temp "$TEMPERATURE" \ --top-p "$TOP_P" \ --reasoning "$REASONING" \ - --reasoning-format none \ + --reasoning-format "$REASONING_FORMAT" \ --reasoning-budget -1 \ -lv "$VERBOSITY"