frank-default-small: 128K context and thinking on
- CTX_SIZE 65536 -> 131072 (user request). KV at 128K is 8 GiB (64 KiB/token f16); estimated ~43 GiB used of the 64 GiB SVM pool, ~21 GiB headroom. - REASONING off -> on with REASONING_FORMAT=deepseek, so <think> reasoning lands in message.reasoning_content (OpenAI-compatible) instead of being left unparsed in content.
This commit is contained in:
1 parent
64cdfc8529
commit
2e1a78aeb4
2 files changed
+14
-7
No files matched your search
@@ -39,9 +39,11 @@ ALIAS=frank-default-small
|
|||||||
BIND_ADDR=0.0.0.0
|
BIND_ADDR=0.0.0.0
|
||||||
LISTEN_PORT=52625
|
LISTEN_PORT=52625
|
||||||
|
|
||||||
# Serving defaults from the model card: 64K context, single slot, f16 KV,
|
# Serving: 128K context (user request, 2026-08-18; model card default is
|
||||||
# context checkpoints, prompt cache. Native MTP depth-4.
|
# 64K), single slot, f16 KV, context checkpoints, prompt cache. Native MTP
|
||||||
CTX_SIZE=65536
|
# depth-4. KV at 128K is 8 GiB (64 KiB/token); estimated total ~43 GiB of
|
||||||
|
# the 64 GiB SVM pool, leaving ~21 GiB headroom.
|
||||||
|
CTX_SIZE=131072
|
||||||
PARALLEL=1
|
PARALLEL=1
|
||||||
BATCH_SIZE=2048
|
BATCH_SIZE=2048
|
||||||
UBATCH_SIZE=2048
|
UBATCH_SIZE=2048
|
||||||
@@ -60,6 +62,10 @@ SPEC_DRAFT_P_SPLIT=0.10
|
|||||||
TEMPERATURE=0.7
|
TEMPERATURE=0.7
|
||||||
TOP_P=0.95
|
TOP_P=0.95
|
||||||
|
|
||||||
REASONING=off
|
# Thinking enabled (user request, 2026-08-18). deepseek format moves the
|
||||||
|
# <think> reasoning into message.reasoning_content (OpenAI-compatible);
|
||||||
|
# none would leave the thought tags unparsed in content.
|
||||||
|
REASONING=on
|
||||||
|
REASONING_FORMAT=deepseek
|
||||||
|
|
||||||
VERBOSITY=4
|
VERBOSITY=4
|
||||||
@@ -37,7 +37,7 @@ fi
|
|||||||
: "${ALIAS:=frank-default-small}"
|
: "${ALIAS:=frank-default-small}"
|
||||||
: "${BIND_ADDR:=0.0.0.0}"
|
: "${BIND_ADDR:=0.0.0.0}"
|
||||||
: "${LISTEN_PORT:=52625}"
|
: "${LISTEN_PORT:=52625}"
|
||||||
: "${CTX_SIZE:=65536}"
|
: "${CTX_SIZE:=131072}"
|
||||||
: "${PARALLEL:=1}"
|
: "${PARALLEL:=1}"
|
||||||
: "${BATCH_SIZE:=2048}"
|
: "${BATCH_SIZE:=2048}"
|
||||||
: "${UBATCH_SIZE:=2048}"
|
: "${UBATCH_SIZE:=2048}"
|
||||||
@@ -51,7 +51,8 @@ fi
|
|||||||
: "${SPEC_DRAFT_P_SPLIT:=0.10}"
|
: "${SPEC_DRAFT_P_SPLIT:=0.10}"
|
||||||
: "${TEMPERATURE:=0.7}"
|
: "${TEMPERATURE:=0.7}"
|
||||||
: "${TOP_P:=0.95}"
|
: "${TOP_P:=0.95}"
|
||||||
: "${REASONING:=off}"
|
: "${REASONING:=on}"
|
||||||
|
: "${REASONING_FORMAT:=deepseek}"
|
||||||
: "${VERBOSITY:=4}"
|
: "${VERBOSITY:=4}"
|
||||||
|
|
||||||
MODEL="$MODEL_DIR/$MODEL_FILE"
|
MODEL="$MODEL_DIR/$MODEL_FILE"
|
||||||
@@ -113,6 +114,6 @@ exec "$LLAMA_BIN" \
|
|||||||
--temp "$TEMPERATURE" \
|
--temp "$TEMPERATURE" \
|
||||||
--top-p "$TOP_P" \
|
--top-p "$TOP_P" \
|
||||||
--reasoning "$REASONING" \
|
--reasoning "$REASONING" \
|
||||||
--reasoning-format none \
|
--reasoning-format "$REASONING_FORMAT" \
|
||||||
--reasoning-budget -1 \
|
--reasoning-budget -1 \
|
||||||
-lv "$VERBOSITY"
|
-lv "$VERBOSITY"
|
||||||
Reference in new issue
Block a user