frank-default-small: 128K context and thinking on

- CTX_SIZE 65536 -> 131072 (user request). KV at 128K is 8 GiB
  (64 KiB/token f16); estimated ~43 GiB used of the 64 GiB SVM pool,
  ~21 GiB headroom.
- REASONING off -> on with REASONING_FORMAT=deepseek, so <think>
  reasoning lands in message.reasoning_content (OpenAI-compatible)
  instead of being left unparsed in content.
This commit is contained in:
Frank committed 2026-08-17 23:11:13 -07:00
1 parent 64cdfc8529
commit 2e1a78aeb4
2 files changed
+14 -7

No files matched your search

+4 -3
View File
@@ -37,7 +37,7 @@ fi
: "${ALIAS:=frank-default-small}"
: "${BIND_ADDR:=0.0.0.0}"
: "${LISTEN_PORT:=52625}"
: "${CTX_SIZE:=65536}"
: "${CTX_SIZE:=131072}"
: "${PARALLEL:=1}"
: "${BATCH_SIZE:=2048}"
: "${UBATCH_SIZE:=2048}"
@@ -51,7 +51,8 @@ fi
: "${SPEC_DRAFT_P_SPLIT:=0.10}"
: "${TEMPERATURE:=0.7}"
: "${TOP_P:=0.95}"
: "${REASONING:=off}"
: "${REASONING:=on}"
: "${REASONING_FORMAT:=deepseek}"
: "${VERBOSITY:=4}"
MODEL="$MODEL_DIR/$MODEL_FILE"
@@ -113,6 +114,6 @@ exec "$LLAMA_BIN" \
--temp "$TEMPERATURE" \
--top-p "$TOP_P" \
--reasoning "$REASONING" \
--reasoning-format none \
--reasoning-format "$REASONING_FORMAT" \
--reasoning-budget -1 \
-lv "$VERBOSITY"