ling: batch 8192 (ubatch 3072); add -lv verbosity + log file for stop-investigation

This commit is contained in:
Frank
2026-08-14 02:12:35 -07:00
parent 321f946660
commit a5444e2c4d
2 changed files with 10 additions and 1 deletions
+6 -1
View File
@@ -32,8 +32,13 @@ DEVICE=Vulkan0
N_GPU_LAYERS=all
FLASH_ATTN=auto
# Batch kept small-leaning for the tight memory budget; tune with CTX_SIZE.
BATCH_SIZE=3072
BATCH_SIZE=8192
UBATCH_SIZE=3072
# Diagnostics: high verbosity + a dedicated log file for investigating the
# premature 'stop' reports. -lv 4 = verbose (ggml); 5 = debug.
VERBOSITY=4
LOG_FILE=/tmp/frank-ling.log
# Sampling defaults (model card: reasoning-capable; leave temp/top to default)
TEMPERATURE=1.0
TOP_K=20
+4
View File
@@ -47,6 +47,8 @@ fi
: "${MIN_P:=0.0}"
: "${THREADS:=12}"
: "${THREADS_HTTP:=1}"
: "${VERBOSITY:=0}" # ggml log verbosity 0-5; 4 = verbose, 5 = debug
: "${LOG_FILE:=}" # optional; if set, --log-file path
# Fork binary; stock llama.cpp cannot load bailingmoe3.
LLAMA_SERVER="${LLAMA_SERVER:-/home/dank/llama.cpp-ling/build/bin/llama-server}"
@@ -62,6 +64,7 @@ if [[ ! -f "$MODEL" ]]; then
fi
exec "$LLAMA_SERVER" \
$(if [[ -n "$LOG_FILE" ]]; then echo "--log-file $LOG_FILE"; fi) \
--model "$MODEL" \
--spec-type "$SPEC_TYPE" \
--spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
@@ -81,6 +84,7 @@ exec "$LLAMA_SERVER" \
--min-p "$MIN_P" \
--threads "$THREADS" \
--threads-http "$THREADS_HTTP" \
-lv "$VERBOSITY" \
--alias "$ALIAS" \
--host "$HOST" \
--port "$PORT"