ling: batch 8192 (ubatch 3072); add -lv verbosity + log file for stop-investigation
This commit is contained in:
@@ -32,8 +32,13 @@ DEVICE=Vulkan0
|
||||
N_GPU_LAYERS=all
|
||||
FLASH_ATTN=auto
|
||||
# Batch kept small-leaning for the tight memory budget; tune with CTX_SIZE.
|
||||
BATCH_SIZE=3072
|
||||
BATCH_SIZE=8192
|
||||
UBATCH_SIZE=3072
|
||||
|
||||
# Diagnostics: high verbosity + a dedicated log file for investigating the
|
||||
# premature 'stop' reports. -lv 4 = verbose (ggml); 5 = debug.
|
||||
VERBOSITY=4
|
||||
LOG_FILE=/tmp/frank-ling.log
|
||||
# Sampling defaults (model card: reasoning-capable; leave temp/top to default)
|
||||
TEMPERATURE=1.0
|
||||
TOP_K=20
|
||||
|
||||
@@ -47,6 +47,8 @@ fi
|
||||
: "${MIN_P:=0.0}"
|
||||
: "${THREADS:=12}"
|
||||
: "${THREADS_HTTP:=1}"
|
||||
: "${VERBOSITY:=0}" # ggml log verbosity 0-5; 4 = verbose, 5 = debug
|
||||
: "${LOG_FILE:=}" # optional; if set, --log-file path
|
||||
|
||||
# Fork binary; stock llama.cpp cannot load bailingmoe3.
|
||||
LLAMA_SERVER="${LLAMA_SERVER:-/home/dank/llama.cpp-ling/build/bin/llama-server}"
|
||||
@@ -62,6 +64,7 @@ if [[ ! -f "$MODEL" ]]; then
|
||||
fi
|
||||
|
||||
exec "$LLAMA_SERVER" \
|
||||
$(if [[ -n "$LOG_FILE" ]]; then echo "--log-file $LOG_FILE"; fi) \
|
||||
--model "$MODEL" \
|
||||
--spec-type "$SPEC_TYPE" \
|
||||
--spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
|
||||
@@ -81,6 +84,7 @@ exec "$LLAMA_SERVER" \
|
||||
--min-p "$MIN_P" \
|
||||
--threads "$THREADS" \
|
||||
--threads-http "$THREADS_HTTP" \
|
||||
-lv "$VERBOSITY" \
|
||||
--alias "$ALIAS" \
|
||||
--host "$HOST" \
|
||||
--port "$PORT"
|
||||
|
||||
Reference in New Issue
Block a user