ling: batch 8192 (ubatch 3072); add -lv verbosity + log file for stop-investigation
This commit is contained in:
1 parent
321f946660
commit
a5444e2c4d
2 files changed
+10
-1
No files matched your search
@@ -32,8 +32,13 @@ DEVICE=Vulkan0
|
|||||||
N_GPU_LAYERS=all
|
N_GPU_LAYERS=all
|
||||||
FLASH_ATTN=auto
|
FLASH_ATTN=auto
|
||||||
# Batch kept small-leaning for the tight memory budget; tune with CTX_SIZE.
|
# Batch kept small-leaning for the tight memory budget; tune with CTX_SIZE.
|
||||||
BATCH_SIZE=3072
|
BATCH_SIZE=8192
|
||||||
UBATCH_SIZE=3072
|
UBATCH_SIZE=3072
|
||||||
|
|
||||||
|
# Diagnostics: high verbosity + a dedicated log file for investigating the
|
||||||
|
# premature 'stop' reports. -lv 4 = verbose (ggml); 5 = debug.
|
||||||
|
VERBOSITY=4
|
||||||
|
LOG_FILE=/tmp/frank-ling.log
|
||||||
# Sampling defaults (model card: reasoning-capable; leave temp/top to default)
|
# Sampling defaults (model card: reasoning-capable; leave temp/top to default)
|
||||||
TEMPERATURE=1.0
|
TEMPERATURE=1.0
|
||||||
TOP_K=20
|
TOP_K=20
|
||||||
|
|||||||
@@ -47,6 +47,8 @@ fi
|
|||||||
: "${MIN_P:=0.0}"
|
: "${MIN_P:=0.0}"
|
||||||
: "${THREADS:=12}"
|
: "${THREADS:=12}"
|
||||||
: "${THREADS_HTTP:=1}"
|
: "${THREADS_HTTP:=1}"
|
||||||
|
: "${VERBOSITY:=0}" # ggml log verbosity 0-5; 4 = verbose, 5 = debug
|
||||||
|
: "${LOG_FILE:=}" # optional; if set, --log-file path
|
||||||
|
|
||||||
# Fork binary; stock llama.cpp cannot load bailingmoe3.
|
# Fork binary; stock llama.cpp cannot load bailingmoe3.
|
||||||
LLAMA_SERVER="${LLAMA_SERVER:-/home/dank/llama.cpp-ling/build/bin/llama-server}"
|
LLAMA_SERVER="${LLAMA_SERVER:-/home/dank/llama.cpp-ling/build/bin/llama-server}"
|
||||||
@@ -62,6 +64,7 @@ if [[ ! -f "$MODEL" ]]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
exec "$LLAMA_SERVER" \
|
exec "$LLAMA_SERVER" \
|
||||||
|
$(if [[ -n "$LOG_FILE" ]]; then echo "--log-file $LOG_FILE"; fi) \
|
||||||
--model "$MODEL" \
|
--model "$MODEL" \
|
||||||
--spec-type "$SPEC_TYPE" \
|
--spec-type "$SPEC_TYPE" \
|
||||||
--spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
|
--spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
|
||||||
@@ -81,6 +84,7 @@ exec "$LLAMA_SERVER" \
|
|||||||
--min-p "$MIN_P" \
|
--min-p "$MIN_P" \
|
||||||
--threads "$THREADS" \
|
--threads "$THREADS" \
|
||||||
--threads-http "$THREADS_HTTP" \
|
--threads-http "$THREADS_HTTP" \
|
||||||
|
-lv "$VERBOSITY" \
|
||||||
--alias "$ALIAS" \
|
--alias "$ALIAS" \
|
||||||
--host "$HOST" \
|
--host "$HOST" \
|
||||||
--port "$PORT"
|
--port "$PORT"
|
||||||
Reference in new issue
Block a user