ling: batch 8192 (ubatch 3072); add -lv verbosity + log file for stop-investigation

This commit is contained in:
Frank committed 2026-08-14 02:12:35 -07:00
1 parent 321f946660
commit a5444e2c4d
2 files changed
+10 -1

No files matched your search

+6 -1
View File
@@ -32,8 +32,13 @@ DEVICE=Vulkan0
N_GPU_LAYERS=all N_GPU_LAYERS=all
FLASH_ATTN=auto FLASH_ATTN=auto
# Batch kept small-leaning for the tight memory budget; tune with CTX_SIZE. # Batch kept small-leaning for the tight memory budget; tune with CTX_SIZE.
BATCH_SIZE=3072 BATCH_SIZE=8192
UBATCH_SIZE=3072 UBATCH_SIZE=3072
# Diagnostics: high verbosity + a dedicated log file for investigating the
# premature 'stop' reports. -lv 4 = verbose (ggml); 5 = debug.
VERBOSITY=4
LOG_FILE=/tmp/frank-ling.log
# Sampling defaults (model card: reasoning-capable; leave temp/top to default) # Sampling defaults (model card: reasoning-capable; leave temp/top to default)
TEMPERATURE=1.0 TEMPERATURE=1.0
TOP_K=20 TOP_K=20
+4
View File
@@ -47,6 +47,8 @@ fi
: "${MIN_P:=0.0}" : "${MIN_P:=0.0}"
: "${THREADS:=12}" : "${THREADS:=12}"
: "${THREADS_HTTP:=1}" : "${THREADS_HTTP:=1}"
: "${VERBOSITY:=0}" # ggml log verbosity 0-5; 4 = verbose, 5 = debug
: "${LOG_FILE:=}" # optional; if set, --log-file path
# Fork binary; stock llama.cpp cannot load bailingmoe3. # Fork binary; stock llama.cpp cannot load bailingmoe3.
LLAMA_SERVER="${LLAMA_SERVER:-/home/dank/llama.cpp-ling/build/bin/llama-server}" LLAMA_SERVER="${LLAMA_SERVER:-/home/dank/llama.cpp-ling/build/bin/llama-server}"
@@ -62,6 +64,7 @@ if [[ ! -f "$MODEL" ]]; then
fi fi
exec "$LLAMA_SERVER" \ exec "$LLAMA_SERVER" \
$(if [[ -n "$LOG_FILE" ]]; then echo "--log-file $LOG_FILE"; fi) \
--model "$MODEL" \ --model "$MODEL" \
--spec-type "$SPEC_TYPE" \ --spec-type "$SPEC_TYPE" \
--spec-draft-n-max "$SPEC_DRAFT_N_MAX" \ --spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
@@ -81,6 +84,7 @@ exec "$LLAMA_SERVER" \
--min-p "$MIN_P" \ --min-p "$MIN_P" \
--threads "$THREADS" \ --threads "$THREADS" \
--threads-http "$THREADS_HTTP" \ --threads-http "$THREADS_HTTP" \
-lv "$VERBOSITY" \
--alias "$ALIAS" \ --alias "$ALIAS" \
--host "$HOST" \ --host "$HOST" \
--port "$PORT" --port "$PORT"