diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 0c518fa..6daf828 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -81,6 +81,7 @@ ARGS=( --parallel "$PARALLEL_SLOTS" --n-gpu-layers 999 --jinja + --metrics --chat-template-file "$TEMPLATE_FILE" --chat-template-kwargs '{"preserve_thinking":true}' --cache-type-k q8_0 diff --git a/scripts/serve-frank-primary.sh b/scripts/serve-frank-primary.sh index 6fd1f4d..d91748d 100755 --- a/scripts/serve-frank-primary.sh +++ b/scripts/serve-frank-primary.sh @@ -56,6 +56,7 @@ fi : "${BATCH_SIZE:=8192}" : "${UBATCH_SIZE:=512}" : "${CACHE_RAM:=16384}" +: "${VERBOSITY:=4}" SERVER_BIN="$RUNTIME_DIR/llama-server" MODEL_PARTS=( @@ -99,6 +100,8 @@ ARGS=( --alias frank-primary --ctx-size "$CONTEXT_SIZE" --parallel "$PARALLEL_SLOTS" + -lv "$VERBOSITY" + --metrics --n-gpu-layers "$NGL" --tensor-split "$TENSOR_SPLIT" --jinja