llama.cpp runtime arm: serve-script runtime switch, harness containerless support, master build

scripts/frank-large.env

@@ -88,6 +88,10 @@ FRANK_LARGE_UTIL=0.45
 FRANK_LARGE_SPEC=
 FRANK_LARGE_SPEC_DIR=/home/dank/models/Qwen3.5-9B
 FRANK_LARGE_SPEC_TOKENS=2
+# AUTORESEARCH: torchao int8 weight-only (quark int8 + all fp8 kernels
+# rejected on gfx1151; torchao is a separate Triton dequant stack)
+FRANK_LARGE_QUANT=torchao
+FRANK_LARGE_QUANT_CONFIG='{"torchao_config":"int8_weight_only"}'
 
 # --- client contract ----------------------------------------------------------
 # vLLM emits the thinking text in `reasoning` (NOT llama.cpp/gufo's

scripts/serve-frank-large.sh

@@ -23,6 +23,39 @@ if [[ -f "$ENV_FILE" ]]; then
     set +a
 fi
 
+# --- runtime switch -----------------------------------------------------------
+# llama: bare llama-server (master build with qwen35 GDN kernels) on GGUF
+# weights. vllm (default): the digest-pinned TheRock container below.
+FRANK_LARGE_RUNTIME="${FRANK_LARGE_RUNTIME:-vllm}"
+if [[ "$FRANK_LARGE_RUNTIME" == "llama" ]]; then
+    : "${FRANK_LARGE_LLAMA_BIN:?FRANK_LARGE_LLAMA_BIN not set}"
+    : "${FRANK_LARGE_GGUF:?FRANK_LARGE_GGUF not set (target GGUF)}"
+    : "${FRANK_LARGE_DRAFT_GGUF:=}"
+    : "${FRANK_LARGE_LLAMA_CTX:=1048576}"   # 4 slots x 262,144, f16 KV
+    for f in "$FRANK_LARGE_LLAMA_BIN" "$FRANK_LARGE_GGUF"; do
+        [[ -x "$f" || -f "$f" ]] || { echo "ERROR: missing $f" >&2; exit 1; }
+    done
+    DRAFT_ARGS=()
+    if [[ -n "$FRANK_LARGE_DRAFT_GGUF" ]]; then
+        [[ -f "$FRANK_LARGE_DRAFT_GGUF" ]] || { echo "ERROR: missing draft $FRANK_LARGE_DRAFT_GGUF" >&2; exit 1; }
+        DRAFT_ARGS=(--model-draft "$FRANK_LARGE_DRAFT_GGUF" --spec-type draft-dflash)
+    fi
+    # ComfyUI co-tenancy on this iGPU: gufo-era law, 2 resident queues.
+    export GPU_MAX_HW_QUEUES=2
+    FRANK_LARGE_PORT="${FRANK_LARGE_PUBLISH:-7770:7770}"
+    FRANK_LARGE_PORT="${FRANK_LARGE_PORT##*:}"
+    exec "$FRANK_LARGE_LLAMA_BIN" \
+        --model "$FRANK_LARGE_GGUF" \
+        "${DRAFT_ARGS[@]}" \
+        --alias "${FRANK_LARGE_SERVED_NAME:=frank-large}" \
+        --host 0.0.0.0 --port "$FRANK_LARGE_PORT" \
+        --parallel 4 --ctx-size "$FRANK_LARGE_LLAMA_CTX" \
+        --n-gpu-layers 99 --flash-attn on --jinja \
+        --reasoning auto --reasoning-format deepseek \
+        --cont-batching
+fi
+
+# --- below: vllm runtime ------------------------------------------------------
 : "${FRANK_LARGE_IMAGE:?FRANK_LARGE_IMAGE not set (digest-pinned kyuz0 vllm-therock image)}"
 : "${FRANK_LARGE_MODEL_DIR:?FRANK_LARGE_MODEL_DIR not set}"
 : "${FRANK_LARGE_MODEL_SHARDS:?FRANK_LARGE_MODEL_SHARDS not set}"
@@ -38,6 +71,8 @@ fi
 : "${FRANK_LARGE_SPEC:=}"
 : "${FRANK_LARGE_SPEC_TOKENS:=8}"
 : "${FRANK_LARGE_MODELS_DIR:=${FRANK_LARGE_MODEL_DIR%/*}}"
+FRANK_LARGE_QUANT="${FRANK_LARGE_QUANT:-}"
+FRANK_LARGE_QUANT_CONFIG="${FRANK_LARGE_QUANT_CONFIG:-}"
 
 # --- gates (fail FAST, before touching the running stack) ------------------
 # Image gate by REPO DIGEST, not the human-readable ref: rootless podman
@@ -104,6 +139,17 @@ if [[ -n "$FRANK_LARGE_SPEC" ]]; then
     SPEC_ARGS=(--speculative-config "{\"method\":\"$FRANK_LARGE_SPEC\",\"model\":\"$FRANK_LARGE_SPEC_CONTAINER_DIR\",\"num_speculative_tokens\":$FRANK_LARGE_SPEC_TOKENS}")
 fi
 
+QUANT_ARGS=()
+if [[ -n "$FRANK_LARGE_QUANT" ]]; then
+    # Online quantization: vLLM quantizes the BF16 checkpoint at load time
+    # (weights >= 6-bit only; KV is untouched — the context stays bf16).
+    QUANT_ARGS=(--quantization "$FRANK_LARGE_QUANT")
+    if [[ -n "$FRANK_LARGE_QUANT_CONFIG" ]]; then
+        QUANT_ARGS+=(--quantization-config "$FRANK_LARGE_QUANT_CONFIG")
+    fi
+fi
+
+
 # Remove stale container identities BEFORE starting: a crashed unit can
 # leave the name registered forever after, and the bench-window identity
 # (vllm-mimo) holds the GPU and port when a window ends without teardown.
@@ -144,6 +190,7 @@ podman run -d \
     --tool-call-parser "$FRANK_LARGE_TOOL_PARSER" \
     --language-model-only \
     --trust-remote-code \
+    "${QUANT_ARGS[@]}" \
     "${SPEC_ARGS[@]}"
 
 # Block on the container so a container death returns non-zero from ExecStart