llama.cpp runtime arm: serve-script runtime switch, harness containerless support, master build

autoresearch.sh

@@ -24,7 +24,7 @@ restore() {
     echo "[harness] tearing down candidate, restoring deployed baseline..." >&2
     ssh "$HOST" 'sudo -n systemctl stop frank-large.service 2>/dev/null;
                  podman rm -f frank-large-mimo 2>/dev/null;
-                 true' >/dev/null 2>&1 || true
+                 pkill -x llama-server 2>/dev/null; true' >/dev/null 2>&1 || true
     ssh "$HOST" 'sudo -n systemctl start frank-large.service' >/dev/null 2>&1 || true
     exit $rc
 }
@@ -40,14 +40,13 @@ rsync -a "$REPO/scripts/frank-large.env" "$REPO/scripts/serve-frank-large.sh" \
 ssh "$HOST" 'sudo -n systemctl stop frank-large.service;
              podman rm -f frank-large-mimo 2>/dev/null; true' >/dev/null
 
-# 3. Boot the candidate detached. The serve script's own gates run first;
+# 3. Boot the candidate detached; the launcher's own output IS the serve
-#    a gate failure exits here with the reason.
+#    stage output (gate failures print there before any container exists).
 echo "[harness] starting candidate..." >&2
-if ! ssh "$HOST" "setsid nohup env FRANK_LARGE_ENV_FILE=$RDIR/frank-large.env \
+ssh "$HOST" "setsid nohup env FRANK_LARGE_ENV_FILE=$RDIR/frank-large.env \
         bash $RDIR/serve-frank-large.sh >$RDIR/serve.log 2>&1 </dev/null & echo ok" \
-        | grep -q ok; then
+    | tee /tmp/autoresearch-launch.log >&2
-    echo "ERROR: could not launch candidate" >&2; exit 1
+grep -q ok /tmp/autoresearch-launch.log || { echo "ERROR: launch failed" >&2; exit 1; }
-fi
 
 # 4. Wait for health (503 while loading; anything else = boot failure).
 echo "[harness] waiting for health..." >&2
@@ -55,10 +54,15 @@ if ! ssh "$HOST" "for i in \$(seq 1 $HEALTH_TRIES); do
         code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 \
               http://127.0.0.1:7770/health 2>/dev/null || echo 000)
         [ \"\$code\" = 200 ] && echo UP && exit 0
+        st=\$(podman inspect -f '{{.State.Status}}' frank-large-mimo 2>/dev/null || echo gone)
+        if [ \"\$st\" = exited ] || { [ \"\$st\" = gone ] && ! pgrep -f serve-frank-large >/dev/null; }; then echo DOWN; exit 1; fi
         sleep 5
     done; exit 1" | grep -q UP; then
     echo "ERROR: candidate never became healthy; serve.log tail:" >&2
-    ssh "$HOST" "tail -30 $RDIR/serve.log" >&2 || true
+    ssh "$HOST" "tail -30 $RDIR/serve.log; echo '--- engine log ---';
+                 podman logs --tail 120 frank-large-mimo 2>&1 |
+                 grep -aiE 'error|raise|failed|traceback|assert|kernel' |
+                 tail -30" >&2 || true
     exit 1
 fi
 

scripts/frank-large.env

@@ -88,6 +88,10 @@ FRANK_LARGE_UTIL=0.45
 FRANK_LARGE_SPEC=
 FRANK_LARGE_SPEC_DIR=/home/dank/models/Qwen3.5-9B
 FRANK_LARGE_SPEC_TOKENS=2
+# AUTORESEARCH: torchao int8 weight-only (quark int8 + all fp8 kernels
+# rejected on gfx1151; torchao is a separate Triton dequant stack)
+FRANK_LARGE_QUANT=torchao
+FRANK_LARGE_QUANT_CONFIG='{"torchao_config":"int8_weight_only"}'
 
 # --- client contract ----------------------------------------------------------
 # vLLM emits the thinking text in `reasoning` (NOT llama.cpp/gufo's

scripts/serve-frank-large.sh

@@ -23,6 +23,39 @@ if [[ -f "$ENV_FILE" ]]; then
     set +a
 fi
 
+# --- runtime switch -----------------------------------------------------------
+# llama: bare llama-server (master build with qwen35 GDN kernels) on GGUF
+# weights. vllm (default): the digest-pinned TheRock container below.
+FRANK_LARGE_RUNTIME="${FRANK_LARGE_RUNTIME:-vllm}"
+if [[ "$FRANK_LARGE_RUNTIME" == "llama" ]]; then
+    : "${FRANK_LARGE_LLAMA_BIN:?FRANK_LARGE_LLAMA_BIN not set}"
+    : "${FRANK_LARGE_GGUF:?FRANK_LARGE_GGUF not set (target GGUF)}"
+    : "${FRANK_LARGE_DRAFT_GGUF:=}"
+    : "${FRANK_LARGE_LLAMA_CTX:=1048576}"   # 4 slots x 262,144, f16 KV
+    for f in "$FRANK_LARGE_LLAMA_BIN" "$FRANK_LARGE_GGUF"; do
+        [[ -x "$f" || -f "$f" ]] || { echo "ERROR: missing $f" >&2; exit 1; }
+    done
+    DRAFT_ARGS=()
+    if [[ -n "$FRANK_LARGE_DRAFT_GGUF" ]]; then
+        [[ -f "$FRANK_LARGE_DRAFT_GGUF" ]] || { echo "ERROR: missing draft $FRANK_LARGE_DRAFT_GGUF" >&2; exit 1; }
+        DRAFT_ARGS=(--model-draft "$FRANK_LARGE_DRAFT_GGUF" --spec-type draft-dflash)
+    fi
+    # ComfyUI co-tenancy on this iGPU: gufo-era law, 2 resident queues.
+    export GPU_MAX_HW_QUEUES=2
+    FRANK_LARGE_PORT="${FRANK_LARGE_PUBLISH:-7770:7770}"
+    FRANK_LARGE_PORT="${FRANK_LARGE_PORT##*:}"
+    exec "$FRANK_LARGE_LLAMA_BIN" \
+        --model "$FRANK_LARGE_GGUF" \
+        "${DRAFT_ARGS[@]}" \
+        --alias "${FRANK_LARGE_SERVED_NAME:=frank-large}" \
+        --host 0.0.0.0 --port "$FRANK_LARGE_PORT" \
+        --parallel 4 --ctx-size "$FRANK_LARGE_LLAMA_CTX" \
+        --n-gpu-layers 99 --flash-attn on --jinja \
+        --reasoning auto --reasoning-format deepseek \
+        --cont-batching
+fi
+
+# --- below: vllm runtime ------------------------------------------------------
 : "${FRANK_LARGE_IMAGE:?FRANK_LARGE_IMAGE not set (digest-pinned kyuz0 vllm-therock image)}"
 : "${FRANK_LARGE_MODEL_DIR:?FRANK_LARGE_MODEL_DIR not set}"
 : "${FRANK_LARGE_MODEL_SHARDS:?FRANK_LARGE_MODEL_SHARDS not set}"
@@ -38,6 +71,8 @@ fi
 : "${FRANK_LARGE_SPEC:=}"
 : "${FRANK_LARGE_SPEC_TOKENS:=8}"
 : "${FRANK_LARGE_MODELS_DIR:=${FRANK_LARGE_MODEL_DIR%/*}}"
+FRANK_LARGE_QUANT="${FRANK_LARGE_QUANT:-}"
+FRANK_LARGE_QUANT_CONFIG="${FRANK_LARGE_QUANT_CONFIG:-}"
 
 # --- gates (fail FAST, before touching the running stack) ------------------
 # Image gate by REPO DIGEST, not the human-readable ref: rootless podman
@@ -104,6 +139,17 @@ if [[ -n "$FRANK_LARGE_SPEC" ]]; then
     SPEC_ARGS=(--speculative-config "{\"method\":\"$FRANK_LARGE_SPEC\",\"model\":\"$FRANK_LARGE_SPEC_CONTAINER_DIR\",\"num_speculative_tokens\":$FRANK_LARGE_SPEC_TOKENS}")
 fi
 
+QUANT_ARGS=()
+if [[ -n "$FRANK_LARGE_QUANT" ]]; then
+    # Online quantization: vLLM quantizes the BF16 checkpoint at load time
+    # (weights >= 6-bit only; KV is untouched — the context stays bf16).
+    QUANT_ARGS=(--quantization "$FRANK_LARGE_QUANT")
+    if [[ -n "$FRANK_LARGE_QUANT_CONFIG" ]]; then
+        QUANT_ARGS+=(--quantization-config "$FRANK_LARGE_QUANT_CONFIG")
+    fi
+fi
+
+
 # Remove stale container identities BEFORE starting: a crashed unit can
 # leave the name registered forever after, and the bench-window identity
 # (vllm-mimo) holds the GPU and port when a window ends without teardown.
@@ -144,6 +190,7 @@ podman run -d \
     --tool-call-parser "$FRANK_LARGE_TOOL_PARSER" \
     --language-model-only \
     --trust-remote-code \
+    "${QUANT_ARGS[@]}" \
     "${SPEC_ARGS[@]}"
 
 # Block on the container so a container death returns non-zero from ExecStart