From cde91c1a0035d7f88b215697f86916a87a005123 Mon Sep 17 00:00:00 2001 From: Frank Date: Thu, 1 Oct 2026 21:56:30 -0700 Subject: [PATCH] llama.cpp runtime arm: serve-script runtime switch, harness containerless support, master build --- autoresearch.sh | 20 +++++++++------ scripts/frank-large.env | 4 +++ scripts/serve-frank-large.sh | 47 ++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/autoresearch.sh b/autoresearch.sh index a52bc36..d1f007c 100755 --- a/autoresearch.sh +++ b/autoresearch.sh @@ -24,7 +24,7 @@ restore() { echo "[harness] tearing down candidate, restoring deployed baseline..." >&2 ssh "$HOST" 'sudo -n systemctl stop frank-large.service 2>/dev/null; podman rm -f frank-large-mimo 2>/dev/null; - true' >/dev/null 2>&1 || true + pkill -x llama-server 2>/dev/null; true' >/dev/null 2>&1 || true ssh "$HOST" 'sudo -n systemctl start frank-large.service' >/dev/null 2>&1 || true exit $rc } @@ -40,14 +40,13 @@ rsync -a "$REPO/scripts/frank-large.env" "$REPO/scripts/serve-frank-large.sh" \ ssh "$HOST" 'sudo -n systemctl stop frank-large.service; podman rm -f frank-large-mimo 2>/dev/null; true' >/dev/null -# 3. Boot the candidate detached. The serve script's own gates run first; -# a gate failure exits here with the reason. +# 3. Boot the candidate detached; the launcher's own output IS the serve +# stage output (gate failures print there before any container exists). echo "[harness] starting candidate..." >&2 -if ! ssh "$HOST" "setsid nohup env FRANK_LARGE_ENV_FILE=$RDIR/frank-large.env \ +ssh "$HOST" "setsid nohup env FRANK_LARGE_ENV_FILE=$RDIR/frank-large.env \ bash $RDIR/serve-frank-large.sh >$RDIR/serve.log 2>&1 &2; exit 1 -fi + | tee /tmp/autoresearch-launch.log >&2 +grep -q ok /tmp/autoresearch-launch.log || { echo "ERROR: launch failed" >&2; exit 1; } # 4. Wait for health (503 while loading; anything else = boot failure). echo "[harness] waiting for health..." >&2 @@ -55,10 +54,15 @@ if ! ssh "$HOST" "for i in \$(seq 1 $HEALTH_TRIES); do code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 \ http://127.0.0.1:7770/health 2>/dev/null || echo 000) [ \"\$code\" = 200 ] && echo UP && exit 0 + st=\$(podman inspect -f '{{.State.Status}}' frank-large-mimo 2>/dev/null || echo gone) + if [ \"\$st\" = exited ] || { [ \"\$st\" = gone ] && ! pgrep -f serve-frank-large >/dev/null; }; then echo DOWN; exit 1; fi sleep 5 done; exit 1" | grep -q UP; then echo "ERROR: candidate never became healthy; serve.log tail:" >&2 - ssh "$HOST" "tail -30 $RDIR/serve.log" >&2 || true + ssh "$HOST" "tail -30 $RDIR/serve.log; echo '--- engine log ---'; + podman logs --tail 120 frank-large-mimo 2>&1 | + grep -aiE 'error|raise|failed|traceback|assert|kernel' | + tail -30" >&2 || true exit 1 fi diff --git a/scripts/frank-large.env b/scripts/frank-large.env index 6086d8f..1bb77d4 100644 --- a/scripts/frank-large.env +++ b/scripts/frank-large.env @@ -88,6 +88,10 @@ FRANK_LARGE_UTIL=0.45 FRANK_LARGE_SPEC= FRANK_LARGE_SPEC_DIR=/home/dank/models/Qwen3.5-9B FRANK_LARGE_SPEC_TOKENS=2 +# AUTORESEARCH: torchao int8 weight-only (quark int8 + all fp8 kernels +# rejected on gfx1151; torchao is a separate Triton dequant stack) +FRANK_LARGE_QUANT=torchao +FRANK_LARGE_QUANT_CONFIG='{"torchao_config":"int8_weight_only"}' # --- client contract ---------------------------------------------------------- # vLLM emits the thinking text in `reasoning` (NOT llama.cpp/gufo's diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 5f141ee..45165f8 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -23,6 +23,39 @@ if [[ -f "$ENV_FILE" ]]; then set +a fi +# --- runtime switch ----------------------------------------------------------- +# llama: bare llama-server (master build with qwen35 GDN kernels) on GGUF +# weights. vllm (default): the digest-pinned TheRock container below. +FRANK_LARGE_RUNTIME="${FRANK_LARGE_RUNTIME:-vllm}" +if [[ "$FRANK_LARGE_RUNTIME" == "llama" ]]; then + : "${FRANK_LARGE_LLAMA_BIN:?FRANK_LARGE_LLAMA_BIN not set}" + : "${FRANK_LARGE_GGUF:?FRANK_LARGE_GGUF not set (target GGUF)}" + : "${FRANK_LARGE_DRAFT_GGUF:=}" + : "${FRANK_LARGE_LLAMA_CTX:=1048576}" # 4 slots x 262,144, f16 KV + for f in "$FRANK_LARGE_LLAMA_BIN" "$FRANK_LARGE_GGUF"; do + [[ -x "$f" || -f "$f" ]] || { echo "ERROR: missing $f" >&2; exit 1; } + done + DRAFT_ARGS=() + if [[ -n "$FRANK_LARGE_DRAFT_GGUF" ]]; then + [[ -f "$FRANK_LARGE_DRAFT_GGUF" ]] || { echo "ERROR: missing draft $FRANK_LARGE_DRAFT_GGUF" >&2; exit 1; } + DRAFT_ARGS=(--model-draft "$FRANK_LARGE_DRAFT_GGUF" --spec-type draft-dflash) + fi + # ComfyUI co-tenancy on this iGPU: gufo-era law, 2 resident queues. + export GPU_MAX_HW_QUEUES=2 + FRANK_LARGE_PORT="${FRANK_LARGE_PUBLISH:-7770:7770}" + FRANK_LARGE_PORT="${FRANK_LARGE_PORT##*:}" + exec "$FRANK_LARGE_LLAMA_BIN" \ + --model "$FRANK_LARGE_GGUF" \ + "${DRAFT_ARGS[@]}" \ + --alias "${FRANK_LARGE_SERVED_NAME:=frank-large}" \ + --host 0.0.0.0 --port "$FRANK_LARGE_PORT" \ + --parallel 4 --ctx-size "$FRANK_LARGE_LLAMA_CTX" \ + --n-gpu-layers 99 --flash-attn on --jinja \ + --reasoning auto --reasoning-format deepseek \ + --cont-batching +fi + +# --- below: vllm runtime ------------------------------------------------------ : "${FRANK_LARGE_IMAGE:?FRANK_LARGE_IMAGE not set (digest-pinned kyuz0 vllm-therock image)}" : "${FRANK_LARGE_MODEL_DIR:?FRANK_LARGE_MODEL_DIR not set}" : "${FRANK_LARGE_MODEL_SHARDS:?FRANK_LARGE_MODEL_SHARDS not set}" @@ -38,6 +71,8 @@ fi : "${FRANK_LARGE_SPEC:=}" : "${FRANK_LARGE_SPEC_TOKENS:=8}" : "${FRANK_LARGE_MODELS_DIR:=${FRANK_LARGE_MODEL_DIR%/*}}" +FRANK_LARGE_QUANT="${FRANK_LARGE_QUANT:-}" +FRANK_LARGE_QUANT_CONFIG="${FRANK_LARGE_QUANT_CONFIG:-}" # --- gates (fail FAST, before touching the running stack) ------------------ # Image gate by REPO DIGEST, not the human-readable ref: rootless podman @@ -104,6 +139,17 @@ if [[ -n "$FRANK_LARGE_SPEC" ]]; then SPEC_ARGS=(--speculative-config "{\"method\":\"$FRANK_LARGE_SPEC\",\"model\":\"$FRANK_LARGE_SPEC_CONTAINER_DIR\",\"num_speculative_tokens\":$FRANK_LARGE_SPEC_TOKENS}") fi +QUANT_ARGS=() +if [[ -n "$FRANK_LARGE_QUANT" ]]; then + # Online quantization: vLLM quantizes the BF16 checkpoint at load time + # (weights >= 6-bit only; KV is untouched — the context stays bf16). + QUANT_ARGS=(--quantization "$FRANK_LARGE_QUANT") + if [[ -n "$FRANK_LARGE_QUANT_CONFIG" ]]; then + QUANT_ARGS+=(--quantization-config "$FRANK_LARGE_QUANT_CONFIG") + fi +fi + + # Remove stale container identities BEFORE starting: a crashed unit can # leave the name registered forever after, and the bench-window identity # (vllm-mimo) holds the GPU and port when a window ends without teardown. @@ -144,6 +190,7 @@ podman run -d \ --tool-call-parser "$FRANK_LARGE_TOOL_PARSER" \ --language-model-only \ --trust-remote-code \ + "${QUANT_ARGS[@]}" \ "${SPEC_ARGS[@]}" # Block on the container so a container death returns non-zero from ExecStart