llama.cpp runtime arm: serve-script runtime switch, harness containerless support, master build
This commit is contained in:
Frank
+12
-8
@@ -24,7 +24,7 @@ restore() {
|
||||
echo "[harness] tearing down candidate, restoring deployed baseline..." >&2
|
||||
ssh "$HOST" 'sudo -n systemctl stop frank-large.service 2>/dev/null;
|
||||
podman rm -f frank-large-mimo 2>/dev/null;
|
||||
true' >/dev/null 2>&1 || true
|
||||
pkill -x llama-server 2>/dev/null; true' >/dev/null 2>&1 || true
|
||||
ssh "$HOST" 'sudo -n systemctl start frank-large.service' >/dev/null 2>&1 || true
|
||||
exit $rc
|
||||
}
|
||||
@@ -40,14 +40,13 @@ rsync -a "$REPO/scripts/frank-large.env" "$REPO/scripts/serve-frank-large.sh" \
|
||||
ssh "$HOST" 'sudo -n systemctl stop frank-large.service;
|
||||
podman rm -f frank-large-mimo 2>/dev/null; true' >/dev/null
|
||||
|
||||
# 3. Boot the candidate detached. The serve script's own gates run first;
|
||||
# a gate failure exits here with the reason.
|
||||
# 3. Boot the candidate detached; the launcher's own output IS the serve
|
||||
# stage output (gate failures print there before any container exists).
|
||||
echo "[harness] starting candidate..." >&2
|
||||
if ! ssh "$HOST" "setsid nohup env FRANK_LARGE_ENV_FILE=$RDIR/frank-large.env \
|
||||
ssh "$HOST" "setsid nohup env FRANK_LARGE_ENV_FILE=$RDIR/frank-large.env \
|
||||
bash $RDIR/serve-frank-large.sh >$RDIR/serve.log 2>&1 </dev/null & echo ok" \
|
||||
| grep -q ok; then
|
||||
echo "ERROR: could not launch candidate" >&2; exit 1
|
||||
fi
|
||||
| tee /tmp/autoresearch-launch.log >&2
|
||||
grep -q ok /tmp/autoresearch-launch.log || { echo "ERROR: launch failed" >&2; exit 1; }
|
||||
|
||||
# 4. Wait for health (503 while loading; anything else = boot failure).
|
||||
echo "[harness] waiting for health..." >&2
|
||||
@@ -55,10 +54,15 @@ if ! ssh "$HOST" "for i in \$(seq 1 $HEALTH_TRIES); do
|
||||
code=\$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 \
|
||||
http://127.0.0.1:7770/health 2>/dev/null || echo 000)
|
||||
[ \"\$code\" = 200 ] && echo UP && exit 0
|
||||
st=\$(podman inspect -f '{{.State.Status}}' frank-large-mimo 2>/dev/null || echo gone)
|
||||
if [ \"\$st\" = exited ] || { [ \"\$st\" = gone ] && ! pgrep -f serve-frank-large >/dev/null; }; then echo DOWN; exit 1; fi
|
||||
sleep 5
|
||||
done; exit 1" | grep -q UP; then
|
||||
echo "ERROR: candidate never became healthy; serve.log tail:" >&2
|
||||
ssh "$HOST" "tail -30 $RDIR/serve.log" >&2 || true
|
||||
ssh "$HOST" "tail -30 $RDIR/serve.log; echo '--- engine log ---';
|
||||
podman logs --tail 120 frank-large-mimo 2>&1 |
|
||||
grep -aiE 'error|raise|failed|traceback|assert|kernel' |
|
||||
tail -30" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
@@ -88,6 +88,10 @@ FRANK_LARGE_UTIL=0.45
|
||||
FRANK_LARGE_SPEC=
|
||||
FRANK_LARGE_SPEC_DIR=/home/dank/models/Qwen3.5-9B
|
||||
FRANK_LARGE_SPEC_TOKENS=2
|
||||
# AUTORESEARCH: torchao int8 weight-only (quark int8 + all fp8 kernels
|
||||
# rejected on gfx1151; torchao is a separate Triton dequant stack)
|
||||
FRANK_LARGE_QUANT=torchao
|
||||
FRANK_LARGE_QUANT_CONFIG='{"torchao_config":"int8_weight_only"}'
|
||||
|
||||
# --- client contract ----------------------------------------------------------
|
||||
# vLLM emits the thinking text in `reasoning` (NOT llama.cpp/gufo's
|
||||
|
||||
@@ -23,6 +23,39 @@ if [[ -f "$ENV_FILE" ]]; then
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- runtime switch -----------------------------------------------------------
|
||||
# llama: bare llama-server (master build with qwen35 GDN kernels) on GGUF
|
||||
# weights. vllm (default): the digest-pinned TheRock container below.
|
||||
FRANK_LARGE_RUNTIME="${FRANK_LARGE_RUNTIME:-vllm}"
|
||||
if [[ "$FRANK_LARGE_RUNTIME" == "llama" ]]; then
|
||||
: "${FRANK_LARGE_LLAMA_BIN:?FRANK_LARGE_LLAMA_BIN not set}"
|
||||
: "${FRANK_LARGE_GGUF:?FRANK_LARGE_GGUF not set (target GGUF)}"
|
||||
: "${FRANK_LARGE_DRAFT_GGUF:=}"
|
||||
: "${FRANK_LARGE_LLAMA_CTX:=1048576}" # 4 slots x 262,144, f16 KV
|
||||
for f in "$FRANK_LARGE_LLAMA_BIN" "$FRANK_LARGE_GGUF"; do
|
||||
[[ -x "$f" || -f "$f" ]] || { echo "ERROR: missing $f" >&2; exit 1; }
|
||||
done
|
||||
DRAFT_ARGS=()
|
||||
if [[ -n "$FRANK_LARGE_DRAFT_GGUF" ]]; then
|
||||
[[ -f "$FRANK_LARGE_DRAFT_GGUF" ]] || { echo "ERROR: missing draft $FRANK_LARGE_DRAFT_GGUF" >&2; exit 1; }
|
||||
DRAFT_ARGS=(--model-draft "$FRANK_LARGE_DRAFT_GGUF" --spec-type draft-dflash)
|
||||
fi
|
||||
# ComfyUI co-tenancy on this iGPU: gufo-era law, 2 resident queues.
|
||||
export GPU_MAX_HW_QUEUES=2
|
||||
FRANK_LARGE_PORT="${FRANK_LARGE_PUBLISH:-7770:7770}"
|
||||
FRANK_LARGE_PORT="${FRANK_LARGE_PORT##*:}"
|
||||
exec "$FRANK_LARGE_LLAMA_BIN" \
|
||||
--model "$FRANK_LARGE_GGUF" \
|
||||
"${DRAFT_ARGS[@]}" \
|
||||
--alias "${FRANK_LARGE_SERVED_NAME:=frank-large}" \
|
||||
--host 0.0.0.0 --port "$FRANK_LARGE_PORT" \
|
||||
--parallel 4 --ctx-size "$FRANK_LARGE_LLAMA_CTX" \
|
||||
--n-gpu-layers 99 --flash-attn on --jinja \
|
||||
--reasoning auto --reasoning-format deepseek \
|
||||
--cont-batching
|
||||
fi
|
||||
|
||||
# --- below: vllm runtime ------------------------------------------------------
|
||||
: "${FRANK_LARGE_IMAGE:?FRANK_LARGE_IMAGE not set (digest-pinned kyuz0 vllm-therock image)}"
|
||||
: "${FRANK_LARGE_MODEL_DIR:?FRANK_LARGE_MODEL_DIR not set}"
|
||||
: "${FRANK_LARGE_MODEL_SHARDS:?FRANK_LARGE_MODEL_SHARDS not set}"
|
||||
@@ -38,6 +71,8 @@ fi
|
||||
: "${FRANK_LARGE_SPEC:=}"
|
||||
: "${FRANK_LARGE_SPEC_TOKENS:=8}"
|
||||
: "${FRANK_LARGE_MODELS_DIR:=${FRANK_LARGE_MODEL_DIR%/*}}"
|
||||
FRANK_LARGE_QUANT="${FRANK_LARGE_QUANT:-}"
|
||||
FRANK_LARGE_QUANT_CONFIG="${FRANK_LARGE_QUANT_CONFIG:-}"
|
||||
|
||||
# --- gates (fail FAST, before touching the running stack) ------------------
|
||||
# Image gate by REPO DIGEST, not the human-readable ref: rootless podman
|
||||
@@ -104,6 +139,17 @@ if [[ -n "$FRANK_LARGE_SPEC" ]]; then
|
||||
SPEC_ARGS=(--speculative-config "{\"method\":\"$FRANK_LARGE_SPEC\",\"model\":\"$FRANK_LARGE_SPEC_CONTAINER_DIR\",\"num_speculative_tokens\":$FRANK_LARGE_SPEC_TOKENS}")
|
||||
fi
|
||||
|
||||
QUANT_ARGS=()
|
||||
if [[ -n "$FRANK_LARGE_QUANT" ]]; then
|
||||
# Online quantization: vLLM quantizes the BF16 checkpoint at load time
|
||||
# (weights >= 6-bit only; KV is untouched — the context stays bf16).
|
||||
QUANT_ARGS=(--quantization "$FRANK_LARGE_QUANT")
|
||||
if [[ -n "$FRANK_LARGE_QUANT_CONFIG" ]]; then
|
||||
QUANT_ARGS+=(--quantization-config "$FRANK_LARGE_QUANT_CONFIG")
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
# Remove stale container identities BEFORE starting: a crashed unit can
|
||||
# leave the name registered forever after, and the bench-window identity
|
||||
# (vllm-mimo) holds the GPU and port when a window ends without teardown.
|
||||
@@ -144,6 +190,7 @@ podman run -d \
|
||||
--tool-call-parser "$FRANK_LARGE_TOOL_PARSER" \
|
||||
--language-model-only \
|
||||
--trust-remote-code \
|
||||
"${QUANT_ARGS[@]}" \
|
||||
"${SPEC_ARGS[@]}"
|
||||
|
||||
# Block on the container so a container death returns non-zero from ExecStart
|
||||
|
||||
Reference in New Issue
Block a user