Upgrade frank-default-large to kingjones Q8_0 AGENT at full 262144 context

frank-default-large now serves the kingjones Qwen3.8-27B-ROCmFPX-Q8_0-AGENT model with a standalone MTP draft head (Q4_0), f16 K + TurboQuant V KV cache, and the Sharp chat template. Context is maxed to 262144 (one slot); reasoning is on with preserve_thinking + preserve_reasoning in the chat kwargs. - frank-default-large.env: Q8_0 AGENT model paths, CTX_SIZE=262144, REASONING=on, REASONING_FORMAT=none, CHAT_TEMPLATE_KWARGS {enable_thinking, preserve_thinking, preserve_reasoning} - serve-frank-large.sh: standalone draft-mtp profile (already switched off the Kairic/PromptForge path) - download-q8agent.sh: reproducible HF fetch of the three payload files - docs/LLM/kingjones-q8agent-rocmfpx.md: Q8_0 AGENT runbook - AGENTS.md: topology + ACTIVE runtime section + TTM/GTT load note - qwen3.8-strix-optimizations, Dankclaw Improvements: TTM pages_limit now 114 GiB (applied 2026-08-25), full 262k KV + Q8 payload fit in one slot - frank_large_config.bats: assert 262144 context, reasoning + preserve_* enabled The TTM/GTT was raised to 114 GiB (29,883,536 pages) to hold the full 262k KV.
2026-08-25 07:12:02 -07:00
parent 24efa98d56
commit 54b2d84613
11 changed files with 350 additions and 129 deletions
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# Download the kingjones Qwen3.8-27B-ROCmFPX-Q8_0-AGENT payload for
+# frank-default-large from HuggingFace into /home/dank/models/.../Q8_0-AGENT.
+#
+# Uses the token in ~/.cache/huggingface/token so the ~28 GB main file does not
+# hit the anonymous 10 GB/30 min rate limit. Run on Dankclaw.
+#
+#   bash scripts/download-q8agent.sh
+#   bash scripts/download-q8agent.sh --verify-only
+set -euo pipefail
+
+REPO="kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF"
+DIR="/home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT"
+EXPECTED=(
+  "Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf:28214425952"
+  "mmproj-Qwen3.8-27B-BF16.gguf:931145888"
+  "mtp-Qwen3.8-27B-Q4_0.gguf:1680271648"
+)
+
+cd -- "$(dirname -- "$0")/.."
+
+main() {
+  local mode="download"
+  [[ "${1:-}" == "--verify-only" ]] && mode="verify"
+  mkdir -p "$DIR"
+  cd "$DIR"
+
+  local token
+  token="$(cat "$HOME/.cache/huggingface/token")"
+  [[ -n "$token" ]] || { echo "no HF token in $HOME/.cache/huggingface/token" >&2; exit 1; }
+
+  dl() {
+    local f="$1"
+    curl -fsSL -C - -o "$f" --retry 5 --retry-delay 3 --retry-max-time 3600 \
+      -H "Authorization: Bearer $token" \
+      "https://huggingface.co/$REPO/resolve/main/$f"
+    echo "  done: $f ($(du -h "$f" | cut -f1))"
+  }
+
+  if [[ "$mode" == "download" ]]; then
+    echo "downloading $REPO into $DIR"
+    dl "Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf"
+    dl "mmproj-Qwen3.8-27B-BF16.gguf"
+    dl "mtp-Qwen3.8-27B-Q4_0.gguf"
+  fi
+
+  echo "verifying sizes"
+  local ok=1 f want act
+  for entry in "${EXPECTED[@]}"; do
+    f="${entry%%:*}"; want="${entry##*:}"
+    act="$(stat -c%s "$f")"
+    if [[ "$act" -eq "$want" ]]; then
+      echo "  OK   $f ($act bytes)"
+    else
+      echo "  BAD  $f ($act bytes, want $want)" >&2
+      ok=0
+    fi
+  done
+  [[ "$ok" -eq 1 ]] || exit 1
+  echo "all files present and sized correctly"
+}
+
+main "$@"
@@ -1,65 +1,56 @@
-# Tunables for frank-default-large (CIRU/Kairic Edge ROCmFPX v1.2, dankclaw,
-# Strix Halo gfx1151).
+# Tunables for frank-default-large (kingjones Qwen3.8-27B ROCmFPX Q8 AGENT,
+# dankclaw, Strix Halo gfx1151).
 #
-# Runtime: ciru-ai/ROCmFPX tag kairic-edge-qwen38-27b-v1.2, commit
-# 205a3e5f40e5542e2f2eb68e3d3f81f918b1d895, built with the pinned gfx1151
-# Prompt Forge / IU4 settings against /opt/rocm. Serves the Kairic Edge
-# authoritative GGUF plus three required .pfs sidecars. Alias
+# Runtime: ROCmFPX llama.cpp with TurboQuant support. Serves the kingjones
+# AGENT GGUF plus its standalone MTP draft and vision projector. Alias
 # frank-default-large, port 7770.

 # WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The ROCm
 # HIP runtime reads those bare env vars (see the 2026-08-13 TEMP incident).
 # llama.cpp reads --host/--port/--temp as CLI flags, never as env vars.

-# ciru-ai/ROCmFPX Kairic Edge v1.2 build.
+# ROCmFPX build with TurboQuant KV-cache support.
 LLAMA_BIN=/home/dank/ciru-qwen38/ROCmFPX/build-kairic/bin/llama-server

-# Model files: authoritative GGUF plus all required Kairic Edge sidecars.
-MODEL_DIR=/home/dank/models/Kairic-Qwen3.8-27B-IU4-Edge
-MODEL_FILE=Qwen3.8-27B-IU4-Kairic-Edge.gguf
-KAIRIC_FFN_SIDECAR=$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs
-KAIRIC_GDN_SIDECAR=$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs
-KAIRIC_GDN_OUTPUT_SIDECAR=$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs
-KAIRIC_EDGE_COMPATIBILITY_MODE=1
-# No separate MTP draft head: the MTP/NextN head is embedded in the main GGUF,
-# so no --model-draft is passed. Kept for scripts that reference the var.
-MTP_DRAFT_FILE=
+# Model files from https://huggingface.co/kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF.
+MODEL_DIR=/home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT
+MODEL_FILE=Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf
+MTP_DRAFT_FILE=$MODEL_DIR/mtp-Qwen3.8-27B-Q4_0.gguf
+MMPROJ_FILE=mmproj-Qwen3.8-27B-BF16.gguf

 # Qwen-Sharp chat template (tracked in the repo, staged on the host). The Sharp
 # template drives thinking on by default (medium effort) and produces the
 # peg-native + deepseek chat/reasoning shape the Pi agent expects.
 CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
-CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true}'
+CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true,"preserve_reasoning":true}'

 # Alias + network.
 ALIAS=frank-default-large
 BIND_ADDR=0.0.0.0
 LISTEN_PORT=7770

-# Serving: 65536 context (192K below the model's 262144 maximum), 1 slot. f16
-# KV, context checkpoints, prompt cache, strict-MTP embedded draft on ROCm0.
-# This leaves sufficient unified-memory headroom for the model plus KV cache.
+# Serving: 262144 context (the model's maximum), 1 slot. f16 KV, context
+# checkpoints, prompt cache, standalone MTP draft on ROCm0. The full 262144
+# context KV cache plus the ~28-30 GB Q8 model + MTP draft + mmproj fit in the
+# lifted 114 GiB TTM/GTT budget on the Strix Halo (see the GTT 114 GiB change).
 # Concurrent requests queue; the server never preempts a running generation.
-CTX_SIZE=65536
+CTX_SIZE=262144
 PARALLEL=1
-# Batch/ubatch follow the qualified ROCmI4 W4A4 profile (see the bench in
-# scripts/bench-ubatch-rocmi4.sh for the ubatch sweep on 8k contexts).
+# Batch/ubatch follow the existing qualified Dankclaw profile.
 BATCH_SIZE=512
 UBATCH_SIZE=512
 NUM_THREADS=16
 THREAD_BATCH=32
 CACHE_TYPE_K=f16
-CACHE_TYPE_V=f16
+CACHE_TYPE_V=turbo4
 CTX_CHECKPOINTS=32
 CACHE_RAM=8192

-# Embedded-MTP (qwen35 NextN) qualified profile. The head is in the main GGUF,
-# so the draft runs with all layers on ROCm0 (no --model-draft).
+# Standalone MTP draft head, loaded from MTP_DRAFT_FILE by the serve script.
 MTP_N_MAX=4
 MTP_N_MIN=0
 MTP_P_MIN=0.0
 MTP_P_SPLIT=0.10
-KAIRIC_EDGE_COMPATIBILITY_MODE=1

 # Sampling (Sharp default).
 TEMPERATURE=0.7
@@ -68,9 +59,10 @@ TOP_P=0.8
 MIN_P=0.0
 PRESENCE_PENALTY=1.5

-# Thinking on (Sharp template drives it; reasoning format projects the tar into
-# reasoning_content).
-REASONING=off
+# Reasoning on (Sharp template drives it; preserve_thinking + preserve_reasoning
+# keep the reasoning trail in the conversation). reasoning-format none lets the
+# template project the reasoning content into reasoning_content.
+REASONING=on
 REASONING_FORMAT=none

 # Strix Halo unified memory + native gfx1151 detection for this ROCmFPX build.
@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
-# Serve Qwen3.8-27B IU4 Kairic Edge v1.2 as frank-default-large.
-# The Kairic runtime requires its authoritative GGUF and all three Prompt Forge
-# sidecars. Compatibility mode is enabled for Qwen's recommended sampling.
+# Serve the kingjones Qwen3.8-27B ROCmFPX Q8 AGENT model as frank-default-large.
+# The deployment uses a standalone MTP draft head and Turbo4 V-side KV cache.
 set -euo pipefail

 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
@@ -14,11 +13,10 @@ if [[ -f "$ENV_FILE" ]]; then
 fi

 : "${LLAMA_BIN:=/home/dank/ciru-qwen38/ROCmFPX/build-kairic/bin/llama-server}"
-: "${MODEL_DIR:=/home/dank/models/Kairic-Qwen3.8-27B-IU4-Edge}"
-: "${MODEL_FILE:=Qwen3.8-27B-IU4-Kairic-Edge.gguf}"
-: "${KAIRIC_FFN_SIDECAR:=$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs}"
-: "${KAIRIC_GDN_SIDECAR:=$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs}"
-: "${KAIRIC_GDN_OUTPUT_SIDECAR:=$MODEL_DIR/Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs}"
+: "${MODEL_DIR:=/home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT}"
+: "${MODEL_FILE:=Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf}"
+: "${MTP_DRAFT_FILE:=$MODEL_DIR/mtp-Qwen3.8-27B-Q4_0.gguf}"
+: "${MMPROJ_FILE:=mmproj-Qwen3.8-27B-BF16.gguf}"
 : "${ALIAS:=frank-default-large}"
 : "${BIND_ADDR:=0.0.0.0}"
 : "${LISTEN_PORT:=7770}"
@@ -29,7 +27,7 @@ fi
 : "${NUM_THREADS:=16}"
 : "${THREAD_BATCH:=32}"
 : "${CACHE_TYPE_K:=f16}"
-: "${CACHE_TYPE_V:=f16}"
+: "${CACHE_TYPE_V:=turbo4}"
 : "${CTX_CHECKPOINTS:=32}"
 : "${CACHE_RAM:=8192}"
 : "${MTP_N_MAX:=4}"
@@ -43,11 +41,10 @@ fi
 : "${PRESENCE_PENALTY:=1.5}"
 : "${REASONING:=off}"
 : "${REASONING_FORMAT:=none}"
-: "${KAIRIC_EDGE_COMPATIBILITY_MODE:=1}"
 : "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}"

 MODEL="$MODEL_DIR/$MODEL_FILE"
-for f in "$LLAMA_BIN" "$MODEL" "$KAIRIC_FFN_SIDECAR" "$KAIRIC_GDN_SIDECAR" "$KAIRIC_GDN_OUTPUT_SIDECAR"; do
+for f in "$LLAMA_BIN" "$MODEL" "$MTP_DRAFT_FILE" "$MODEL_DIR/$MMPROJ_FILE" "$CHAT_TEMPLATE_FILE"; do
    if [[ ! -r "$f" ]]; then
        echo "missing or unreadable required artifact: $f" >&2
        exit 1
@@ -57,11 +54,6 @@ if [[ ! -x "$LLAMA_BIN" ]]; then
    echo "missing executable runtime: $LLAMA_BIN" >&2
    exit 1
 fi
-if [[ "$KAIRIC_EDGE_COMPATIBILITY_MODE" != 1 ]]; then
-    echo "KAIRIC_EDGE_COMPATIBILITY_MODE must be 1 for this deployment" >&2
-    exit 1
-fi
-
 SERVER_DIR="$(cd -- "$(dirname -- "$LLAMA_BIN")" && pwd)"
 ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
 LIBRARY_PATH="$SERVER_DIR:$ROCM_PATH/lib:$ROCM_PATH/lib/rocm_sysdeps/lib:$ROCM_PATH/lib/llvm/lib:$ROCM_PATH/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
@@ -74,35 +66,14 @@ exec /usr/bin/env \
    GGML_CUDA_GRAPH_OPT=0 \
    LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0 \
    LLAMA_MTP_CPU_ARGMAX_FASTPATH=1 \
-    PROMPTFORGE_TARGET_ONLY=0 \
-    PROMPTFORGE_MODE=iu4_ffn \
-    PROMPTFORGE_IU4_SIDECAR="$KAIRIC_FFN_SIDECAR" \
-    PROMPTFORGE_SIDECAR="$KAIRIC_FFN_SIDECAR" \
-    PROMPTFORGE_IU4_HADAMARD=1 \
-    PROMPTFORGE_IU4_SEGMENTED=0 \
-    PROMPTFORGE_ENABLE_FFN_KEEPERS=0 \
-    PROMPTFORGE_FFN_KEEPERS=late6 \
-    PROMPTFORGE_ENABLE_GDN=1 \
-    PROMPTFORGE_ENABLE_GDN_W8=0 \
-    PROMPTFORGE_GDN_SIDECAR_OVERRIDE="$KAIRIC_GDN_SIDECAR" \
-    PROMPTFORGE_GDN_SIDECAR="$KAIRIC_GDN_SIDECAR" \
-    PROMPTFORGE_GDN_IU4_HADAMARD=1 \
-    PROMPTFORGE_ENABLE_GDN_KEEPERS=0 \
-    PROMPTFORGE_ENABLE_GDN_OUTPUT=1 \
-    PROMPTFORGE_GDN_OUTPUT_SIDECAR="$KAIRIC_GDN_OUTPUT_SIDECAR" \
-    PROMPTFORGE_GDN_OUTPUT_KEEPERS=v3_lateq6 \
-    PROMPTFORGE_ENABLE_SMALLM_IU4=1 \
-    PROMPTFORGE_ENABLE_SMALLM_GDN_IU4=1 \
-    PROMPTFORGE_ENABLE_SMALLM_GDN_OUTPUT_IU4=0 \
-    PROMPTFORGE_ENABLE_IU4_DECODE_M2_M5=0 \
-    PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0 \
    "$LLAMA_BIN" -m "$MODEL" --alias "$ALIAS" \
    --host "$BIND_ADDR" --port "$LISTEN_PORT" --jinja \
    -dev ROCm0 -ngl 999 -np "$PARALLEL" -c "$CTX_SIZE" \
    -b "$BATCH_SIZE" -ub "$UBATCH_SIZE" -fa on \
    -ctk "$CACHE_TYPE_K" -ctv "$CACHE_TYPE_V" -t "$NUM_THREADS" -tb "$THREAD_BATCH" \
    -ctxcp "$CTX_CHECKPOINTS" --cache-ram "$CACHE_RAM" --cache-prompt --cache-idle-slots --metrics \
-    --kairic-edge --spec-type draft-mtp \
+    --spec-type draft-mtp --model-draft "$MTP_DRAFT_FILE" \
+    --mmproj "$MODEL_DIR/$MMPROJ_FILE" \
    --spec-draft-device ROCm0 --spec-draft-ngl 999 \
    --spec-draft-type-k "$CACHE_TYPE_K" --spec-draft-type-v "$CACHE_TYPE_V" \
    --spec-draft-n-max "$MTP_N_MAX" --spec-draft-n-min "$MTP_N_MIN" \