frank-medium: serve the new R9700s, retire frank-small
The RTX 3090 left Dankserver; two AMD Radeon AI PRO R9700 cards (Navi 48, gfx1201, 32,624 MiB each) replaced it. FreeToken/CUDA cannot run there at all, so the `frank-small` service is retired and `frank-medium` takes the endpoint. New service: charlie12345/ROCmFPX llama.cpp (built for gfx1201 against the fork's self-contained ROCm 7.14 toolchain, so the host's stale /opt/rocm-6.3.3 never participates) serving kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF -- a Q8_0_ROCMFPX_AGENT (ftype 115) checkpoint that stock llama.cpp cannot load. Placeholder config: --split-mode tensor across both cards, 131072 context, 1 slot, f16 KV, MTP draft head on ROCm0. deploy-remote.sh grows retire_root_llm_small(): the CUDA-era unit is stopped, disabled and moved to ~/.AGENTTRASH/20260915-frank-small-retire/host-units before frank-medium starts, because `systemctl disable` has been observed to delete the unit file itself on this host (2026-08-26 incident). The retired frank-small repo files (env, unit, serve script) move to scripts/.AGENTTRASH/, and the planner, dispatcher, service allow-list and the deployment test seams now name frank-medium.
This commit is contained in:
Frank
@@ -60,7 +60,7 @@ SERVICE_ORDER = (
|
||||
"lidarr",
|
||||
"readarr",
|
||||
"frank-large",
|
||||
"frank-small",
|
||||
"frank-medium",
|
||||
)
|
||||
|
||||
DANKCLAW_SERVICES = {
|
||||
@@ -107,7 +107,7 @@ DANKSERVER_SERVICES = {
|
||||
"sonarr",
|
||||
"lidarr",
|
||||
"readarr",
|
||||
"frank-small",
|
||||
"frank-medium",
|
||||
}
|
||||
|
||||
# FrankBook-Pro runs the *arr stack + qBittorrent inside a podman machine on
|
||||
@@ -368,12 +368,13 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
|
||||
if parts[0] == "scripts":
|
||||
name = p.name
|
||||
if name in {"frank-small.env", "frank-small.service", "serve-frank-small.sh"}:
|
||||
_add(services, "frank-small")
|
||||
if name in {"frank-medium.env", "frank-medium.service", "serve-frank-medium.sh"}:
|
||||
_add(services, "frank-medium")
|
||||
elif name == "chat-template.jinja":
|
||||
# Only frank-small consumes this template now: frank-large runs
|
||||
# halogen-flash-server, which owns its own chat handling.
|
||||
_add(services, "frank-small")
|
||||
# frank-medium reads this template only when its env file points
|
||||
# at it (the GGUF's embedded template is the default); frank-large
|
||||
# runs halogen-flash-server, which owns its own chat handling.
|
||||
_add(services, "frank-medium")
|
||||
elif name in {"homepage-status.py", "homepage-status.service", "homepage-status.timer"}:
|
||||
_add(services, "homepage")
|
||||
# Deployment/test/benchmark scripts and other repo tooling do not
|
||||
|
||||
+30
-10
@@ -95,7 +95,7 @@ service_is_allowed() {
|
||||
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:plex|dankserver:tautulli|\
|
||||
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
|
||||
dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
|
||||
dankserver:frank-small|\
|
||||
dankserver:frank-medium|\
|
||||
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
|
||||
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
|
||||
frankbook-pro:readarr)
|
||||
@@ -181,7 +181,6 @@ add_pending_services() {
|
||||
continue
|
||||
fi
|
||||
[[ -n "$recorded" && "$recorded" != "$REVISION" ]] || continue
|
||||
if [[ "$TARGET" == dankclaw && "$service" == frank-small ]]; then continue; fi
|
||||
if [[ "$TARGET" == dankserver && "$service" == frank-large ]]; then continue; fi
|
||||
if ! changed="$(python3 scripts/deploy-plan.py --target "$TARGET" \
|
||||
--repo "$REMOTE_DIR" --base "$recorded" --head "$REVISION")"; then
|
||||
@@ -992,14 +991,35 @@ wait_for_llm_large_ready() {
|
||||
return 0
|
||||
}
|
||||
|
||||
deploy_llm_small() {
|
||||
run_privileged systemctl stop frank-default-small || true
|
||||
run_privileged systemctl disable frank-default-small || true
|
||||
relink_unit frank-small
|
||||
# deploy_llm_medium — frank-medium replaced frank-small on 2026-09-15 when the
|
||||
# RTX 3090 left the host for two Radeon AI PRO R9700 cards. The old unit is a
|
||||
# CUDA-era FreeToken service that can only crash-loop on this hardware, so it
|
||||
# is retired here (stop, disable, move the installed unit to the trash) before
|
||||
# its replacement starts. Same failure mode as frank-large's retirement: this
|
||||
# host's `systemctl disable` has been observed to delete the unit file itself
|
||||
# (2026-08-26 incident), so both outcomes are tolerated.
|
||||
retire_root_llm_small() {
|
||||
local trash="$HOME/.AGENTTRASH/20260915-frank-small-retire/host-units"
|
||||
run_privileged systemctl stop frank-default-small 2>/dev/null || true
|
||||
run_privileged systemctl disable frank-default-small 2>/dev/null || true
|
||||
run_privileged systemctl stop frank-small.service 2>/dev/null || true
|
||||
run_privileged systemctl disable frank-small.service 2>/dev/null || true
|
||||
if [[ -e /etc/systemd/system/frank-small.service || -L /etc/systemd/system/frank-small.service ]]; then
|
||||
mkdir -p "$trash"
|
||||
run_privileged mv -- /etc/systemd/system/frank-small.service \
|
||||
"$trash/frank-small.service"
|
||||
echo "retired root unit frank-small.service -> $trash"
|
||||
fi
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-small
|
||||
run_privileged systemctl restart frank-small
|
||||
wait_for_health frank-small http://127.0.0.1:7770/health 1200
|
||||
}
|
||||
|
||||
deploy_llm_medium() {
|
||||
retire_root_llm_small
|
||||
relink_unit frank-medium
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-medium
|
||||
run_privileged systemctl restart frank-medium
|
||||
wait_for_health frank-medium http://127.0.0.1:7770/health 1200
|
||||
}
|
||||
|
||||
wait_for_health() {
|
||||
@@ -1187,7 +1207,7 @@ deploy_service() {
|
||||
seerr) deploy_seerr ;;
|
||||
marreta) deploy_marreta ;;
|
||||
frank-large) deploy_llm_large ;;
|
||||
frank-small) deploy_llm_small ;;
|
||||
frank-medium) deploy_llm_medium ;;
|
||||
hermes|hermes-dashboard|hermes-webui) deploy_hermes ;;
|
||||
romm|romm-mariadb|romm-redis|mariadb|redis) deploy_romm_stack "$service" ;;
|
||||
jellyfin) deploy_jellyfin ;;
|
||||
|
||||
+2
-2
@@ -35,7 +35,7 @@ Options:
|
||||
Examples:
|
||||
./scripts/deploy.sh dankclaw
|
||||
./scripts/deploy.sh dankclaw --service searxng
|
||||
./scripts/deploy.sh dankserver --service frank-small
|
||||
./scripts/deploy.sh dankserver --service frank-medium
|
||||
./scripts/deploy.sh dankclaw --dry-run
|
||||
EOF
|
||||
}
|
||||
@@ -145,7 +145,7 @@ valid_service() {
|
||||
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:plex|dankserver:tautulli|\
|
||||
dankserver:qbittorrent|dankserver:radarr|dankserver:sonarr|\
|
||||
dankserver:lidarr|dankserver:readarr|dankserver:node-exporter|\
|
||||
dankserver:frank-small|\
|
||||
dankserver:frank-medium|\
|
||||
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
|
||||
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
|
||||
frankbook-pro:readarr)
|
||||
|
||||
+35
-12
@@ -1,12 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the kingjones Qwen3.8-27B-ROCmFPX-Q8_0-AGENT payload for
|
||||
# frank-large from HuggingFace into /home/dank/models/.../Q8_0-AGENT.
|
||||
# frank-medium from HuggingFace into
|
||||
# /home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT.
|
||||
#
|
||||
# Uses the token in ~/.cache/huggingface/token so the ~28 GB main file does not
|
||||
# hit the anonymous 10 GB/30 min rate limit. Run on Dankclaw.
|
||||
# Run on Dankserver (the box that serves it). The repo is public, so the
|
||||
# download works anonymously; a token in ~/.cache/huggingface/token is used
|
||||
# when present, which keeps the ~28 GB main file clear of the anonymous
|
||||
# rate limit at full speed.
|
||||
#
|
||||
# bash scripts/download-q8agent.sh
|
||||
# bash scripts/download-q8agent.sh --verify-only
|
||||
#
|
||||
# The script is resumable (curl -C -) and verifies every artifact's size
|
||||
# afterwards, so a truncated file fails loudly instead of load-crashing.
|
||||
set -euo pipefail
|
||||
|
||||
REPO="kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF"
|
||||
@@ -16,6 +22,12 @@ EXPECTED=(
|
||||
"mmproj-Qwen3.8-27B-BF16.gguf:931145888"
|
||||
"mtp-Qwen3.8-27B-Q4_0.gguf:1680271648"
|
||||
)
|
||||
# The model card's MTP prompt-cache fix ships in the same repo as a patch;
|
||||
# fetched for reference only. This fork's main already carries the
|
||||
# speculative boundary in the context checkpoint, so nothing applies it.
|
||||
EXTRA=(
|
||||
"patches/mtp-prompt-cache-fix.patch"
|
||||
)
|
||||
|
||||
cd -- "$(dirname -- "$0")/.."
|
||||
|
||||
@@ -25,30 +37,38 @@ main() {
|
||||
mkdir -p "$DIR"
|
||||
cd "$DIR"
|
||||
|
||||
local token
|
||||
token="$(cat "$HOME/.cache/huggingface/token")"
|
||||
[[ -n "$token" ]] || { echo "no HF token in $HOME/.cache/huggingface/token" >&2; exit 1; }
|
||||
local token="" auth=()
|
||||
if [[ -r "$HOME/.cache/huggingface/token" ]]; then
|
||||
token="$(cat "$HOME/.cache/huggingface/token")"
|
||||
auth=(-H "Authorization: Bearer $token")
|
||||
else
|
||||
echo "no HF token at $HOME/.cache/huggingface/token; downloading anonymously"
|
||||
fi
|
||||
|
||||
dl() {
|
||||
local f="$1"
|
||||
curl -fsSL -C - -o "$f" --retry 5 --retry-delay 3 --retry-max-time 3600 \
|
||||
-H "Authorization: Bearer $token" \
|
||||
mkdir -p "$(dirname "$f")"
|
||||
curl -fsSL -C - -o "$f" --retry 10 --retry-delay 5 --retry-max-time 10800 \
|
||||
"${auth[@]}" \
|
||||
"https://huggingface.co/$REPO/resolve/main/$f"
|
||||
echo " done: $f ($(du -h "$f" | cut -f1))"
|
||||
}
|
||||
|
||||
if [[ "$mode" == "download" ]]; then
|
||||
echo "downloading $REPO into $DIR"
|
||||
dl "Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf"
|
||||
dl "mmproj-Qwen3.8-27B-BF16.gguf"
|
||||
dl "mtp-Qwen3.8-27B-Q4_0.gguf"
|
||||
for entry in "${EXPECTED[@]}"; do
|
||||
dl "${entry%%:*}"
|
||||
done
|
||||
for f in "${EXTRA[@]}"; do
|
||||
dl "$f"
|
||||
done
|
||||
fi
|
||||
|
||||
echo "verifying sizes"
|
||||
local ok=1 f want act
|
||||
for entry in "${EXPECTED[@]}"; do
|
||||
f="${entry%%:*}"; want="${entry##*:}"
|
||||
act="$(stat -c%s "$f")"
|
||||
act="$(stat -c%s "$f" 2>/dev/null || echo 0)"
|
||||
if [[ "$act" -eq "$want" ]]; then
|
||||
echo " OK $f ($act bytes)"
|
||||
else
|
||||
@@ -56,6 +76,9 @@ main() {
|
||||
ok=0
|
||||
fi
|
||||
done
|
||||
if [[ -z "$token" ]]; then
|
||||
echo "note: no token was used; sizes above are the integrity check"
|
||||
fi
|
||||
[[ "$ok" -eq 1 ]] || exit 1
|
||||
echo "all files present and sized correctly"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
# Tunables for frank-medium (kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF,
|
||||
# charlie12345/ROCmFPX llama.cpp, Dankserver, 2x AMD Radeon AI PRO R9700).
|
||||
#
|
||||
# Hardware (2026-09-15): the RTX 3090 left Dankserver and two Radeon AI PRO
|
||||
# R9700 cards (Navi 48, gfx1201, RDNA4, 32,624 MiB each, PCIe 5.0 x16) took
|
||||
# its place. The Raphael iGPU (gfx1036) is still enumerated by HIP as the
|
||||
# third device. There is no XGMI link between the two cards and they sit in
|
||||
# different IOMMU groups, so any cross-GPU traffic goes over PCIe.
|
||||
#
|
||||
# Runtime: charlie12345/ROCmFPX (llama.cpp fork, ROCmFPX quant family; the
|
||||
# Q8_0_ROCMFPX_AGENT tensor type exists nowhere else - stock llama.cpp cannot
|
||||
# load this GGUF). Built for gfx1201 with the fork's self-contained ROCm
|
||||
# 7.14 toolchain so the host's stale /opt/rocm-6.3.3 install is never used:
|
||||
#
|
||||
# cd /home/dank/GitPackages/charlie-ROCmFPX
|
||||
# env JOBS=8 ./scripts/build-rocmfp4-rocm714-local.sh
|
||||
#
|
||||
# gfx1201 must be the build target: a gfx1200 binary links, loads the model
|
||||
# and then segfaults on this card (fork's docs/BUILD-AMD-ARCHITECTURES.md).
|
||||
#
|
||||
# Model: kingjones Qwen3.8-27B-ROCmFPX-Q8_0-AGENT - Q8_0_ROCMFPX_AGENT
|
||||
# (ftype 115), 26.28 GiB, 8.39 bpw, arch qwen35 (MoE), quantized from a
|
||||
# 51.3 GiB BF16 source. The AGENT routing keeps 12 more attention-output
|
||||
# projections at full Q8_0 than the plain 8-bit build, which buys a higher
|
||||
# MTP draft acceptance (0.953 vs 0.911) and that is the whole point of this
|
||||
# checkpoint. Model card documents 131,072 context.
|
||||
#
|
||||
# Parallelism (--split-mode tensor): tensor parallel splits the weights AND
|
||||
# the KV cache across both cards through the fused "meta device". It is the
|
||||
# only split mode here that gives each card half the KV cache; layer split
|
||||
# would give each card the KV of the layers it owns, with one PCIe handoff
|
||||
# per token per boundary. TENSOR mode's own constraints, all met below:
|
||||
# flash attention ON, non-quantized KV cache, and no auto-fit.
|
||||
#
|
||||
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The
|
||||
# ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
|
||||
# incident). llama.cpp reads --host/--port/--temp as CLI flags, never as
|
||||
# env vars.
|
||||
|
||||
# charlie12345/ROCmFPX llama-server (self-contained ROCm 7.14 build, RPATH
|
||||
# points at the bundled runtime libs; no LD_LIBRARY_PATH needed).
|
||||
LLAMA_BIN=/home/dank/GitPackages/charlie-ROCmFPX/build-rdna4-rocm714/bin/llama-server
|
||||
|
||||
# Model package, fetched with scripts/download-q8agent.sh:
|
||||
# Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf 26.28 GiB main weights
|
||||
# mtp-Qwen3.8-27B-Q4_0.gguf 1.68 GiB MTP draft head
|
||||
# mmproj-Qwen3.8-27B-BF16.gguf 0.93 GiB vision tower (unused)
|
||||
MODEL_DIR=/home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT
|
||||
MODEL_FILE=Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf
|
||||
DRAFT_MODEL_FILE=mtp-Qwen3.8-27B-Q4_0.gguf
|
||||
|
||||
# Alias + network. The port is the fleet's dankserver endpoint; the alias is
|
||||
# what clients ask for, and it replaces the retired `frank-small`.
|
||||
ALIAS=frank-medium
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7770
|
||||
|
||||
# Device placement. HIP_VISIBLE_DEVICES hides the iGPU (device 2) so nothing
|
||||
# can drift onto shared system RAM; DEVICES is what llama.cpp splits across.
|
||||
HIP_VISIBLE_DEVICES=0,1
|
||||
DEVICES=ROCm0,ROCm1
|
||||
SPLIT_MODE=tensor
|
||||
|
||||
# TENSOR_SPLIT empty = an even split between the two cards.
|
||||
TENSOR_SPLIT=
|
||||
|
||||
NGL=999
|
||||
# Tensor mode ignores auto-fit, and auto-fit has no business resizing a
|
||||
# context we set explicitly.
|
||||
FIT=off
|
||||
FLASH_ATTN=on
|
||||
|
||||
# 131,072 tokens, one slot (the model card's documented window). Tensor mode
|
||||
# puts half the KV cache on each card, so the window is not limited by a
|
||||
# single card's free memory.
|
||||
CTX_SIZE=131072
|
||||
PARALLEL=1
|
||||
|
||||
# 26.28 GiB of weights + MTP draft + KV cache leaves ~15 GiB per card at this
|
||||
# context, so the batch shape is chosen for prefill throughput, not memory.
|
||||
BATCH_SIZE=2048
|
||||
UBATCH_SIZE=512
|
||||
NUM_THREADS=8
|
||||
THREAD_BATCH=8
|
||||
|
||||
# f16 K/V: tensor mode refuses a quantized KV cache, and an agent checkpoint
|
||||
# is the wrong place to spend KV precision anyway.
|
||||
CACHE_TYPE_K=f16
|
||||
CACHE_TYPE_V=f16
|
||||
CTX_CHECKPOINTS=8
|
||||
CACHE_RAM=8192
|
||||
|
||||
# Sampling: the checkpoint's instruct table.
|
||||
TEMPERATURE=0.7
|
||||
TOP_P=0.8
|
||||
TOP_K=20
|
||||
MIN_P=0.0
|
||||
PRESENCE_PENALTY=1.5
|
||||
|
||||
# Reasoning on; DeepSeek format projects the trace into reasoning_content
|
||||
# instead of literal thinking tags in content.
|
||||
REASONING=on
|
||||
REASONING_FORMAT=deepseek
|
||||
REASONING_BUDGET=-1
|
||||
|
||||
# Chat template: empty = the template embedded in the GGUF. Point this at
|
||||
# /home/dank/serverAppConfigs/scripts/chat-template.jinja to force the repo's
|
||||
# Sharp agent template instead.
|
||||
CHAT_TEMPLATE_FILE=
|
||||
CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true}'
|
||||
|
||||
# MTP speculative decoding. n_max 6 / p_min 0.6 is the fork's measured
|
||||
# sweet spot for a MTP head with high acceptance; p_min 0.0 drafts
|
||||
# low-probability tokens that get rejected and makes MTP a net loss.
|
||||
SPEC_TYPE=draft-mtp
|
||||
SPEC_DRAFT_N_MAX=6
|
||||
SPEC_DRAFT_N_MIN=0
|
||||
SPEC_DRAFT_P_MIN=0.6
|
||||
SPEC_DRAFT_P_SPLIT=0.10
|
||||
SPEC_DRAFT_NGL=999
|
||||
SPEC_DRAFT_TYPE_K=f16
|
||||
SPEC_DRAFT_TYPE_V=f16
|
||||
SPEC_DRAFT_POLL=1
|
||||
SPEC_DRAFT_POLL_BATCH=1
|
||||
# The draft head must run on ONE device. With --split-mode tensor the draft
|
||||
# otherwise joins the meta-device allocation and the KV-cache build aborts:
|
||||
# ggml/src/ggml-backend.cpp:119: GGML_ASSERT(buffer) failed
|
||||
# <- ggml_backend_meta_alloc_ctx_tensors_from_buft <- llama_kv_cache ctor
|
||||
# (measured 2026-09-15 on the built runtime: identical flags are healthy with
|
||||
# --spec-draft-device ROCm0 and abort with it empty). ROCm0 is the card that
|
||||
# also holds the target model's main GPU.
|
||||
SPEC_DRAFT_DEVICE=ROCm0
|
||||
@@ -0,0 +1,16 @@
|
||||
[Unit]
|
||||
Description=charlie12345/ROCmFPX llama.cpp serving kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT qwen35 MoE (dankserver, frank-medium, 2x Radeon AI PRO R9700)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
Environment="FRANK_MEDIUM_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-medium.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-medium.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,56 +0,0 @@
|
||||
# Tunables for frank-small (batsclamp/Huihui-Qwen3.6-35B-A3B-
|
||||
# abliterated-FP8, qwen3_5_moe hybrid MoE, Dankserver, RTX 3090).
|
||||
#
|
||||
# Runtime: FreeToken v0.1.2 (edge-native MoE serving engine).
|
||||
# Installed via uv into /home/dank/freetoken/.venv (CPython 3.12).
|
||||
# FreeToken keeps expert weights in host RAM with an LRU GPU cache;
|
||||
# KV is paged with radix prefix sharing. Only the 10 full-attention
|
||||
# layers page KV (~20 KiB/token); the 30 GDN layers use constant-size
|
||||
# per-sequence state (mamba slot pool).
|
||||
#
|
||||
# Model: batsclamp/Huihui-Qwen3.6-35B-A3B-abliterated-FP8 (HuggingFace
|
||||
# safetensors). Abliterated (refusal-removed) Huihui finetune of
|
||||
# Qwen3.6-35B-A3B, quantized to the SAME native format the previous
|
||||
# official checkpoint used: modelopt block-fp8 e4m3, dynamic
|
||||
# activations, 128x128 weight_block_size, bf16 weight_scale_inv;
|
||||
# lm_head/embeds/norms/router gates/visual/MTP kept BF16. Un-fused
|
||||
# per-expert layout matches the official Qwen FP8, so the FreeToken
|
||||
# loader path is identical; ~35 GB on disk. Ships the fixed Qwen3.6
|
||||
# chat template (froggeric/Qwen-Fixed-Chat-Templates) — FreeToken
|
||||
# reads it from the checkpoint; no template override needed.
|
||||
# Replaced Qwen/Qwen3.6-35B-A3B-FP8 on 2026-08-30; pool numbers
|
||||
# unchanged — same arch, same weight mass.
|
||||
#
|
||||
# Pool sizing: GPU budget (~17.6 GiB after weights + fixed caches) is
|
||||
# split MoE-first. MOE_CACHE_SIZE pins resident expert slots; NUM_TOKENS
|
||||
# pins the KV pool. At 2048 slots / 453005 tokens the split is ~20%
|
||||
# residency (2048/10240) and ~8.6 GiB KV. Live resize via
|
||||
# `ft ctl cache --kv N --moe M` can grow KV by shrinking MoE without
|
||||
# restart (the reverse requires a restart).
|
||||
#
|
||||
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE.
|
||||
# The ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
|
||||
# incident). This box is NVIDIA/CUDA, but the rule keeps the env portable.
|
||||
|
||||
# Model directory (safetensors checkpoint, ~35 GB, block-fp8).
|
||||
MODEL_DIR=/home/dank/models/huihui-qwen36-35b-a3b-abliterated-fp8
|
||||
|
||||
# FreeToken venv (uv-managed CPython 3.12, ft CLI, flashinfer, torch+cu130).
|
||||
FT_VENV=/home/dank/freetoken/.venv
|
||||
CUDA_HOME_DIR=/home/dank/freetoken/.venv/lib/python3.12/site-packages/nvidia/cu13
|
||||
|
||||
ALIAS=frank-small
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7770
|
||||
|
||||
# Expert residency: 2048 of 10240 total expert instances (20%).
|
||||
# Each slot ~3 MiB → ~6.0 GiB VRAM for the MoE cache.
|
||||
MOE_CACHE_SIZE=2048
|
||||
|
||||
# KV pool: 453005 tokens (~8.6 GiB at ~20 KiB/token).
|
||||
# Covers 4 concurrent sessions at ~113k unique context each
|
||||
# (more with radix prefix sharing for common prefixes).
|
||||
NUM_TOKENS=453005
|
||||
|
||||
# Max concurrent decoding requests (also caps mamba slot allocation).
|
||||
MAX_RUNNING_REQUESTS=4
|
||||
@@ -1,16 +0,0 @@
|
||||
[Unit]
|
||||
Description=FreeToken Huihui-Qwen3.6-35B-A3B-abliterated-FP8 qwen3_5_moe hybrid MoE (dankserver, frank-small)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
Environment="FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Executable
+213
@@ -0,0 +1,213 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF as frank-medium via
|
||||
# the charlie12345/ROCmFPX llama.cpp runtime on Dankserver
|
||||
# (2x AMD Radeon AI PRO R9700, Navi 48, gfx1201, 32 GB each).
|
||||
#
|
||||
# Runtime: charlie12345/ROCmFPX main (c49ebdb, 2026-08-22), built for
|
||||
# gfx1201 with the fork's self-contained ROCm toolchain:
|
||||
#
|
||||
# cd /home/dank/GitPackages/charlie-ROCmFPX
|
||||
# env JOBS=8 ./scripts/build-rocmfp4-rocm714-local.sh
|
||||
# -> build-rdna4-rocm714/bin/llama-server
|
||||
#
|
||||
# That script downloads the gfx120X-all TheRock ROCm 7.14 toolchain into
|
||||
# ~/.local/share/rocm-7.14.0a20260624, builds with
|
||||
# -DCMAKE_HIP_ARCHITECTURES="gfx1200;gfx1201", copies the ROCm runtime
|
||||
# (hip/hsa/rocblas/hipblaslt + their kernel libraries) next to the binary and
|
||||
# sets RPATH=$ORIGIN/../lib on it. The resulting service is self-contained:
|
||||
# no LD_LIBRARY_PATH, and the host's stale /opt/rocm-6.3.3 cannot leak in.
|
||||
# gfx1201 is NOT interchangeable with gfx1200 on this card (the fork's
|
||||
# docs/BUILD-AMD-ARCHITECTURES.md: a gfx1200 build links, loads and then
|
||||
# segfaults), and the build verifies the emitted code objects afterwards.
|
||||
#
|
||||
# Model: kingjones Qwen3.8-27B-ROCmFPX-Q8_0-AGENT (Q8_0_ROCMFPX_AGENT,
|
||||
# ftype 115, arch qwen35 MoE), 26.28 GiB main file, plus a separate
|
||||
# MTP draft head and an mmproj vision tower. 8.39 bpw; the AGENT routing
|
||||
# keeps 13 of the 17 attention-output projections at full Q8_0, which is
|
||||
# what makes the MTP draft land ~0.95 of its guesses. A stock llama.cpp
|
||||
# cannot load this GGUF at all - Q8_0_ROCMFPX exists only in this fork.
|
||||
#
|
||||
# Parallelism: --split-mode tensor across both cards. Tensor parallel
|
||||
# splits the weights AND the KV cache across the participating devices via
|
||||
# the fused "meta device" (unlike -sm layer, which pipelines layers and
|
||||
# keeps each layer's KV on its own card). It requires flash attention and
|
||||
# a non-quantized KV cache, and it is disabled from auto-fit (--fit off).
|
||||
#
|
||||
# Tunables come from frank-medium.env unless FRANK_MEDIUM_ENV_FILE
|
||||
# overrides that path.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_MEDIUM_ENV_FILE:-$SCRIPT_DIR/frank-medium.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -----------------
|
||||
: "${LLAMA_BIN:=/home/dank/GitPackages/charlie-ROCmFPX/build-rdna4-rocm714/bin/llama-server}"
|
||||
: "${MODEL_DIR:=/home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT}"
|
||||
: "${MODEL_FILE:=Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf}"
|
||||
: "${DRAFT_MODEL_FILE:=mtp-Qwen3.8-27B-Q4_0.gguf}"
|
||||
: "${ALIAS:=frank-medium}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7770}"
|
||||
|
||||
# Devices: ROCm0/ROCm1 are the two R9700s; the Raphael iGPU enumerates as
|
||||
# ROCm2 (gfx1036) and is hidden from HIP so nothing can drift onto it.
|
||||
: "${HIP_VISIBLE_DEVICES:=0,1}"
|
||||
: "${DEVICES:=ROCm0,ROCm1}"
|
||||
: "${SPLIT_MODE:=tensor}"
|
||||
: "${TENSOR_SPLIT:=}"
|
||||
: "${NGL:=999}"
|
||||
: "${FIT:=off}"
|
||||
: "${FLASH_ATTN:=on}"
|
||||
|
||||
: "${CTX_SIZE:=131072}"
|
||||
: "${PARALLEL:=1}"
|
||||
: "${BATCH_SIZE:=2048}"
|
||||
: "${UBATCH_SIZE:=512}"
|
||||
: "${NUM_THREADS:=8}"
|
||||
: "${THREAD_BATCH:=8}"
|
||||
: "${CACHE_TYPE_K:=f16}"
|
||||
: "${CACHE_TYPE_V:=f16}"
|
||||
: "${CTX_CHECKPOINTS:=8}"
|
||||
: "${CACHE_RAM:=8192}"
|
||||
|
||||
# Sampling: the checkpoint's instruct table.
|
||||
: "${TEMPERATURE:=0.7}"
|
||||
: "${TOP_P:=0.8}"
|
||||
: "${TOP_K:=20}"
|
||||
: "${MIN_P:=0.0}"
|
||||
: "${PRESENCE_PENALTY:=1.5}"
|
||||
|
||||
# Reasoning: the model is an agent checkpoint; keep the trace and project it
|
||||
# into the API's reasoning_content field rather than literal thinking tags.
|
||||
: "${REASONING:=on}"
|
||||
: "${REASONING_FORMAT:=deepseek}"
|
||||
: "${REASONING_BUDGET:=-1}"
|
||||
|
||||
# Chat template: empty means "use the template embedded in the GGUF".
|
||||
: "${CHAT_TEMPLATE_FILE:=}"
|
||||
: "${CHAT_TEMPLATE_KWARGS:={\"enable_thinking\":true,\"preserve_thinking\":true}}"
|
||||
|
||||
# MTP draft head (separate GGUF, --spec-type draft-mtp). SPEC_DRAFT_DEVICE
|
||||
# must name exactly one device: under tensor mode an unsplit draft joins the
|
||||
# meta-device KV allocation and aborts while building the KV cache
|
||||
# (GGML_ASSERT(buffer) in ggml-backend.cpp, 2026-09-15).
|
||||
: "${SPEC_TYPE:=draft-mtp}"
|
||||
: "${SPEC_DRAFT_N_MAX:=6}"
|
||||
: "${SPEC_DRAFT_N_MIN:=0}"
|
||||
: "${SPEC_DRAFT_P_MIN:=0.6}"
|
||||
: "${SPEC_DRAFT_P_SPLIT:=0.10}"
|
||||
: "${SPEC_DRAFT_NGL:=999}"
|
||||
: "${SPEC_DRAFT_DEVICE:=}"
|
||||
: "${SPEC_DRAFT_TYPE_K:=f16}"
|
||||
: "${SPEC_DRAFT_TYPE_V:=f16}"
|
||||
: "${SPEC_DRAFT_POLL:=1}"
|
||||
: "${SPEC_DRAFT_POLL_BATCH:=1}"
|
||||
|
||||
# Text only: the server must not try to autoload the vision tower. Vision
|
||||
# stays on frank-large (Dankclaw), which is where the fleet routes images.
|
||||
NO_MMPROJ="${NO_MMPROJ:-1}"
|
||||
|
||||
export HIP_VISIBLE_DEVICES
|
||||
|
||||
MODEL="$MODEL_DIR/$MODEL_FILE"
|
||||
if [[ ! -x "$LLAMA_BIN" ]]; then
|
||||
echo "missing executable runtime: $LLAMA_BIN" >&2
|
||||
echo "build it with: cd $(dirname "$(dirname "$LLAMA_BIN")")/.. && env JOBS=8 ./scripts/build-rocmfp4-rocm714-local.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -r "$MODEL" ]]; then
|
||||
echo "missing or unreadable required artifact: $MODEL" >&2
|
||||
echo "fetch it with: bash $SCRIPT_DIR/download-q8agent.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n "$CHAT_TEMPLATE_FILE" && ! -r "$CHAT_TEMPLATE_FILE" ]]; then
|
||||
echo "missing or unreadable chat template: $CHAT_TEMPLATE_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- speculative decoding (MTP draft head) --------------------------------
|
||||
SPEC_ARGS=()
|
||||
if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then
|
||||
SPEC_ARGS=(
|
||||
--spec-type "$SPEC_TYPE"
|
||||
--spec-draft-n-max "$SPEC_DRAFT_N_MAX"
|
||||
--spec-draft-n-min "$SPEC_DRAFT_N_MIN"
|
||||
--spec-draft-p-min "$SPEC_DRAFT_P_MIN"
|
||||
--spec-draft-p-split "$SPEC_DRAFT_P_SPLIT"
|
||||
)
|
||||
if [[ "$SPEC_TYPE" == *draft-mtp* ]]; then
|
||||
DRAFT_MODEL="$MODEL_DIR/$DRAFT_MODEL_FILE"
|
||||
if [[ ! -r "$DRAFT_MODEL" ]]; then
|
||||
echo "missing or unreadable draft model: $DRAFT_MODEL" >&2
|
||||
exit 1
|
||||
fi
|
||||
SPEC_ARGS+=(
|
||||
--spec-draft-model "$DRAFT_MODEL"
|
||||
--spec-draft-ngl "$SPEC_DRAFT_NGL"
|
||||
--spec-draft-type-k "$SPEC_DRAFT_TYPE_K"
|
||||
--spec-draft-type-v "$SPEC_DRAFT_TYPE_V"
|
||||
--spec-draft-poll "$SPEC_DRAFT_POLL"
|
||||
--spec-draft-poll-batch "$SPEC_DRAFT_POLL_BATCH"
|
||||
)
|
||||
if [[ -n "$SPEC_DRAFT_DEVICE" ]]; then
|
||||
SPEC_ARGS+=(--spec-draft-device "$SPEC_DRAFT_DEVICE")
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- template + placement -------------------------------------------------
|
||||
TEMPLATE_ARGS=()
|
||||
if [[ -n "$CHAT_TEMPLATE_FILE" ]]; then
|
||||
TEMPLATE_ARGS+=(--chat-template-file "$CHAT_TEMPLATE_FILE")
|
||||
TEMPLATE_ARGS+=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS")
|
||||
fi
|
||||
|
||||
PLACEMENT_ARGS=(--device "$DEVICES" --split-mode "$SPLIT_MODE")
|
||||
if [[ -n "$TENSOR_SPLIT" ]]; then
|
||||
PLACEMENT_ARGS+=(--tensor-split "$TENSOR_SPLIT")
|
||||
fi
|
||||
|
||||
MMPROJ_ARGS=()
|
||||
if [[ "$NO_MMPROJ" == "1" ]]; then
|
||||
MMPROJ_ARGS=(--no-mmproj)
|
||||
fi
|
||||
|
||||
exec "$LLAMA_BIN" \
|
||||
--model "$MODEL" \
|
||||
--alias "$ALIAS" \
|
||||
--host "$BIND_ADDR" \
|
||||
--port "$LISTEN_PORT" \
|
||||
--jinja \
|
||||
--gpu-layers "$NGL" \
|
||||
--fit "$FIT" \
|
||||
--parallel "$PARALLEL" \
|
||||
--ctx-size "$CTX_SIZE" \
|
||||
--batch-size "$BATCH_SIZE" \
|
||||
--ubatch-size "$UBATCH_SIZE" \
|
||||
--flash-attn "$FLASH_ATTN" \
|
||||
--cache-type-k "$CACHE_TYPE_K" \
|
||||
--cache-type-v "$CACHE_TYPE_V" \
|
||||
--threads "$NUM_THREADS" \
|
||||
--threads-batch "$THREAD_BATCH" \
|
||||
--ctx-checkpoints "$CTX_CHECKPOINTS" \
|
||||
--cache-ram "$CACHE_RAM" \
|
||||
--cache-prompt \
|
||||
--metrics \
|
||||
--temp "$TEMPERATURE" \
|
||||
--top-p "$TOP_P" \
|
||||
--top-k "$TOP_K" \
|
||||
--min-p "$MIN_P" \
|
||||
--presence-penalty "$PRESENCE_PENALTY" \
|
||||
--reasoning "$REASONING" \
|
||||
--reasoning-format "$REASONING_FORMAT" \
|
||||
--reasoning-budget "$REASONING_BUDGET" \
|
||||
"${TEMPLATE_ARGS[@]}" \
|
||||
"${PLACEMENT_ARGS[@]}" \
|
||||
"${MMPROJ_ARGS[@]}" \
|
||||
"${SPEC_ARGS[@]}"
|
||||
@@ -1,77 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve batsclamp/Huihui-Qwen3.6-35B-A3B-abliterated-FP8 as
|
||||
# frank-small via FreeToken on Dankserver (RTX 3090, 60 GB RAM).
|
||||
#
|
||||
# Runtime: FreeToken v0.1.2 (edge-native MoE serving engine).
|
||||
# - Experts live in host RAM; an LRU cache of expert slots is resident
|
||||
# on the GPU. Misses are streamed over PCIe.
|
||||
# - Paged KV cache (radix prefix sharing). Only the 10 full-attention
|
||||
# layers page KV (~20 KiB/token); the 30 GDN/linear-attention layers
|
||||
# hold constant-size per-sequence state (mamba slots).
|
||||
# - OpenAI-compatible API (/v1/chat/completions, /v1/models).
|
||||
#
|
||||
# Model: batsclamp/Huihui-Qwen3.6-35B-A3B-abliterated-FP8 — Huihui
|
||||
# abliterated finetune of Qwen3.6-35B-A3B in the same native modelopt
|
||||
# block-fp8 format (e4m3, 128x128 weight_block_size, un-fused per-expert
|
||||
# layout) as the official FP8 checkpoint this service ran before; the
|
||||
# FreeToken loader path is unchanged. 40 layers (10 attention + 30
|
||||
# GDN), 256 experts / 8 used, 2 KV heads × 256 dim. ~35 GB on disk.
|
||||
# The checkpoint ships the fixed Qwen3.6 chat template; FreeToken reads
|
||||
# it from the model directory.
|
||||
#
|
||||
# Pool sizing: the GPU budget is split MoE-first between expert residency
|
||||
# and KV pages. --moe-cache-size pins the number of resident expert
|
||||
# slots; --num-tokens pins the KV pool capacity. Live resize is possible
|
||||
# via `ft ctl cache --kv N --moe M` (grows KV by shrinking MoE; shrinking
|
||||
# KV requires a restart due to the swap-during-rebuild memory cost).
|
||||
#
|
||||
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE
|
||||
# overrides that path.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -----------------
|
||||
: "${MODEL_DIR:=/home/dank/models/huihui-qwen36-35b-a3b-abliterated-fp8}"
|
||||
: "${ALIAS:=frank-small}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7770}"
|
||||
: "${FT_VENV:=/home/dank/freetoken/.venv}"
|
||||
: "${CUDA_HOME_DIR:=${FT_VENV}/lib/python3.12/site-packages/nvidia/cu13}"
|
||||
|
||||
# Pool sizing (expert residency vs KV cache tradeoff).
|
||||
: "${MOE_CACHE_SIZE:=2048}"
|
||||
: "${NUM_TOKENS:=453005}"
|
||||
: "${MAX_RUNNING_REQUESTS:=4}"
|
||||
|
||||
# Paths
|
||||
FT_BIN="${FT_VENV}/bin/ft"
|
||||
MODEL="${MODEL_DIR}"
|
||||
|
||||
if [[ ! -x "$FT_BIN" ]]; then
|
||||
echo "missing FreeToken CLI: $FT_BIN" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -d "$MODEL" ]]; then
|
||||
echo "missing model directory: $MODEL" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export PATH="${FT_VENV}/bin:${PATH}"
|
||||
export CUDA_HOME="${CUDA_HOME_DIR}"
|
||||
|
||||
exec "$FT_BIN" serve \
|
||||
--model "$MODEL" \
|
||||
--served-model-name "$ALIAS" \
|
||||
--host "$BIND_ADDR" \
|
||||
--port "$LISTEN_PORT" \
|
||||
--moe-cache-size "$MOE_CACHE_SIZE" \
|
||||
--num-tokens "$NUM_TOKENS" \
|
||||
--max-running-requests "$MAX_RUNNING_REQUESTS"
|
||||
@@ -179,7 +179,7 @@ for path in "${CHANGED[@]}"; do
|
||||
scripts/mount-entropy.sh)
|
||||
add_test tests/mount_entropy.bats
|
||||
;;
|
||||
scripts/frank-small.env|scripts/serve-frank-small.sh)
|
||||
scripts/frank-medium.env|scripts/serve-frank-medium.sh)
|
||||
add_test tests/llm_units.bats
|
||||
add_test tests/llm_secrets.bats
|
||||
add_test tests/scripts_syntax.bats
|
||||
|
||||
Reference in New Issue
Block a user