feat(llm): rename inference services dropping 'default' from names

frank-default-large -> frank-large (Dankclaw :7770), frank-default-small ->
frank-small (Dankserver :7770), frank-default-sightsound -> frank-npu
(Dankclaw :7771). Units, env files, serve scripts, deploy-plan mappings,
deploy-remote dispatch (stop/disable old units + relink + enable new), and the
test seams move together. Retired frank-default-npu.* artifacts moved to
scripts/.AGENTTRASH/ (tracked-in-history; ignored on disk).

Served aliases become frank-large / frank-small via --alias; the NPU endpoint
serves the whole FLM registry, so its rename changes no served tags.
This commit is contained in:
Frank
2026-09-03 16:15:43 -07:00
parent 1e491fa69f
commit 69f87c8483
34 changed files with 136 additions and 137 deletions
+3 -3
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
"""
Decode throughput benchmark for frank-default-large.
Decode throughput benchmark for frank-large.
Starts the server with a given --threads-batch value, sends an 8K+ token prompt,
generates 256 tokens, measures generation tok/s.
@@ -13,7 +13,7 @@ SERVE = "/home/dank/GitPackages/ROCmFPX/build-qwen38/bin/llama-server"
PORT = 7770
N_GEN = 256
PROMPT_FILE = "/tmp/bench-prompts/prompt_code.txt"
ENV_FILE = os.path.join(os.path.dirname(__file__), "frank-default-large.env")
ENV_FILE = os.path.join(os.path.dirname(__file__), "frank-large.env")
SERVER_DIR = os.path.dirname(SERVE)
ROCM_PATH = "/opt/rocm"
@@ -59,7 +59,7 @@ def wait_ready():
def send_request(prompt, max_tokens=N_GEN):
"""Send chat completion, measure generation throughput."""
body = json.dumps({
"model": "frank-default-large",
"model": "frank-large",
"messages": [{"role": "user", "content": prompt}],
"max_tokens": max_tokens,
"stream": False,
+3 -3
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
# Decode throughput benchmark for frank-default-large.
# Decode throughput benchmark for frank-large.
# Measures generation tok/s with 8K+ token prompts and 256 generated tokens.
set -euo pipefail
@@ -57,7 +57,7 @@ run_bench() {
# Warmup (small request)
curl -s -m 60 http://127.0.0.1:$PORT/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{"model":"frank-default-large","messages":[{"role":"user","content":"Hi"}],"max_tokens":16,"stream":false}' > /dev/null 2>&1 || true
-d '{"model":"frank-large","messages":[{"role":"user","content":"Hi"}],"max_tokens":16,"stream":false}' > /dev/null 2>&1 || true
# Build 8K prompt
local prompt
@@ -75,7 +75,7 @@ print(json.dumps(' '.join(words)))
t0=$(date +%s%N)
resp=$(curl -s -m 600 http://127.0.0.1:$PORT/v1/chat/completions \
-H "Content-Type: application/json" \
-d "{\"model\":\"frank-default-large\",\"messages\":[{\"role\":\"user\",\"content\":$prompt}],\"max_tokens\":$N_GEN,\"stream\":false}" 2>&1)
-d "{\"model\":\"frank-large\",\"messages\":[{\"role\":\"user\",\"content\":$prompt}],\"max_tokens\":$N_GEN,\"stream\":false}" 2>&1)
t1=$(date +%s%N)
if ! echo "$resp" | python3 -c "import sys,json; json.load(sys.stdin)" 2>/dev/null; then
+3 -3
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
"""
Benchmark n-gram speculative decoding vs no-spec for frank-default-large.
Benchmark n-gram speculative decoding vs no-spec for frank-large.
Tests prompt-processing throughput with 16K token prompts (manageable request size).
Code-heavy prompt (high n-gram redundancy) vs prose prompt (low redundancy).
@@ -103,7 +103,7 @@ def start_server(port, spec_type):
def run_test(prompt_text, spec_type, port):
"""Send a request and extract prompt processing metrics."""
data = json.dumps({
"model": "frank-default-large",
"model": "frank-large",
"messages": [{"role": "user", "content": prompt_text}],
"max_tokens": TEST_TOKENS,
"stream": False,
@@ -189,7 +189,7 @@ def main():
print(f" Prose: {prose_prompt_file.stat().st_size:,} bytes")
print("=" * 60)
print(" N-gram Mod Benchmark — frank-default-large")
print(" N-gram Mod Benchmark — frank-large")
print(f" STRIX ROCmFP4 · 64K context · 16K prompts")
print("=" * 60)
+6 -6
View File
@@ -1,19 +1,19 @@
#!/usr/bin/env bash
# Benchmark n-gram speculative decoding vs no-spec for frank-default-large.
# Benchmark n-gram speculative decoding vs no-spec for frank-large.
#
# This tests prompt-processing throughput at 64k context with two distinct
# prompts: one code-heavy and one prose-heavy. Uses llama-server on scratch
# ports with the OpenAI-compatible API to measure prompt_eval_duration_ms.
#
# Run on Dankclaw after stopping the production service:
# sudo systemctl stop frank-default-large
# sudo systemctl stop frank-large
# ./scripts/bench-ngram-mod.sh
# sudo systemctl start frank-default-large
# sudo systemctl start frank-large
set -euo pipefail
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}"
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
if [[ -f "$ENV_FILE" ]]; then
set -a; source "$ENV_FILE"; set +a
fi
@@ -184,7 +184,7 @@ run_single_test() {
response=$(curl -s -X POST "http://127.0.0.1:$port/v1/completions" \
-H "Content-Type: application/json" \
-d "{
\"model\": \"frank-default-large\",
\"model\": \"frank-large\",
\"prompt\": \"$prompt_text\",
\"max_tokens\": 128,
\"temperature\": 0,
@@ -220,7 +220,7 @@ run_single_test() {
}
echo "╔══════════════════════════════════════════════════════════╗"
echo "║ N-gram Mod Benchmark — frank-default-large ║"
echo "║ N-gram Mod Benchmark — frank-large ║"
echo "║ STRIX ROCmFP4 · 64k context · two prompts ║"
echo "╚══════════════════════════════════════════════════════════╝"
echo ""
+3 -3
View File
@@ -15,9 +15,9 @@ import time
import urllib.request
import urllib.error
SERVICE = "frank-default-large"
ENV_FILE = "/home/dank/serverAppConfigs/scripts/frank-default-large.env"
ALIAS = "frank-default-large"
SERVICE = "frank-large"
ENV_FILE = "/home/dank/serverAppConfigs/scripts/frank-large.env"
ALIAS = "frank-large"
DEFAULT_HOST = "localhost"
DEFAULT_PORT = 7770
DEFAULT_MAX_TOKENS = 16384
+2 -2
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
"""
HTTP-based prefill-throughput sweep for frank-default-large.
HTTP-based prefill-throughput sweep for frank-large.
Starts the server with a given --threads-batch value, sends a 64K-prompt via
the chat API (max_tokens=0), measures prompt_tokens/wall_time.
@@ -39,7 +39,7 @@ def send_prefill_request():
with open(PROMPT_FILE, "r") as f:
prompt = f.read()
body = json.dumps({
"model": "frank-default-large",
"model": "frank-large",
"messages": [{"role": "user", "content": prompt}],
"max_tokens": 1,
"stream": False
+2 -2
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
# Threads-batch sweep for frank-default-large on dankclaw
# Threads-batch sweep for frank-large on dankclaw
# Benches prefill throughput at 65536 prompt tokens across threads-batch values.
set -uo pipefail
@@ -10,7 +10,7 @@ TB_LIST=(16 32 48 64 80 96 128)
ENV_FILE="/home/dank/GitPackages/ROCmFPX/build-qwen38/bin"
echo "========================================"
echo "THREADS-BATCH SWEEP — frank-default-large"
echo "THREADS-BATCH SWEEP — frank-large"
echo "Model: $(basename "$MODEL") ($(du -h "$MODEL" | cut -f1))"
echo "Threads-batch values: ${TB_LIST[*]}"
echo "Prompt: 65536 tokens, n-gen: 0 (prefill-only)"
+4 -4
View File
@@ -1,14 +1,14 @@
#!/usr/bin/env bash
# Benchmark ubatch with llama-bench for frank-default-large.
# Benchmark ubatch with llama-bench for frank-large.
#
# This is intentionally a ubatch-only benchmark. It uses one llama-bench
# process, an 8K prompt workload, and equal batch/ubatch values. No server
# process, HTTP requests, or log scraping are used.
#
# Run on Dankclaw after stopping frank-default-large:
# sudo systemctl stop frank-default-large
# Run on Dankclaw after stopping frank-large:
# sudo systemctl stop frank-large
# ./scripts/bench-ubatch.sh
# sudo systemctl start frank-default-large
# sudo systemctl start frank-large
set -euo pipefail
: "${LLAMA_BENCH:=/home/dank/GitPackages/ROCmFPX/build-strix-rocmfp4/bin/llama-bench}"
+13 -17
View File
@@ -58,10 +58,9 @@ SERVICE_ORDER = (
"sonarr",
"lidarr",
"readarr",
"frank-default-large",
"frank-default-npu",
"frank-default-sightsound",
"frank-default-small",
"frank-large",
"frank-npu",
"frank-small",
)
DANKCLAW_SERVICES = {
@@ -89,9 +88,8 @@ DANKCLAW_SERVICES = {
"authentik-outpost-ldap",
"caddy",
"homepage",
"frank-default-large",
"frank-default-npu",
"frank-default-sightsound",
"frank-large",
"frank-npu",
}
DANKSERVER_SERVICES = {
@@ -109,7 +107,7 @@ DANKSERVER_SERVICES = {
"sonarr",
"lidarr",
"readarr",
"frank-default-small",
"frank-small",
}
# FrankBook-Pro runs the *arr stack + qBittorrent inside a podman machine on
@@ -352,17 +350,15 @@ def services_for_path(path: str, target: str) -> set[str]:
if parts[0] == "scripts":
name = p.name
if name in {"frank-default-large.env", "frank-default-large.service", "serve-frank-large.sh"}:
_add(services, "frank-default-large")
elif name in {"frank-default-npu.env", "frank-default-npu.service", "serve-frank-default-npu.sh"}:
_add(services, "frank-default-npu")
elif name in {"frank-default-sightsound.env", "frank-default-sightsound.service", "serve-frank-default-sightsound.sh"}:
_add(services, "frank-default-sightsound")
elif name in {"frank-default-small.env", "frank-default-small.service", "serve-frank-small.sh"}:
_add(services, "frank-default-small")
if name in {"frank-large.env", "frank-large.service", "serve-frank-large.sh"}:
_add(services, "frank-large")
elif name in {"frank-npu.env", "frank-npu.service", "serve-frank-npu.sh"}:
_add(services, "frank-npu")
elif name in {"frank-small.env", "frank-small.service", "serve-frank-small.sh"}:
_add(services, "frank-small")
elif name == "chat-template.jinja":
# Only the selected target's LLM consumes this shared template.
_add(services, "frank-default-large" if target == "dankclaw" else "frank-default-small")
_add(services, "frank-large" if target == "dankclaw" else "frank-small")
elif name in {"homepage-status.py", "homepage-status.service", "homepage-status.timer"}:
_add(services, "homepage")
# Deployment/test/benchmark scripts and other repo tooling do not
+37 -33
View File
@@ -90,13 +90,12 @@ service_is_allowed() {
dankclaw:authentik-postgres|dankclaw:authentik-server|\
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|\
dankclaw:frank-default-large|dankclaw:frank-default-npu|\
dankclaw:frank-default-sightsound|\
dankclaw:frank-large|dankclaw:frank-npu|\
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:plex|dankserver:tautulli|\
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
dankserver:frank-default-small|\
dankserver:frank-small|\
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
frankbook-pro:readarr)
@@ -182,8 +181,9 @@ add_pending_services() {
continue
fi
[[ -n "$recorded" && "$recorded" != "$REVISION" ]] || continue
if [[ "$TARGET" == dankclaw && "$service" == frank-default-small ]]; then continue; fi
if [[ "$TARGET" == dankserver && "$service" == frank-default-large ]]; then continue; fi
if [[ "$TARGET" == dankclaw && "$service" == frank-small ]]; then continue; fi
if [[ "$TARGET" == dankserver && "$service" == frank-large ]]; then continue; fi
if [[ "$TARGET" == dankserver && "$service" == frank-npu ]]; then continue; fi
if ! changed="$(python3 scripts/deploy-plan.py --target "$TARGET" \
--repo "$REMOTE_DIR" --base "$recorded" --head "$REVISION")"; then
echo "WARN: unable to compare state for $service; retaining it as pending" >&2
@@ -826,40 +826,45 @@ deploy_anubis() {
wait_container_healthy anubis 60 || return 1
}
# Rename cutover (2026-09-03): frank-default-large -> frank-large,
# frank-default-small -> frank-small, frank-default-sightsound -> frank-npu.
# The legacy units are stopped and disabled before the renamed unit takes the
# port so no legacy process is left holding it; stop/disable of an already
# unloaded legacy unit fail harmlessly and are tolerated.
deploy_llm_large() {
relink_unit frank-default-large
# NPU is intentionally not restarted here. It is a separate, retired
# endpoint and its unit is only relinked when explicitly selected.
run_privileged systemctl stop frank-default-large || true
run_privileged systemctl disable frank-default-large || true
relink_unit frank-large
run_privileged systemctl daemon-reload
run_privileged systemctl enable frank-default-large
run_privileged systemctl restart frank-default-large
wait_for_health frank-default-large http://127.0.0.1:7770/health 1200
run_privileged systemctl enable frank-large
run_privileged systemctl restart frank-large
wait_for_health frank-large http://127.0.0.1:7770/health 1200
}
deploy_llm_small() {
relink_unit frank-default-small
run_privileged systemctl stop frank-default-small || true
run_privileged systemctl disable frank-default-small || true
relink_unit frank-small
run_privileged systemctl daemon-reload
run_privileged systemctl enable frank-default-small
run_privileged systemctl restart frank-default-small
wait_for_health frank-default-small http://127.0.0.1:7770/health 1200
run_privileged systemctl enable frank-small
run_privileged systemctl restart frank-small
wait_for_health frank-small http://127.0.0.1:7770/health 1200
}
# frank-npu is a live serving endpoint (FLM on the XDNA2 NPU), not a
# relink-only unit: it is enabled at boot and restarted on deploy. The health
# gate polls /v1/version, FLM's cheapest always-on route (there is no /health
# route in the FLM server).
deploy_llm_npu() {
relink_unit frank-default-npu
run_privileged systemctl stop frank-default-sightsound || true
run_privileged systemctl disable frank-default-sightsound || true
run_privileged systemctl stop frank-default-npu || true
run_privileged systemctl disable frank-default-npu || true
relink_unit frank-npu
run_privileged systemctl daemon-reload
echo "Relinked frank-default-npu; it was not restarted."
}
# frank-default-sightsound is a live serving endpoint (FLM on the XDNA2 NPU),
# not a relink-only unit: it is enabled at boot and restarted on deploy. The
# health gate polls /v1/version, FLM's cheapest always-on route (there is no
# /health route in the FLM server).
deploy_llm_sightsound() {
relink_unit frank-default-sightsound
run_privileged systemctl daemon-reload
run_privileged systemctl enable frank-default-sightsound
run_privileged systemctl restart frank-default-sightsound
wait_for_health frank-default-sightsound http://127.0.0.1:7771/v1/version 300
run_privileged systemctl enable frank-npu
run_privileged systemctl restart frank-npu
wait_for_health frank-npu http://127.0.0.1:7771/v1/version 300
}
wait_for_health() {
@@ -1045,10 +1050,9 @@ deploy_service() {
;;
anubis) deploy_anubis ;;
seerr) deploy_seerr ;;
frank-default-large) deploy_llm_large ;;
frank-default-small) deploy_llm_small ;;
frank-default-npu) deploy_llm_npu ;;
frank-default-sightsound) deploy_llm_sightsound ;;
frank-large) deploy_llm_large ;;
frank-small) deploy_llm_small ;;
frank-npu) deploy_llm_npu ;;
hermes|hermes-dashboard|hermes-webui) deploy_hermes ;;
romm|romm-mariadb|romm-redis|mariadb|redis) deploy_romm_stack "$service" ;;
jellyfin) deploy_jellyfin ;;
+3 -4
View File
@@ -34,7 +34,7 @@ Options:
Examples:
./scripts/deploy.sh dankclaw
./scripts/deploy.sh dankclaw --service searxng
./scripts/deploy.sh dankserver --service frank-default-small
./scripts/deploy.sh dankserver --service frank-small
./scripts/deploy.sh dankclaw --dry-run
EOF
}
@@ -134,13 +134,12 @@ valid_service() {
dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|dankclaw:loki|dankclaw:alloy|\
dankclaw:node-exporter|dankclaw:authentik-postgres|dankclaw:authentik-server|\
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|dankclaw:seerr|\
dankclaw:frank-default-large|dankclaw:frank-default-npu|\
dankclaw:frank-default-sightsound|\
dankclaw:frank-large|dankclaw:frank-npu|\
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:plex|dankserver:tautulli|\
dankserver:qbittorrent|dankserver:radarr|dankserver:sonarr|\
dankserver:lidarr|dankserver:readarr|dankserver:node-exporter|\
dankserver:frank-default-small|\
dankserver:frank-small|\
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
frankbook-pro:readarr)
+1 -1
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
# Download the kingjones Qwen3.8-27B-ROCmFPX-Q8_0-AGENT payload for
# frank-default-large from HuggingFace into /home/dank/models/.../Q8_0-AGENT.
# frank-large from HuggingFace into /home/dank/models/.../Q8_0-AGENT.
#
# Uses the token in ~/.cache/huggingface/token so the ~28 GB main file does not
# hit the anonymous 10 GB/30 min rate limit. Run on Dankclaw.
-17
View File
@@ -1,17 +0,0 @@
[Unit]
Description=FastFlowLM NPU sight+sound inference (frank-default-sightsound) on Dankclaw
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=dank
Group=dank
Environment="FLM_SIGHTSOUND_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-default-sightsound.env"
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-default-sightsound.sh
LimitMEMLOCK=infinity
Restart=on-failure
RestartSec=10
[Install]
WantedBy=multi-user.target
@@ -1,4 +1,4 @@
# Tunables for frank-default-large (kingjones777/Qwen3.8-Flash-Next-ROCmFP4-
# Tunables for frank-large (kingjones777/Qwen3.8-Flash-Next-ROCmFP4-
# STRIX-GGUF on the kingjones30/ROCmFPX llama.cpp runtime, Dankclaw,
# Strix Halo gfx1151, unified memory).
#
@@ -56,7 +56,7 @@ CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true,"preserve_reasoning":true}'
# Alias + network.
ALIAS=frank-default-large
ALIAS=frank-large
BIND_ADDR=0.0.0.0
LISTEN_PORT=7770
@@ -1,5 +1,5 @@
[Unit]
Description=ROCmFPX (kingjones30 fork) llama.cpp Qwen3.8-Flash-Next ROCmFP4 STRIX 177B MoE (dankclaw, frank-default-large)
Description=ROCmFPX (kingjones30 fork) llama.cpp Qwen3.8-Flash-Next ROCmFP4 STRIX 177B MoE (dankclaw, frank-large)
After=network-online.target
Wants=network-online.target
@@ -7,7 +7,7 @@ Wants=network-online.target
Type=simple
User=dank
Group=dank
Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-default-large.env"
Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
Restart=on-failure
RestartSec=10
@@ -1,11 +1,11 @@
# Tunables for frank-default-sightsound (FastFlowLM NPU sight+sound endpoint, Dankclaw)
# Tunables for frank-npu (FastFlowLM NPU sight+sound endpoint, Dankclaw)
# Runtime: FLM 1.0.1 (/usr/bin/flm) on the XDNA2 NPU (50 TOPS). Model:
# gemma4-it:e4b = Gemma4-E4B-IT-NPU2 (8B total / ~4.5B effective params,
# Q4_1; vision + audio + reasoning; ~9.1 GB footprint). It is the only NPU2
# model family with BOTH an image and an audio tower — the uncensored
# candidate qwythos:9b is text-only and fails the audio requirement, so this
# endpoint trades "uncensored" for "sight + sound". The NPU draws from the
# unified RAM pool shared with frank-default-large (iGPU); ~31 GiB was
# unified RAM pool shared with frank-large (iGPU); ~31 GiB was
# available there at idle, so a 9.1 GB pack fits with headroom.
FLM_MODEL=qwen3.8-distilled-heretic:9b
FLM_HOST=0.0.0.0
+17
View File
@@ -0,0 +1,17 @@
[Unit]
Description=FastFlowLM NPU sight+sound inference (frank-npu) on Dankclaw
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=dank
Group=dank
Environment="FRANK_NPU_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-npu.env"
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-npu.sh
LimitMEMLOCK=infinity
Restart=on-failure
RestartSec=10
[Install]
WantedBy=multi-user.target
@@ -1,4 +1,4 @@
# Tunables for frank-default-small (batsclamp/Huihui-Qwen3.6-35B-A3B-
# Tunables for frank-small (batsclamp/Huihui-Qwen3.6-35B-A3B-
# abliterated-FP8, qwen3_5_moe hybrid MoE, Dankserver, RTX 3090).
#
# Runtime: FreeToken v0.1.2 (edge-native MoE serving engine).
@@ -39,7 +39,7 @@ MODEL_DIR=/home/dank/models/huihui-qwen36-35b-a3b-abliterated-fp8
FT_VENV=/home/dank/freetoken/.venv
CUDA_HOME_DIR=/home/dank/freetoken/.venv/lib/python3.12/site-packages/nvidia/cu13
ALIAS=frank-default-small
ALIAS=frank-small
BIND_ADDR=0.0.0.0
LISTEN_PORT=7770
@@ -1,5 +1,5 @@
[Unit]
Description=FreeToken Huihui-Qwen3.6-35B-A3B-abliterated-FP8 qwen3_5_moe hybrid MoE (dankserver, frank-default-small)
Description=FreeToken Huihui-Qwen3.6-35B-A3B-abliterated-FP8 qwen3_5_moe hybrid MoE (dankserver, frank-small)
After=network-online.target
Wants=network-online.target
@@ -7,7 +7,7 @@ Wants=network-online.target
Type=simple
User=dank
Group=dank
Environment="FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-default-small.env"
Environment="FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env"
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh
Restart=on-failure
RestartSec=10
+3 -3
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
# Serve kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX-GGUF as
# frank-default-large via the kingjones30/ROCmFPX llama.cpp runtime on
# frank-large via the kingjones30/ROCmFPX llama.cpp runtime on
# Dankclaw (Strix Halo gfx1151, unified memory).
#
# Runtime: kingjones30/ROCmFPX (fork of charlie12345/ROCmFPX with the
@@ -29,7 +29,7 @@
set -euo pipefail
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}"
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
if [[ -f "$ENV_FILE" ]]; then
set -a
# shellcheck disable=SC1090
@@ -44,7 +44,7 @@ fi
: "${MMPROJ_PATH:=/home/dank/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST/mmproj/mmproj-Qwen3.8-Flash-Next-f16.gguf}"
: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
: "${CHAT_TEMPLATE_KWARGS:={}}"
: "${ALIAS:=frank-default-large}"
: "${ALIAS:=frank-large}"
: "${BIND_ADDR:=0.0.0.0}"
: "${LISTEN_PORT:=7770}"
: "${CTX_SIZE:=401408}"
@@ -1,11 +1,11 @@
#!/usr/bin/env bash
# Serve the FastFlowLM (FLM) sight+sound endpoint — frank-default-sightsound.
# Serve the FastFlowLM (FLM) sight+sound endpoint — frank-npu.
#
# Runs `flm serve <tag>` on the XDNA2 NPU (50 TOPS, Dankclaw). The NPU is a
# single-lane device: generations serialize in FIFO order, so this is a
# background/triage endpoint, not an interactive multi-user chat. The NPU
# draws from the unified RAM pool shared with frank-default-large (the iGPU
# llama.cpp service); see serverAppConfigs-docs/LLM/frank-default-sightsound.md
# draws from the unified RAM pool shared with frank-large (the iGPU
# llama.cpp service); see serverAppConfigs-docs/LLM/frank-npu.md
# for the memory budget and the model-selection tradeoff.
#
# NOTE: FLM requires unlimited memlock. The unit sets LimitMEMLOCK=infinity.
@@ -18,7 +18,7 @@
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FLM_SIGHTSOUND_ENV_FILE:-$SCRIPT_DIR/frank-default-sightsound.env}"
ENV_FILE="${FRANK_NPU_ENV_FILE:-$SCRIPT_DIR/frank-npu.env}"
if [[ -f "$ENV_FILE" ]]; then
set -a
# shellcheck disable=SC1090
+4 -4
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
# Serve batsclamp/Huihui-Qwen3.6-35B-A3B-abliterated-FP8 as
# frank-default-small via FreeToken on Dankserver (RTX 3090, 60 GB RAM).
# frank-small via FreeToken on Dankserver (RTX 3090, 60 GB RAM).
#
# Runtime: FreeToken v0.1.2 (edge-native MoE serving engine).
# - Experts live in host RAM; an LRU cache of expert slots is resident
@@ -25,12 +25,12 @@
# via `ft ctl cache --kv N --moe M` (grows KV by shrinking MoE; shrinking
# KV requires a restart due to the swap-during-rebuild memory cost).
#
# Tunables come from frank-default-small.env unless FRANK_SMALL_ENV_FILE
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE
# overrides that path.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-default-small.env}"
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
if [[ -f "$ENV_FILE" ]]; then
set -a
# shellcheck disable=SC1090
@@ -40,7 +40,7 @@ fi
# --- defaults (used when the env file does not set them) -----------------
: "${MODEL_DIR:=/home/dank/models/huihui-qwen36-35b-a3b-abliterated-fp8}"
: "${ALIAS:=frank-default-small}"
: "${ALIAS:=frank-small}"
: "${BIND_ADDR:=0.0.0.0}"
: "${LISTEN_PORT:=7770}"
: "${FT_VENV:=/home/dank/freetoken/.venv}"
+2 -2
View File
@@ -160,12 +160,12 @@ for path in "${CHANGED[@]}"; do
scripts/mount-entropy.sh)
add_test tests/mount_entropy.bats
;;
scripts/frank-default-large.env|scripts/serve-frank-large.sh)
scripts/frank-large.env|scripts/serve-frank-large.sh)
add_test tests/llm_units.bats
add_test tests/llm_secrets.bats
add_test tests/scripts_syntax.bats
;;
scripts/frank-default-small.env|scripts/serve-frank-small.sh)
scripts/frank-small.env|scripts/serve-frank-small.sh)
add_test tests/llm_units.bats
add_test tests/llm_secrets.bats
add_test tests/scripts_syntax.bats
+1 -1
View File
@@ -73,7 +73,7 @@ load helpers
}
@test "inference restarts are absent from the top-level dispatcher" {
! grep -qE 'systemctl .*restart (frank-default-small|frank-default-large)' "$REPO_ROOT/scripts/deploy.sh"
! grep -qE 'systemctl .*restart (frank-small|frank-large|frank-npu)' "$REPO_ROOT/scripts/deploy.sh"
}
@test "remote runner health-checks an inference endpoint" {
+9 -9
View File
@@ -18,26 +18,26 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
@test "deployment planner maps large-model changes to the large model only" {
run python3 "$PLANNER" --target dankclaw --paths \
scripts/frank-default-large.env scripts/frank-default-large.service
scripts/frank-large.env scripts/frank-large.service
[ "$status" -eq 0 ]
[ "$output" = "frank-default-large" ]
[ "$output" = "frank-large" ]
}
@test "shared chat-template changes map to the selected host's inference service" {
run python3 "$PLANNER" --target dankclaw --paths scripts/chat-template.jinja
[ "$status" -eq 0 ]
[ "$output" = "frank-default-large" ]
[ "$output" = "frank-large" ]
run python3 "$PLANNER" --target dankserver --paths scripts/chat-template.jinja
[ "$status" -eq 0 ]
[ "$output" = "frank-default-small" ]
[ "$output" = "frank-small" ]
}
@test "deployment planner maps small-model changes to the small model only" {
run python3 "$PLANNER" --target dankserver --paths \
scripts/frank-default-small.service
scripts/frank-small.service
[ "$status" -eq 0 ]
[ "$output" = "frank-default-small" ]
[ "$output" = "frank-small" ]
}
@test "documentation-only changes plan no runtime deployment" {
@@ -69,9 +69,9 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
[ "$status" -eq 0 ]
[ "$output" = "hermes" ]
run python3 "$PLANNER" --target dankclaw --paths scripts/serve-frank-default-npu.sh scripts/frank-default-npu.env
run python3 "$PLANNER" --target dankclaw --paths scripts/serve-frank-npu.sh scripts/frank-npu.env
[ "$status" -eq 0 ]
[ "$output" = "frank-default-npu" ]
[ "$output" = "frank-npu" ]
}
@test "homepage is owned by Dankclaw's consolidated proxy" {
@@ -155,7 +155,7 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
grep -q 'deploy_llm_large' "$REMOTE_DEPLOY"
grep -q 'deploy_llm_small' "$REMOTE_DEPLOY"
grep -q 'deploy_service' "$REMOTE_DEPLOY"
! grep -qE 'for .*frank-default-(small|large)' "$REMOTE_DEPLOY"
! grep -qE 'for .*frank-(small|large|npu)-?' "$REMOTE_DEPLOY"
}
@test "remote runner rejects services belonging to the other host" {
+4 -4
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
# Seam: LLM systemd unit integrity.
# Every LLM unit (frank-default-*.service) must point at files that exist in
# Every LLM unit (frank-*.service) must point at files that exist in
# this repo. The homepage-status unit is a different pattern (it installs to
# /usr/local/bin and runs from /srv at deploy time) and is not checked here.
# The units reference absolute paths on the target checkout
@@ -50,7 +50,7 @@ unit_repo_path() {
echo "$bad"
return 1
fi
done < <(find "$REPO_ROOT/scripts" -name 'frank-default-*.service' -type f | sort)
done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
[ "$any" -eq 1 ] || skip "no LLM units yet"
}
@@ -65,7 +65,7 @@ unit_repo_path() {
echo "unit $(basename "$unit") env file $envfile not in repo"
return 1
}
done < <(find "$REPO_ROOT/scripts" -name 'frank-default-*.service' -type f | sort)
done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
}
@test "LLM serve scripts exist for every unit" {
@@ -78,5 +78,5 @@ unit_repo_path() {
echo "unit $svc: ExecStart script $script not in repo"
return 1
}
done < <(find "$REPO_ROOT/scripts" -name 'frank-default-*.service' -type f | sort)
done < <(find "$REPO_ROOT/scripts" -name "frank-*.service" -type f | grep -v AGENTTRASH | sort)
}
+2 -2
View File
@@ -4,7 +4,7 @@
# The old open-webui was a bare container on dankserver (port 3000) pointing at a
# dead LLM backend (:9090). This unit stands one up on dankclaw on a free port
# (3005), on the CPU image (dankclaw is CPU-only), wired to the live
# llama.cpp backend (192.168.0.143:7770, model frank-default-large).
# llama.cpp backend (192.168.0.143:7770, model frank-large).
load helpers
@@ -63,7 +63,7 @@ QDIR="$REPO_ROOT/dankclaw/open-webui/quadlet"
# gateway (10.89.3.1) lives in the rootless netns, NOT the host netns
# where llama-server runs -- it refuses. The host's LAN IP reaches it
# through the rootless NAT (verified from the container, 2026-08-22).
# 2026-08-22: frank-default-large moved from :52625 to :7770.
# 2026-08-22: frank-large moved from :52625 to :7770.
grep -qE '^Environment=OPENAI_API_BASE_URL=http://192\.168\.0\.143:7770' \
"$QDIR/open-webui.container" \
|| { echo "open-webui.container must set OPENAI_API_BASE_URL to the LLM at 192.168.0.143:7770"; return 1; }