inference: frank-large returns to halogen, frank-small takes the 3090
Dankclaw's endpoints are re-split. frank-large is the halogen-flash-server quadlet again -- the 2026-09-13 unit file was never touched by the two-day ik_llama.cpp interlude, so the runtime was still in the repo and its image is still in local storage -- retuned to the shape asked for: HALOGEN_CTX 200192 with a 400,384-position pool and two slots, exactly two 200k conversations. frank-small is new: mainline llama.cpp on the RTX 3090 serving ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF at IQ3_S, using the -mtp build (it carries blk.64.nextn.*, which is what lets --spec-type draft-mtp run on the main model's own weights instead of a second 12 GiB copy) plus the BF16 vision projector, as one 131,072-token slot with f16 KV on port 7771. The host is shared now, and that is what the halogen retune pays for: HALOGEN_MAX_TOK 24576 (arena 20.6 -> 16.4 GiB, ~4-5% prefill) and the vision tower dropped (1.97 GiB) hand ~6 GiB back to the host, and the reserve rises to 20 to keep the fit check honest. Vision moved wholesale to frank-small. Retired to scripts/.AGENTTRASH/: the ik trio (env, service, serve script), the Qwen-Sharp template only that runtime read, and four benchmark harnesses bound to retired runtimes. Because retiring a file makes any test that greps it pass by absence, the seams assert the archive exists AND the live path does not.
This commit is contained in:
Frank
@@ -60,6 +60,7 @@ SERVICE_ORDER = (
|
||||
"lidarr",
|
||||
"readarr",
|
||||
"frank-large",
|
||||
"frank-small",
|
||||
"frank-medium",
|
||||
)
|
||||
|
||||
@@ -90,6 +91,7 @@ DANKCLAW_SERVICES = {
|
||||
"caddy",
|
||||
"homepage",
|
||||
"frank-large",
|
||||
"frank-small",
|
||||
}
|
||||
|
||||
DANKSERVER_SERVICES = {
|
||||
@@ -390,14 +392,12 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
name = p.name
|
||||
if name in {"homepage-status.py", "homepage-status.service", "homepage-status.timer"}:
|
||||
_add(services, "homepage")
|
||||
if name in {"frank-large.env", "frank-large.service", "serve-frank-large.sh"}:
|
||||
# The ik_llama.cpp runtime (2026-09-15 cutover) keeps its unit,
|
||||
# env and serve files here; each one selects frank-large only.
|
||||
_add(services, "frank-large")
|
||||
if name == "chat-template.jinja":
|
||||
# The shared chat template is read by frank-large's serve script
|
||||
# again; the vllm-radiance runtime on Dankserver ships its own.
|
||||
_add(services, "frank-large")
|
||||
if name in {"frank-small.env", "frank-small.service", "serve-frank-small.sh"}:
|
||||
# llama.cpp frank-small (2026-09-16) keeps its unit, env and serve
|
||||
# files here; each one selects frank-small only. The ik_llama.cpp
|
||||
# trio that lived here for two days is in scripts/.AGENTTRASH/,
|
||||
# which is archive material and plans nothing.
|
||||
_add(services, "frank-small")
|
||||
# Deployment/test/benchmark scripts and other repo tooling do not
|
||||
# restart a runtime service.
|
||||
return services
|
||||
|
||||
+91
-37
@@ -90,7 +90,7 @@ service_is_allowed() {
|
||||
dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
||||
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
|
||||
dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
|
||||
dankclaw:frank-large|\
|
||||
dankclaw:frank-large|dankclaw:frank-small|\
|
||||
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
|
||||
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:plex|dankserver:tautulli|\
|
||||
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
|
||||
@@ -965,51 +965,104 @@ deploy_anubis() {
|
||||
wait_container_healthy anubis 60 || return 1
|
||||
}
|
||||
|
||||
# frank-large runs as a ROOT unit again since 2026-09-15: ik_llama.cpp
|
||||
# (qwen4exp support merged upstream) replaced the halogen-flash-server
|
||||
# rootless Quadlet, and the ik server is a native binary — CUDA on the RTX
|
||||
# 3090 for dense/attention/KV, the Strix Halo CPU path for the MoE experts.
|
||||
# There is no container image to pull and no generated unit to avoid:
|
||||
# relink_unit points /etc/systemd/system/frank-large.service at the repo
|
||||
# file, and boot persistence is the unit's [Install] section.
|
||||
# frank-large runs as a rootless Quadlet unit again since 2026-09-16: the
|
||||
# halogen-flash-server runtime is back (it replaced ik_llama.cpp, which had
|
||||
# replaced it), and the RTX 3090 it shared during that interlude now belongs
|
||||
# to frank-small.
|
||||
#
|
||||
# The halogen Quadlet is retired first. Both managers must never know this
|
||||
# unit name at once (they would race for port 7770): stop the user unit,
|
||||
# poll until it is really gone, move the installed .container into the home
|
||||
# trash (never rm), then reload the user manager.
|
||||
LLM_LARGE_CHECKPOINT="${LLM_LARGE_CHECKPOINT:-/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M/Qwen3.8-Flash-Next-Uncensored.i1-Q4_K_M.gguf}"
|
||||
# TWO MANAGERS, ONE UNIT NAME. The ik incarnation ran as a ROOT unit whose
|
||||
# file is a symlink into this checkout; the halogen unit is a USER (quadlet)
|
||||
# unit. Both must never exist at once — they would race for port 7770 — so the
|
||||
# root unit is retired first: stop, disable, move the /etc symlink into the
|
||||
# home trash (never `rm`), reload the root manager. This host's `systemctl
|
||||
# disable` has been observed to delete the unit file itself (2026-08-26
|
||||
# incident), so the move tolerates a file that is already gone.
|
||||
#
|
||||
# FIRST START fetches ~127 GB of weights, and /health cannot answer while it
|
||||
# does. wait_for_llm_large_ready recognises that (checkpoint absent) and waits
|
||||
# for the file instead of burning the health budget on a failure that is not
|
||||
# one.
|
||||
LLM_LARGE_CHECKPOINT="${LLM_LARGE_CHECKPOINT:-/home/dank/models/halogen/qwen38-flash-next-w4b.hgn}"
|
||||
|
||||
retire_quadlet_llm_large() {
|
||||
local trash="$HOME/.AGENTTRASH/20260915-ik-swap/halogen-quadlet"
|
||||
systemctl --user stop frank-large.service 2>/dev/null || true
|
||||
# A stopping quadlet runs podman stop (SIGTERM, then SIGKILL at the
|
||||
# timeout); llama-family engines never preempt an in-flight generation.
|
||||
# Poll until the user unit is really inactive before removing its file.
|
||||
local i
|
||||
for ((i = 0; i < 30; i++)); do
|
||||
systemctl --user is-active --quiet frank-large.service 2>/dev/null || break
|
||||
sleep 2
|
||||
done
|
||||
if [[ -e "$HOME/.config/containers/systemd/frank-large.container" || -L "$HOME/.config/containers/systemd/frank-large.container" ]]; then
|
||||
retire_root_llm_large() {
|
||||
local trash="$HOME/.AGENTTRASH/20260916-halogen-restore/host-units"
|
||||
run_privileged systemctl stop frank-large.service 2>/dev/null || true
|
||||
run_privileged systemctl disable frank-large.service 2>/dev/null || true
|
||||
if [[ -e /etc/systemd/system/frank-large.service || -L /etc/systemd/system/frank-large.service ]]; then
|
||||
mkdir -p "$trash"
|
||||
mv -- "$HOME/.config/containers/systemd/frank-large.container" \
|
||||
"$trash/frank-large.container"
|
||||
echo "retired halogen quadlet -> $trash"
|
||||
run_privileged mv -- /etc/systemd/system/frank-large.service \
|
||||
"$trash/frank-large.service"
|
||||
echo "retired root unit frank-large.service -> $trash"
|
||||
fi
|
||||
systemctl --user daemon-reload
|
||||
run_privileged systemctl daemon-reload
|
||||
}
|
||||
|
||||
deploy_llm_large() {
|
||||
retire_quadlet_llm_large
|
||||
if [[ ! -f "$LLM_LARGE_CHECKPOINT" ]]; then
|
||||
echo "ERROR: checkpoint absent at $LLM_LARGE_CHECKPOINT" >&2
|
||||
echo "frank-large: download the mradermacher i1-Q4_K_M GGUF first; refusing to start a unit that cannot load." >&2
|
||||
retire_root_llm_large
|
||||
# Pull once, then run the localhost/-aliased tag: this host's quadlet path
|
||||
# cannot resolve registry-prefixed refs from local storage at create time
|
||||
# (same workaround as anubis/hermes), and Pull=never keeps boots off the
|
||||
# registry.
|
||||
local image="ghcr.io/peonist-ai/halogen-flash-server:0.5.8"
|
||||
local alias="localhost/peonist-ai/halogen-flash-server:0.5.8"
|
||||
podman pull "$image"
|
||||
podman tag "$image" "$alias"
|
||||
# The weights volume must exist, and the container needs it read-write:
|
||||
# that is where a first start downloads into.
|
||||
mkdir -p /home/dank/models/halogen
|
||||
install_quadlet frank-large dankclaw/frank-large/quadlet frank-large
|
||||
# No enable: generated units cannot be enabled on this host (v1 quadlet
|
||||
# generator); boot persistence comes from [Install] in the unit file.
|
||||
systemctl --user restart frank-large.service
|
||||
wait_for_llm_large_ready
|
||||
wait_for_health frank-large http://127.0.0.1:7770/health 1200
|
||||
}
|
||||
|
||||
# Bootstrap-aware gate: if the checkpoint is absent the container is fetching
|
||||
# it; wait for the file (the fetch resumes, and the unit restarts it on
|
||||
# failure) rather than gating on a /health that cannot answer yet. Reports
|
||||
# success either way, so a first deploy does not look like a failure while the
|
||||
# engine does exactly what it was told to.
|
||||
wait_for_llm_large_ready() {
|
||||
[[ -f "$LLM_LARGE_CHECKPOINT" ]] && return 0
|
||||
echo "frank-large: checkpoint absent at $LLM_LARGE_CHECKPOINT"
|
||||
echo "frank-large: first start — the container is downloading ~127 GB from"
|
||||
echo "frank-large: peonist-ai/halogen-qwen3.8-flash-next (resumes if interrupted)."
|
||||
local i
|
||||
for ((i = 1; i <= 1200; i++)); do
|
||||
if [[ -f "$LLM_LARGE_CHECKPOINT" ]]; then
|
||||
echo "frank-large: weights landed after approximately $((i * 2))s; gating on /health"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "WARN: frank-large download still in progress after 40m — NOT a failure." >&2
|
||||
echo "WARN: the unit keeps fetching; re-run this deploy (or re-check /health) later." >&2
|
||||
return 0
|
||||
}
|
||||
|
||||
# deploy_llm_small — frank-small is mainline llama.cpp on the RTX 3090, as a
|
||||
# ROOT unit: the runtime is a native CUDA binary, so there is no container
|
||||
# image to install and no generated unit to avoid. relink_unit points
|
||||
# /etc/systemd/system/frank-small.service at the repo file and boot persistence
|
||||
# is that unit's [Install] section.
|
||||
#
|
||||
# It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
|
||||
# not shared: frank-large's halogen engine runs on the gfx1151 iGPU.
|
||||
LLM_SMALL_MODEL="${LLM_SMALL_MODEL:-/home/dank/models/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}"
|
||||
|
||||
deploy_llm_small() {
|
||||
if [[ ! -f "$LLM_SMALL_MODEL" ]]; then
|
||||
echo "ERROR: model absent at $LLM_SMALL_MODEL" >&2
|
||||
echo "frank-small: fetch ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF (the -mtp build plus the mmproj) first;" >&2
|
||||
echo "frank-small: refusing to start a unit that cannot load." >&2
|
||||
return 1
|
||||
fi
|
||||
relink_unit frank-large
|
||||
run_privileged systemctl enable frank-large.service
|
||||
run_privileged systemctl restart frank-large.service
|
||||
wait_for_health frank-large http://127.0.0.1:7770/health 1200
|
||||
relink_unit frank-small
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-small.service
|
||||
run_privileged systemctl restart frank-small.service
|
||||
wait_for_health frank-small http://127.0.0.1:7771/health 900
|
||||
}
|
||||
|
||||
# deploy_llm_medium — frank-medium runs as a rootless Quadlet since the
|
||||
@@ -1255,6 +1308,7 @@ deploy_service() {
|
||||
seerr) deploy_seerr ;;
|
||||
marreta) deploy_marreta ;;
|
||||
frank-large) deploy_llm_large ;;
|
||||
frank-small) deploy_llm_small ;;
|
||||
frank-medium) deploy_llm_medium ;;
|
||||
hermes|hermes-dashboard|hermes-webui) deploy_hermes ;;
|
||||
romm|romm-mariadb|romm-redis|mariadb|redis) deploy_romm_stack "$service" ;;
|
||||
|
||||
+1
-1
@@ -140,7 +140,7 @@ valid_service() {
|
||||
dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|dankclaw:loki|dankclaw:alloy|\
|
||||
dankclaw:node-exporter|dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
||||
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|dankclaw:seerr|\
|
||||
dankclaw:frank-large|dankclaw:frank-npu|\
|
||||
dankclaw:frank-large|dankclaw:frank-small|\
|
||||
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
|
||||
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:plex|dankserver:tautulli|\
|
||||
dankserver:qbittorrent|dankserver:radarr|dankserver:sonarr|\
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
# Tunables for frank-small (ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF, the IQ3_S
|
||||
# build with the MTP head, plus the BF16 vision projector) on mainline
|
||||
# llama.cpp, Dankclaw's RTX 3090, since 2026-09-16.
|
||||
#
|
||||
# Runtime: llama.cpp master 37b53fd (2026-09-16), CUDA build for sm_86:
|
||||
# cmake -B build-cuda -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=ON \
|
||||
# -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86
|
||||
# cmake --build build-cuda --target llama-server -j
|
||||
# GGML_CUDA_FA_ALL_QUANTS is deliberately OFF in that build: it is only needed
|
||||
# for QUANTIZED KV with flash attention, and this service runs f16 KV (below).
|
||||
#
|
||||
# Model: ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF, the `-mtp` build of IQ3_S.
|
||||
# GSQ-RCO assigns a separate quantization type to every tensor from a
|
||||
# gradient-based search under a total size budget, rather than applying one
|
||||
# type throughout; IQ3_S at 3.50 bpw is the release's task-lossless operating
|
||||
# point (AIME25 100.00 and LiveCodeBench v6 85.71 match BF16 exactly,
|
||||
# GPQA-Diamond 0.51 behind, at 11.8 GB against 53.8).
|
||||
#
|
||||
# WHY THAT FILE: the `-mtp` build is ~0.35 GB larger because it carries the
|
||||
# model's Multi-Token Prediction head as GGUF block `blk.64.nextn.*`
|
||||
# (qwen35.nextn_predict_layers = 1). llama.cpp loads it from the SAME file --
|
||||
# `--spec-type draft-mtp` sets load_mtp, and the draft context runs on the main
|
||||
# model's weights (common/common.cpp passes shares_model when there is no
|
||||
# --model-draft; speculative.cpp gives it LLAMA_CONTEXT_TYPE_MTP), so there is
|
||||
# no second 12 GB copy in VRAM. MTP is therefore a property of the FILE, not of
|
||||
# the flags: the plain IQ3_S build has no MTP tensors to load.
|
||||
#
|
||||
# Architecture (GGUF arch qwen35): 64 layers laid out as 16 x (3 x (Gated
|
||||
# DeltaNet -> FFN) + 1 x (Gated Attention -> FFN)) -- 48 linear-attention
|
||||
# layers with CONSTANT-SIZE per-sequence state and 16 full-attention layers,
|
||||
# 4 KV heads x 256 head_dim. Only the attention layers take KV, so a token
|
||||
# costs 2 (K,V) x 4 heads x 256 x 2 B = 64 KiB at f16 (34 KiB at q8_0, 18 KiB
|
||||
# at q4_0), and the DeltaNet layers add ~0.16 GiB per sequence in f32 at any
|
||||
# depth.
|
||||
#
|
||||
# SIZING (RTX 3090, 24576 MiB) -- modelled, and the loader's own ledger
|
||||
# ("KV self size", "compute buffer size") outranks it:
|
||||
# weights 11.29 GiB (12,120,016,960 B)
|
||||
# mmproj 0.87 GiB (931,146,528 B, 27 blocks)
|
||||
# MTP draft KV 0.50 GiB (its context is n_ctx long
|
||||
# over one nextn layer: 4 KiB/token)
|
||||
# DeltaNet state 0.16 GiB (one sequence)
|
||||
# CUDA context + prefill/decode bufs ~1.5 GiB
|
||||
# KV at CTX_SIZE 8.00 GiB (131,072 x 64 KiB)
|
||||
# --------
|
||||
# ~22.3 GiB of 24, ~2 GiB of slack
|
||||
# The binding number is the KV: ~9.5 GiB is left after the fixed terms, which
|
||||
# at 64 KiB/token is ~155k tokens across all slots. The deployed shape is one
|
||||
# slot at 131,072, full f16 precision, no KV quantization risk, no rebuild.
|
||||
# Alternatives inside the same budget: 2 x 65536 at f16; 4 x 65536 or
|
||||
# 1 x 262144 at q8_0, both of which need a rebuild with
|
||||
# -DGGML_CUDA_FA_ALL_QUANTS=ON (without it quantized-KV prefill collapses
|
||||
# ~12x on CUDA -- see the CUDA lesson in AGENTS.md).
|
||||
# --parallel N DIVIDES --ctx-size between slots; it does not multiply. One slot
|
||||
# means a second concurrent request queues behind the first.
|
||||
#
|
||||
# Sampling: greedy by default, which is the fleet contract (a request that
|
||||
# sends no temperature decodes greedy; a client that sends one gets its own).
|
||||
# Qwen's own agentic evaluations used temperature 1.0 / top_p 0.95, and the
|
||||
# harness applies exactly that per model in models.yml (extraBody), the same
|
||||
# way it did for the earlier frank-large runtime.
|
||||
#
|
||||
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (HIP reads
|
||||
# those bare vars on the ROCm hosts -- see the 2026-08-13 incident in
|
||||
# AGENTS.md). This box is CUDA, but the rule keeps env files portable.
|
||||
|
||||
# llama.cpp runtime binary: mainline master 37b53fd, CUDA sm_86.
|
||||
LLAMA_BIN=/home/dank/GitPackages/llama.cpp-mainline/build-cuda/bin/llama-server
|
||||
|
||||
# Model package. The -mtp file is required for speculative decoding.
|
||||
MODEL_DIR=/home/dank/models/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp
|
||||
MODEL_FILE=Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf
|
||||
MMPROJ_PATH=/home/dank/models/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp/mmproj-Qwen3.8-27B-BF16.gguf
|
||||
|
||||
# Fleet identity and network. 7771 is this service's own endpoint on Dankclaw;
|
||||
# 7770 stays frank-large's. The alias is the id at /v1/models and what clients
|
||||
# ask for -- the harness entry in models.yml names it.
|
||||
ALIAS=frank-small
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7771
|
||||
|
||||
# One slot at 131,072 tokens with f16 KV (see the sizing note above).
|
||||
CTX_SIZE=131072
|
||||
PARALLEL=1
|
||||
CACHE_TYPE_K=f16
|
||||
CACHE_TYPE_V=f16
|
||||
|
||||
# Batch geometry: llama.cpp's own defaults (2048 logical / 512 physical). The
|
||||
# prefill wall clock at this model size is set by the card, not by ubatch, and
|
||||
# the small ubatch keeps the DeltaNet chunk buffers small. Raise UBATCH_SIZE
|
||||
# only after reading the compute buffer size the loader reports.
|
||||
BATCH_SIZE=2048
|
||||
UBATCH_SIZE=512
|
||||
|
||||
# CPU threads: this service runs entirely on the GPU (NGL=999), so threads
|
||||
# cover tokenization, sampling and the HTTP layer only. Deliberately modest --
|
||||
# this host's CPU is shared with halogen's engine, which is CPU-hungry.
|
||||
NUM_THREADS=8
|
||||
THREAD_BATCH=8
|
||||
|
||||
# Every layer on the 3090, with flash attention.
|
||||
NGL=999
|
||||
FLASH_ATTN=on
|
||||
|
||||
# Speculative decoding from the model's own MTP head (see the model note).
|
||||
# Draft depth stays llama.cpp's default of 3 tokens.
|
||||
SPEC_TYPE=draft-mtp
|
||||
|
||||
# Greedy default (fleet contract); reasoning is extracted into
|
||||
# reasoning_content with an unlimited budget (the model decides).
|
||||
TEMPERATURE=0.0
|
||||
REASONING_FORMAT=deepseek
|
||||
REASONING_BUDGET=-1
|
||||
@@ -0,0 +1,20 @@
|
||||
[Unit]
|
||||
Description=llama.cpp Qwen3.8-27B GSQ-RCO IQ3_S MTP+vision (dankclaw, frank-small)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
# CUDA pins its staging buffers, so the process needs the memlock budget even
|
||||
# though the weights themselves are copied into VRAM. Parity with the halogen
|
||||
# quadlet's Ulimit=memlock=-1:-1 and the retired ik unit.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Executable
+104
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp as frank-small on Dankclaw's
|
||||
# RTX 3090 via mainline llama.cpp.
|
||||
#
|
||||
# Topology: the whole model lives in VRAM (-ngl 999) -- 11.29 GiB of IQ3_S-mtp
|
||||
# weights, 0.87 GiB of BF16 vision projector and ~8 GiB of f16 KV inside the
|
||||
# card's 24 GiB. The host CPU only tokenizes and samples here; this box's CPU
|
||||
# belongs to the halogen engine that serves frank-large.
|
||||
#
|
||||
# MTP: the `-mtp` build carries the NextN block (blk.64.nextn.*), and
|
||||
# `--spec-type draft-mtp` makes llama.cpp load it and run the draft context on
|
||||
# the SAME weights. There is no --model-draft here on purpose: a separate draft
|
||||
# path would load a second 12 GB copy of a 12 GB model. Draft depth is
|
||||
# llama.cpp's default (--spec-draft-n-max 3).
|
||||
#
|
||||
# Vision: --mmproj loads the BF16 projector, so image content parts work. The
|
||||
# fleet's image path moved here from frank-large on 2026-09-16, which is why
|
||||
# halogen no longer sets HALOGEN_VISION_TOWER.
|
||||
#
|
||||
# Sampling: greedy by default (the fleet contract). Clients that send their own
|
||||
# temperature get their own values, and the harness overrides per model from
|
||||
# models.yml. --jinja is pinned even though this build defaults it on: the
|
||||
# model's own template supplies reasoning_effort, preserve_thinking and the
|
||||
# tool-call grammar, and a future build that flips that default must not
|
||||
# silently take tools away.
|
||||
#
|
||||
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides it.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -------------------
|
||||
: "${LLAMA_BIN:=/home/dank/GitPackages/llama.cpp-mainline/build-cuda/bin/llama-server}"
|
||||
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp}"
|
||||
: "${MODEL_FILE:=Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}"
|
||||
: "${MMPROJ_PATH:=$MODEL_DIR/mmproj-Qwen3.8-27B-BF16.gguf}"
|
||||
: "${ALIAS:=frank-small}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7771}"
|
||||
: "${CTX_SIZE:=131072}"
|
||||
: "${PARALLEL:=1}"
|
||||
: "${CACHE_TYPE_K:=f16}"
|
||||
: "${CACHE_TYPE_V:=f16}"
|
||||
: "${BATCH_SIZE:=2048}"
|
||||
: "${UBATCH_SIZE:=512}"
|
||||
: "${NUM_THREADS:=8}"
|
||||
: "${THREAD_BATCH:=8}"
|
||||
: "${NGL:=999}"
|
||||
: "${FLASH_ATTN:=on}"
|
||||
: "${SPEC_TYPE:=draft-mtp}"
|
||||
: "${TEMPERATURE:=0.0}"
|
||||
: "${REASONING_FORMAT:=deepseek}"
|
||||
: "${REASONING_BUDGET:=-1}"
|
||||
|
||||
# Fail loudly rather than half-loading a model into VRAM: a missing file here
|
||||
# is a download that has not finished, and the unit's restart policy would
|
||||
# otherwise turn that into a crash loop that reads like a runtime fault.
|
||||
for path in "$LLAMA_BIN" "$MODEL_DIR/$MODEL_FILE" "$MMPROJ_PATH"; do
|
||||
[[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
|
||||
done
|
||||
|
||||
# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
|
||||
# open after boot; make sure it exists before the server needs it.
|
||||
nvidia-modprobe -u -c=0 2>/dev/null || true
|
||||
|
||||
MMPROJ_ARGS=()
|
||||
if [[ -n "${MMPROJ_PATH:-}" && -f "$MMPROJ_PATH" ]]; then
|
||||
MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH")
|
||||
fi
|
||||
|
||||
SPEC_ARGS=()
|
||||
if [[ -n "${SPEC_TYPE:-}" && "$SPEC_TYPE" != "none" ]]; then
|
||||
SPEC_ARGS=(--spec-type "$SPEC_TYPE")
|
||||
fi
|
||||
|
||||
exec "$LLAMA_BIN" \
|
||||
--model "$MODEL_DIR/$MODEL_FILE" \
|
||||
"${MMPROJ_ARGS[@]}" \
|
||||
--alias "$ALIAS" \
|
||||
--host "$BIND_ADDR" \
|
||||
--port "$LISTEN_PORT" \
|
||||
--gpu-layers "$NGL" \
|
||||
--flash-attn "$FLASH_ATTN" \
|
||||
--ctx-size "$CTX_SIZE" \
|
||||
--parallel "$PARALLEL" \
|
||||
--batch-size "$BATCH_SIZE" \
|
||||
--ubatch-size "$UBATCH_SIZE" \
|
||||
--threads "$NUM_THREADS" \
|
||||
--threads-batch "$THREAD_BATCH" \
|
||||
--cache-type-k "$CACHE_TYPE_K" \
|
||||
--cache-type-v "$CACHE_TYPE_V" \
|
||||
"${SPEC_ARGS[@]}" \
|
||||
--jinja \
|
||||
--metrics \
|
||||
--temp "$TEMPERATURE" \
|
||||
--reasoning-format "$REASONING_FORMAT" \
|
||||
--reasoning-budget "$REASONING_BUDGET"
|
||||
+15
-4
@@ -145,11 +145,12 @@ for path in "${CHANGED[@]}"; do
|
||||
add_test tests/homepage_caddy.bats
|
||||
add_test tests/deployment_scoping.bats
|
||||
;;
|
||||
# frank-large moved to a rootless quadlet (halogen-flash-server) on
|
||||
# 2026-09-11; its unit carries the fleet's GPU and podman posture, so
|
||||
# it gets its own seam rather than the generic fallback below.
|
||||
# frank-large is a rootless quadlet again since 2026-09-16 (the
|
||||
# halogen-flash-server restore). Its unit carries the fleet's GPU and
|
||||
# podman posture, so it gets its own seam rather than the generic
|
||||
# fallback below.
|
||||
dankclaw/frank-large/*)
|
||||
add_test tests/frank_large_quadlet.bats
|
||||
add_test tests/frank_large_unit.bats
|
||||
add_test tests/deployment_scoping.bats
|
||||
add_test tests/docs_completeness.bats
|
||||
;;
|
||||
@@ -187,6 +188,16 @@ for path in "${CHANGED[@]}"; do
|
||||
add_test tests/deployment_scoping.bats
|
||||
add_test tests/docs_completeness.bats
|
||||
;;
|
||||
# frank-small is llama.cpp on the 3090 as a ROOT unit: a native CUDA
|
||||
# binary, so no generated quadlet is involved. Its env, unit and serve
|
||||
# script all live in scripts/, which is why this case has to precede
|
||||
# the generic scripts/*.service and scripts/* fallbacks below.
|
||||
scripts/frank-small.*|scripts/serve-frank-small.sh)
|
||||
add_test tests/frank_small_unit.bats
|
||||
add_test tests/llm_units.bats
|
||||
add_test tests/deployment_scoping.bats
|
||||
add_test tests/scripts_syntax.bats
|
||||
;;
|
||||
scripts/*.service)
|
||||
add_test tests/llm_units.bats
|
||||
add_test tests/scripts_syntax.bats
|
||||
|
||||
Reference in New Issue
Block a user