rename frank-small → frank-secondary (Dankclaw llama.cpp :7771)
Cross-cutting rename of the current llama.cpp endpoint on the RTX 3090
(port 7771) from frank-small to frank-secondary:
- Service trio: scripts/{frank-small.env,frank-small.service,
serve-frank-small.sh,download-frank-small.sh} → frank-secondary.*
- Test: tests/frank_small_unit.bats → tests/frank_secondary_unit.bats
- Runbooks: serverAppConfigs-docs/LLM/frank-small-*.md →
frank-secondary-*.md (9 files); the FreeToken runbook stays
freetoken-frank-small.md (it documents the retired CUDA-era service)
- Identifiers: FRANK_SMALL→FRANK_SECONDARY, llm_small→llm_secondary,
LLM_SMALL_*→LLM_SECONDARY_*, deploy_llm_small→deploy_llm_secondary
- Historical references to the OLD frank-small (FreeToken/CUDA on the
3090, and the Dankserver service that became frank-medium) are
preserved verbatim — they describe what the service was called
at the time.
This commit is contained in:
Frank committed
1 parent
c71a3066c0
commit
87721ab0fe
87 files changed
+460
-460
No files matched your search
@@ -62,7 +62,7 @@ SERVICE_ORDER = (
|
||||
"lidarr",
|
||||
"readarr",
|
||||
"frank-large",
|
||||
"frank-small",
|
||||
"frank-secondary",
|
||||
"frank-primary",
|
||||
)
|
||||
|
||||
@@ -94,7 +94,7 @@ DANKCLAW_SERVICES = {
|
||||
"caddy",
|
||||
"homepage",
|
||||
"frank-large",
|
||||
"frank-small",
|
||||
"frank-secondary",
|
||||
}
|
||||
|
||||
DANKSERVER_SERVICES = {
|
||||
@@ -400,12 +400,12 @@ def services_for_path(path: str, target: str) -> set[str]:
|
||||
# change to it redeploying alertmanager is the only path that
|
||||
# reaches it on the host.
|
||||
_add(services, "alertmanager")
|
||||
if name in {"frank-small.env", "frank-small.service", "serve-frank-small.sh"}:
|
||||
# llama.cpp frank-small (2026-09-16) keeps its unit, env and serve
|
||||
# files here; each one selects frank-small only. The ik_llama.cpp
|
||||
if name in {"frank-secondary.env", "frank-secondary.service", "serve-frank-secondary.sh"}:
|
||||
# llama.cpp frank-secondary (2026-09-16) keeps its unit, env and serve
|
||||
# files here; each one selects frank-secondary only. The ik_llama.cpp
|
||||
# trio that lived here for two days is in scripts/.AGENTTRASH/,
|
||||
# which is archive material and plans nothing.
|
||||
_add(services, "frank-small")
|
||||
_add(services, "frank-secondary")
|
||||
if name in {"frank-large.env", "frank-large.service", "serve-frank-large.sh"}:
|
||||
# CIRU v4.4.1 frank-large (2026-09-19) keeps its unit, env and
|
||||
# serve files here the same way: a root unit on a host-built
|
||||
|
||||
+25
-25
@@ -90,7 +90,7 @@ service_is_allowed() {
|
||||
dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
||||
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
|
||||
dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
|
||||
dankclaw:frank-large|dankclaw:frank-small|\
|
||||
dankclaw:frank-large|dankclaw:frank-secondary|\
|
||||
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
|
||||
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
|
||||
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
|
||||
@@ -1202,7 +1202,7 @@ deploy_llm_large() {
|
||||
# `podman run --name frank-large-halogen` / `podman wait` but NOT in
|
||||
# this deployer's own command line (`bash deploy-remote.sh
|
||||
# frank-large`) — so it cannot shoot itself in the foot (the fleet
|
||||
# pkill law), and it can never match frank-small's llama-server.
|
||||
# pkill law), and it can never match frank-secondary's llama-server.
|
||||
run_privileged systemctl stop frank-large.service 2>/dev/null || true
|
||||
pkill -f 'frank-large-halogen' 2>/dev/null || true
|
||||
podman rm -f frank-large frank-large-halogen >/dev/null 2>&1 || true
|
||||
@@ -1216,57 +1216,57 @@ deploy_llm_large() {
|
||||
wait_for_health frank-large http://127.0.0.1:7770/health 1200
|
||||
}
|
||||
|
||||
# deploy_llm_small — frank-small is llama.cpp (master build, CUDA sm_86) on
|
||||
# deploy_llm_secondary — frank-secondary is llama.cpp (master build, CUDA sm_86) on
|
||||
# the RTX 3090 as a ROOT unit: the runtime is a native binary tree under
|
||||
# ~/GitPackages/llama-master/build-cuda, so there is no container image and
|
||||
# no generated unit to avoid. relink_unit points
|
||||
# /etc/systemd/system/frank-small.service at the repo file and boot
|
||||
# /etc/systemd/system/frank-secondary.service at the repo file and boot
|
||||
# persistence is that unit's [Install] section.
|
||||
#
|
||||
# It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
|
||||
# not shared: frank-large's engine runs on the gfx1151 iGPU.
|
||||
LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/Swift15-UncIQ3XXS/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf}"
|
||||
LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
||||
LLM_SECONDARY_MODEL_FILE="${LLM_SECONDARY_MODEL_FILE:-/home/dank/models/Swift15-UncIQ3XXS/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf}"
|
||||
LLM_SECONDARY_RUNTIME="${LLM_SECONDARY_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
||||
# The projector IS the vision capability here: the GGUF carries no vision
|
||||
# tensors, and the publisher ships mmproj-BF16.gguf in the same repo, so a
|
||||
# deploy that starts without this file silently ships a text-only
|
||||
# endpoint. Checked unconditionally, like frank-large's.
|
||||
LLM_SMALL_MMPROJ="${LLM_SMALL_MMPROJ:-/home/dank/models/Swift15-UncIQ3XXS/mmproj-BF16.gguf}"
|
||||
LLM_SECONDARY_MMPROJ="${LLM_SECONDARY_MMPROJ:-/home/dank/models/Swift15-UncIQ3XXS/mmproj-BF16.gguf}"
|
||||
|
||||
deploy_llm_small() {
|
||||
deploy_llm_secondary() {
|
||||
# Refuse rather than restart-loop: a unit whose runtime or payload is
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the log to say why.
|
||||
if [[ ! -x "$LLM_SMALL_RUNTIME" ]]; then
|
||||
echo "ERROR: llama.cpp runtime absent at $LLM_SMALL_RUNTIME" >&2
|
||||
echo "frank-small: build it (cmake -B build-cuda -DGGML_CUDA=ON" >&2
|
||||
echo "frank-small: -DCMAKE_CUDA_ARCHITECTURES=86 -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/gcc-13" >&2
|
||||
echo "frank-small: && cmake --build build-cuda --target llama-server) first." >&2
|
||||
if [[ ! -x "$LLM_SECONDARY_RUNTIME" ]]; then
|
||||
echo "ERROR: llama.cpp runtime absent at $LLM_SECONDARY_RUNTIME" >&2
|
||||
echo "frank-secondary: build it (cmake -B build-cuda -DGGML_CUDA=ON" >&2
|
||||
echo "frank-secondary: -DCMAKE_CUDA_ARCHITECTURES=86 -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/gcc-13" >&2
|
||||
echo "frank-secondary: && cmake --build build-cuda --target llama-server) first." >&2
|
||||
return 1
|
||||
fi
|
||||
for f in "$LLM_SMALL_MODEL_FILE" "$LLM_SMALL_MMPROJ"; do
|
||||
for f in "$LLM_SECONDARY_MODEL_FILE" "$LLM_SECONDARY_MMPROJ"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "ERROR: frank-small payload file absent at $f" >&2
|
||||
echo "frank-small: run scripts/download-frank-small.sh first;" >&2
|
||||
echo "frank-small: refusing to start a unit that cannot load." >&2
|
||||
echo "ERROR: frank-secondary payload file absent at $f" >&2
|
||||
echo "frank-secondary: run scripts/download-frank-secondary.sh first;" >&2
|
||||
echo "frank-secondary: refusing to start a unit that cannot load." >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
relink_unit frank-small
|
||||
relink_unit frank-secondary
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-small.service
|
||||
run_privileged systemctl restart frank-small.service
|
||||
wait_for_health frank-small http://127.0.0.1:7771/health 900 || return 1
|
||||
run_privileged systemctl enable frank-secondary.service
|
||||
run_privileged systemctl restart frank-secondary.service
|
||||
wait_for_health frank-secondary http://127.0.0.1:7771/health 900 || return 1
|
||||
# False-green hardening (frank-large's lesson): the port answering
|
||||
# proves nothing about WHO answers — the unit must be active with a
|
||||
# live MainPID, or a stray SGLang holdover is masquerading as deployed.
|
||||
local main_pid
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-small.service)"
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-secondary.service)"
|
||||
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
|
||||
echo "FAIL: :7771 answered but frank-small.service has no MainPID — a stray process holds the port" >&2
|
||||
echo "FAIL: :7771 answered but frank-secondary.service has no MainPID — a stray process holds the port" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "frank-small: health answered by unit MainPID $main_pid"
|
||||
echo "frank-secondary: health answered by unit MainPID $main_pid"
|
||||
}
|
||||
|
||||
# deploy_llm_primary — frank-primary is the radiance engine (StillDeadcode,
|
||||
@@ -1567,7 +1567,7 @@ deploy_service() {
|
||||
seerr) deploy_seerr ;;
|
||||
marreta) deploy_marreta ;;
|
||||
frank-large) deploy_llm_large ;;
|
||||
frank-small) deploy_llm_small ;;
|
||||
frank-secondary) deploy_llm_secondary ;;
|
||||
frank-primary) deploy_llm_primary ;;
|
||||
hermes|hermes-dashboard|hermes-webui) deploy_hermes ;;
|
||||
romm|romm-mariadb|romm-redis|mariadb|redis) deploy_romm_stack "$service" ;;
|
||||
|
||||
+1
-1
@@ -141,7 +141,7 @@ valid_service() {
|
||||
dankclaw:alertmanager|\
|
||||
dankclaw:node-exporter|dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
||||
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|dankclaw:seerr|\
|
||||
dankclaw:frank-large|dankclaw:frank-small|\
|
||||
dankclaw:frank-large|dankclaw:frank-secondary|\
|
||||
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
|
||||
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
|
||||
dankserver:qbittorrent|dankserver:radarr|dankserver:sonarr|\
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the frank-small payload for llama.cpp on the RTX 3090:
|
||||
# Download the frank-secondary payload for llama.cpp on the RTX 3090:
|
||||
# ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF:RCO-IQ3_XXS
|
||||
# **with the MTP head in the file** (the -mtp variant), plus the
|
||||
# PUBLISHER's OWN vision projector (mmproj-BF16.gguf, same repo). Public
|
||||
@@ -24,7 +24,7 @@
|
||||
# re-fetch recipes are in git history (predecessors of this script).
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-small.sh [DEST_DIR] [--verify-only]
|
||||
# scripts/download-frank-secondary.sh [DEST_DIR] [--verify-only]
|
||||
set -euo pipefail
|
||||
|
||||
DEST=""
|
||||
@@ -42,7 +42,7 @@
|
||||
# weights ~62 GiB resident + KV 10.7 + arena (HALOGEN_MAX_TOK=8192 =>
|
||||
# ~8 GiB per FLAGS.md arena formula 3.8+8.4*(MT/16384)) + vision ~2
|
||||
# = ~82.7 GiB engine-side; HALOGEN_HOST_RESERVE_GIB=20 keeps the host
|
||||
# honest; baseline co-tenancy (frank-small, ComfyUI, Caddy/observ)
|
||||
# honest; baseline co-tenancy (frank-secondary, ComfyUI, Caddy/observ)
|
||||
# ~10-15. ~18 GiB margin. The boot ledger is the referee — read it
|
||||
# after deploy and re-size if the engine prints a clamp.
|
||||
# HALOGEN_SPEC_ADAPT=0 is MANDATORY (card): the default adaptive policy
|
||||
|
||||
@@ -23,7 +23,7 @@ TimeoutStopSec=120
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
# The engine pins ~62 GiB of v2 weights and keeps a multi-GiB working set
|
||||
# on a box that also hosts frank-small, ComfyUI and the Caddy/observability
|
||||
# on a box that also hosts frank-secondary, ComfyUI and the Caddy/observability
|
||||
# stack; make it a late OOM candidate, not the first.
|
||||
OOMScoreAdjust=-500
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# frank-small — llama.cpp (master, CUDA sm_86) serving
|
||||
# frank-secondary — llama.cpp (master, CUDA sm_86) serving
|
||||
# ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF:RCO-IQ3_XXS
|
||||
# **with the MTP head in the file** on the RTX 3090, :7771.
|
||||
#
|
||||
@@ -45,15 +45,15 @@ MODEL_FILE=/home/dank/models/Swift15-UncIQ3XXS/Swift-1.5-Qwen3.8-27B-Uncensored-
|
||||
|
||||
# Vision: the publisher SHIPS the projector — mmproj-BF16.gguf
|
||||
# (931,145,888 B) from the same repo, fetched by
|
||||
# scripts/download-frank-small.sh. No substitution needed for this
|
||||
# scripts/download-frank-secondary.sh. No substitution needed for this
|
||||
# payload (the unsloth base-tower proof of 2026-10-05 remains the
|
||||
# fallback argument for the IQ3_S rollback payload).
|
||||
# Empty string = text-only on purpose.
|
||||
MMPROJ_FILE=/home/dank/models/Swift15-UncIQ3XXS/mmproj-BF16.gguf
|
||||
|
||||
# Fleet endpoint: same port and alias as every frank-small before it, so the
|
||||
# Fleet endpoint: same port and alias as every frank-secondary before it, so the
|
||||
# harness models.yml id and the homepage probe do not move.
|
||||
ALIAS=frank-small
|
||||
ALIAS=frank-secondary
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7771
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[Unit]
|
||||
Description=llama.cpp Swift-1.5-Qwen3.8-27B Uncensored RCO IQ3_XXS GGUF (qwen35 hybrid, 2x131072 slots, q8_0 K+V, draft-mtp depth 3, mmproj vision) on the RTX 3090 (dankclaw, frank-small)
|
||||
Description=llama.cpp Swift-1.5-Qwen3.8-27B Uncensored RCO IQ3_XXS GGUF (qwen35 hybrid, 2x131072 slots, q8_0 K+V, draft-mtp depth 3, mmproj vision) on the RTX 3090 (dankclaw, frank-secondary)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
@@ -10,8 +10,8 @@ Group=dank
|
||||
# CUDA pins its staging buffers, so the process needs the memlock budget even
|
||||
# though the weights themselves are copied into VRAM.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh
|
||||
Environment="FRANK_SECONDARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-secondary.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-secondary.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF
|
||||
# (RCO-IQ3_XXS, **MTP head in the file**) as frank-small on Dankclaw's
|
||||
# (RCO-IQ3_XXS, **MTP head in the file**) as frank-secondary on Dankclaw's
|
||||
# RTX 3090 via llama.cpp (master build, CUDA sm_86).
|
||||
#
|
||||
# 2026-10-06 (Frank): TWO 131,072 conversations (262144 / 2) at the 128k
|
||||
@@ -24,11 +24,11 @@
|
||||
# Sampling: llama.cpp takes it per request (the fleet harness sends its
|
||||
# recipe from models.yml); greedy is the harness default.
|
||||
#
|
||||
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides.
|
||||
# Tunables come from frank-secondary.env unless FRANK_SECONDARY_ENV_FILE overrides.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
|
||||
ENV_FILE="${FRANK_SECONDARY_ENV_FILE:-$SCRIPT_DIR/frank-secondary.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
@@ -39,13 +39,13 @@ fi
|
||||
# --- defaults (used when the env file does not set them) -------------------
|
||||
: "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
||||
: "${MODEL_FILE:=/home/dank/models/Swift15-UncIQ3XXS/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf}"
|
||||
: "${ALIAS:=frank-small}"
|
||||
: "${ALIAS:=frank-secondary}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7771}"
|
||||
# ctx-size is the POOL llama.cpp divides across slots: 262144 / 2 gives
|
||||
# TWO 131,072 conversations — Frank's 128k cap each. 2026-10-06 (Frank):
|
||||
# drop MTP, drop ubatch to 512, take two slots; the fit is tight by his
|
||||
# explicit order (frank-small.env carries the arithmetic and the floor
|
||||
# explicit order (frank-secondary.env carries the arithmetic and the floor
|
||||
# violation).
|
||||
: "${CTX_SIZE:=262144}"
|
||||
: "${PARALLEL:=2}"
|
||||
@@ -76,11 +76,11 @@ fi
|
||||
# has not finished, and Restart=always would turn it into a crash loop that
|
||||
# reads like a runtime fault.
|
||||
for path in "$LLAMA_BIN" "$MODEL_FILE"; do
|
||||
[[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
|
||||
[[ -e "$path" ]] || { echo "frank-secondary: missing $path" >&2; exit 1; }
|
||||
done
|
||||
EXTRA_ARGS=()
|
||||
if [[ -n "$MMPROJ_FILE" ]]; then
|
||||
[[ -e "$MMPROJ_FILE" ]] || { echo "frank-small: missing $MMPROJ_FILE" >&2; exit 1; }
|
||||
[[ -e "$MMPROJ_FILE" ]] || { echo "frank-secondary: missing $MMPROJ_FILE" >&2; exit 1; }
|
||||
EXTRA_ARGS+=(--mmproj "$MMPROJ_FILE")
|
||||
fi
|
||||
|
||||
@@ -97,7 +97,7 @@ nvidia-modprobe -u -c=0 2>/dev/null || true
|
||||
|
||||
# Flash attention stays pinned on: it is REQUIRED for quantized V (q8_0),
|
||||
# and the build carries GGML_CUDA_FA_ALL_QUANTS=ON (since 2026-09-17), so
|
||||
# the q8_0 KV contract in frank-small.env is a real path — without this
|
||||
# the q8_0 KV contract in frank-secondary.env is a real path — without this
|
||||
# flag or that build option the engine silently falls back and the
|
||||
# contract becomes a lie.
|
||||
ARGS=(
|
||||
@@ -204,12 +204,12 @@ for path in "${CHANGED[@]}"; do
|
||||
scripts/qwen-sharp-v22.4.0.jinja)
|
||||
add_test tests/frank_large_unit.bats
|
||||
;;
|
||||
# frank-small is llama.cpp on the 3090 as a ROOT unit: a native CUDA
|
||||
# frank-secondary is llama.cpp on the 3090 as a ROOT unit: a native CUDA
|
||||
# binary, so no generated quadlet is involved. Its env, unit and serve
|
||||
# script all live in scripts/, which is why this case has to precede
|
||||
# the generic scripts/*.service and scripts/* fallbacks below.
|
||||
scripts/frank-small.*|scripts/serve-frank-small.sh)
|
||||
add_test tests/frank_small_unit.bats
|
||||
scripts/frank-secondary.*|scripts/serve-frank-secondary.sh)
|
||||
add_test tests/frank_secondary_unit.bats
|
||||
add_test tests/llm_units.bats
|
||||
add_test tests/deployment_scoping.bats
|
||||
add_test tests/scripts_syntax.bats
|
||||
|
||||
Reference in new issue
Block a user