Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Frank
|
1f85e57863 | ||
|
Frank
|
6bddf0a72a | ||
|
Frank
|
469bfaf979 | ||
|
Frank
|
726b9a35ec | ||
|
Frank
|
5ed9bf0410 | ||
|
Frank
|
c530e16880 | ||
|
Frank
|
9d00f36d2b | ||
|
Frank
|
9227e1a8b9 | ||
|
Frank
|
659fba6a5d | ||
|
Frank
|
9c1f777423 | ||
|
Frank
|
4edcada2a8 |
@@ -1,71 +0,0 @@
|
||||
# Tunables for frank-default-small (Tiel-Coder-35B-A3B UD-Q6_K_XL,
|
||||
# qwen35moe hybrid MoE, Dankserver, RTX 3090).
|
||||
#
|
||||
# Runtime: upstream llama.cpp (latest master, CUDA build with
|
||||
# -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86) on RTX 3090
|
||||
# 24 GB; Ryzen 7 7700X; 62 GB RAM. Replaced the ik_llama.cpp Ornith
|
||||
# CRACK Q8_0 deployment on 2026-09-01.
|
||||
#
|
||||
# Model: peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF, UD-Q6_K_XL quant
|
||||
# (30 GB). Architecture: qwen35moe — 40 layers, full_attention_interval=4
|
||||
# (10 attention + 30 SSM/Mamba), 256 experts / 8 used, 2 KV heads × 256
|
||||
# dim, embedding 2048, rope dimension_count 64, rope freq_base 10 000 000.
|
||||
# Text-only (no mmproj). Built-in chat template in the GGUF.
|
||||
#
|
||||
# Expert offload: --cpu-moe keeps all 256 expert weights in CPU RAM.
|
||||
# Non-expert weights (~1.6 GB) fit entirely on the 24 GB card.
|
||||
#
|
||||
# KV cache: Q8_0 for both K and V. With CTX_SIZE=200700 (66 900 per slot
|
||||
# × 3 parallel) and 10 attention layers, the KV cache is ~2 GB — the
|
||||
# requested 2 GB VRAM buffer.
|
||||
#
|
||||
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The
|
||||
# ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
|
||||
# incident). This box is NVIDIA/CUDA, but the rule keeps the env portable.
|
||||
|
||||
# Model file (UD-Q6_K_XL, 30 GB, text-only MoE hybrid).
|
||||
MODEL_DIR=/home/dank/models/tiel-coder-35b
|
||||
MODEL_FILE=Tiel-Coder-35B-A3B-UD-Q6_K_XL.gguf
|
||||
|
||||
# Upstream llama.cpp llama-server (CUDA build, latest master).
|
||||
# Built with -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86
|
||||
# using the staged CUDA 13.2 toolkit at /home/dank/cuda13-stage.
|
||||
LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server
|
||||
|
||||
ALIAS=frank-default-small
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7770
|
||||
|
||||
# Total context budget across all parallel slots. 200700 / 3 = 66 900
|
||||
# per slot. The model's context_length is 262 144, so 66 900 is well
|
||||
# within range.
|
||||
CTX_SIZE=200700
|
||||
PARALLEL=3
|
||||
|
||||
# Expert offload: all MoE expert weights pinned to CPU RAM.
|
||||
CPU_MOE=1
|
||||
|
||||
# All non-expert layers on GPU (auto-fit will determine the exact count;
|
||||
# with --cpu-moe the non-expert footprint is ~1.6 GB, leaving ample room).
|
||||
N_GPU_LAYERS=999
|
||||
|
||||
# KV cache: Q8_0 for both K and V. ~2 GB total for 200700 ctx × 3 slots
|
||||
# × 10 attention layers × (2 KV heads × 256 dim × 1 byte) × 2 (K+V).
|
||||
CACHE_TYPE_K=q8_0
|
||||
CACHE_TYPE_V=q8_0
|
||||
|
||||
VERBOSITY=2
|
||||
|
||||
# Prefill batch sizing.
|
||||
BATCH_SIZE=4096
|
||||
UBATCH_SIZE=1024
|
||||
|
||||
# Sampling.
|
||||
TEMPERATURE=1.0
|
||||
TOP_K=20
|
||||
TOP_P=0.95
|
||||
MIN_P=0.0
|
||||
|
||||
# Compute threads lowered so other processes keep a core.
|
||||
THREADS=7
|
||||
THREADS_HTTP=8
|
||||
@@ -1,16 +0,0 @@
|
||||
[Unit]
|
||||
Description=llama.cpp Tiel-Coder-35B-A3B Q6_K_XL qwen35moe hybrid MoE (dankserver, frank-default-small)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
Environment="FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-default-small.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,126 +0,0 @@
|
||||
# Tunables for frank-large (kingjones777/Qwen3.8-Flash-Next-ROCmFP4-
|
||||
# STRIX-GGUF on the kingjones30/ROCmFPX llama.cpp runtime, Dankclaw,
|
||||
# Strix Halo gfx1151, unified memory).
|
||||
#
|
||||
# Runtime: kingjones30/ROCmFPX (fork of charlie12345/ROCmFPX with the
|
||||
# qwen4exp-on-rocmfpx-d3ca537 patch applied — the model card's pinned
|
||||
# runtime: ROCmFP4 tensor types AND the qwen4exp architecture in one
|
||||
# tree; stock llama.cpp and charlie's fork alone cannot load this GGUF).
|
||||
# Built per the card recipe:
|
||||
# cmake -B build -DGGML_HIP=ON -DGPU_TARGETS=gfx1151 -DGGML_NATIVE=ON -DCMAKE_BUILD_TYPE=Release
|
||||
# at /home/dank/GitPackages/kingjones-ROCmFPX/build.
|
||||
# Replaced the agentionai ROCmFP4-FAST imatrix deployment (charlie's
|
||||
# build-qwen38 runtime) on 2026-08-30.
|
||||
#
|
||||
# Model: Qwen3.8-Flash-Next base (qwen4exp MoE: 512 experts / 10 used,
|
||||
# 48 layers = 12 attention + 36 SSM, QSA indexer, hyper-connections, PLE
|
||||
# at layer 1), Q4_0_ROCMFP4_STRIX (file_type 105), 113.47 GiB single
|
||||
# file. STRIX is the top quality tier: PLE table at Q8_0 (50.66 GiB),
|
||||
# token embeddings + lm head at Q6_K. The PLE table NEVER enters GPU
|
||||
# memory — it streams from the file through the OS page cache. Two hard
|
||||
# card constraints:
|
||||
# 1. mmap MUST stay enabled (default) — --no-mmap forces the 50.66 GiB
|
||||
# table into anonymous memory and the cgroup killer takes the
|
||||
# process with nothing in the server log (only dmesg).
|
||||
# 2. Do NOT pin the PLE tensor to CPU with -ot — measured gen fell
|
||||
# ~23 -> 13.4 tok/s; the kernel streams it better.
|
||||
# NOTE: the card's own build_attn_qsa still calls ggml_fill on the
|
||||
# F16-cast kq mask (the path that aborted our charlie build with
|
||||
# --flash-attn on before scripts/patches/qwen4exp-qsa-f32-mask.patch).
|
||||
# If a clean kingjones-tree build aborts graph build with
|
||||
# GGML_ASSERT(a-type == GGML_TYPE_F32), port that fix (F32 base mask +
|
||||
# cast at the attention call) into this tree and rebuild.
|
||||
#
|
||||
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
|
||||
# incident). llama.cpp reads --host/--port/--temp as CLI flags, never as
|
||||
# env vars.
|
||||
|
||||
# kingjones30/ROCmFPX runtime binary (card recipe build).
|
||||
LLAMA_BIN=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server
|
||||
|
||||
# Model package from
|
||||
# https://huggingface.co/kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX-GGUF
|
||||
# Layout: single unsharded GGUF, text-only (the repo ships no mmproj;
|
||||
# the vision tower below is borrowed from the agentionai FAST package —
|
||||
# same base model, f16 — kept only if it loads clean, drop
|
||||
# MMPROJ_PATH if it does not).
|
||||
MODEL_DIR=/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX
|
||||
MODEL_FILE=Qwen3.8-Flash-Next-Q4_0-ROCmFP4-STRIX.gguf
|
||||
MMPROJ_PATH=/home/dank/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST/mmproj/mmproj-Qwen3.8-Flash-Next-f16.gguf
|
||||
|
||||
# Qwen-Sharp chat template (tracked in the repo, staged on the host). The
|
||||
# Sharp template drives thinking on by default (medium effort); the server
|
||||
# emits reasoning in DeepSeek format so clients receive reasoning_content
|
||||
# separately instead of literal thinking tags in content.
|
||||
CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
|
||||
CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true,"preserve_reasoning":true}'
|
||||
|
||||
# Alias + network.
|
||||
ALIAS=frank-large
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7770
|
||||
|
||||
# Serving: 401408-token context, TWO slots (parallel 2), batch 4096 /
|
||||
# ubatch 512, 16+16 threads, f16 KV, 16 context checkpoints, 8 GiB
|
||||
# prompt cache. --fit off: auto-fit segfaults on this model's
|
||||
# hyper-connection/indexer config; -ngl 999 puts every layer on the
|
||||
# SVM-backed GPU pool. ubatch 512 matches the card's re-measurement
|
||||
# (-ub 1024/2048/4096 showed no effect; -t 32 vs 16 is noise).
|
||||
#
|
||||
# CTX_SIZE=401408 / PARALLEL=2: carried over from the FAST deployment
|
||||
# where it loaded clean (~74 s, health 200). The card measures the full
|
||||
# 262144 window at ~71.7 GiB GTT at load on the same 128 GB board; QSA
|
||||
# caps KV growth so the extra window is cheap. KV footprint scales
|
||||
# with the 12 attention layers only. Verify load on first STRIX boot.
|
||||
CTX_SIZE=401408
|
||||
PARALLEL=2
|
||||
|
||||
BATCH_SIZE=4096
|
||||
UBATCH_SIZE=512
|
||||
NUM_THREADS=16
|
||||
THREAD_BATCH=16
|
||||
CACHE_TYPE_K=f16
|
||||
CACHE_TYPE_V=f16
|
||||
CTX_CHECKPOINTS=16
|
||||
CACHE_RAM=8192
|
||||
SLOT_SAVE_PATH=/home/dank/serverAppConfigs/scripts/slot-state
|
||||
SPLIT_MODE=none
|
||||
FIT=off
|
||||
NGL=999
|
||||
FLASH_ATTN=on
|
||||
|
||||
# Sampling (card instruct table, non-thinking).
|
||||
TEMPERATURE=0.7
|
||||
TOP_K=20
|
||||
TOP_P=0.8
|
||||
MIN_P=0.0
|
||||
PRESENCE_PENALTY=1.5
|
||||
|
||||
# Reasoning on (Sharp template drives it; preserve_thinking + preserve_reasoning
|
||||
# keep the reasoning trail in the conversation). DeepSeek format projects the
|
||||
# reasoning into the API's reasoning_content field instead of literal thinking
|
||||
# tags.
|
||||
REASONING=on
|
||||
REASONING_FORMAT=deepseek
|
||||
|
||||
# Strix Halo unified memory + native gfx1151.
|
||||
HSA_OVERRIDE_GFX_VERSION=11.5.1
|
||||
|
||||
VERBOSITY=4
|
||||
|
||||
|
||||
# Speculative decoding: disabled. The card's Usage block ships no spec
|
||||
# flags. The Flash-Next MTP heads do not load on this loader (requires
|
||||
# per-layer blk.N.hc_attn_norm; every head carries only blk.48 —
|
||||
# verified 3x on build-qwen38). DRAFT_MODEL_* point at the retired
|
||||
# agentionai FAST head and are unused while SPEC_TYPE=none.
|
||||
DRAFT_MODEL_DIR=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST
|
||||
DRAFT_MODEL_FILE=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf
|
||||
SPEC_TYPE=none
|
||||
SPEC_DRAFT_N_MAX=3
|
||||
SPEC_DRAFT_N_MIN=0
|
||||
SPEC_DRAFT_P_MIN=0.75
|
||||
SPEC_DRAFT_NGL=999
|
||||
SPEC_DRAFT_DEVICE=ROCm0
|
||||
SPEC_DRAFT_TYPE_K=f16
|
||||
SPEC_DRAFT_TYPE_V=f16
|
||||
@@ -1,16 +0,0 @@
|
||||
[Unit]
|
||||
Description=ROCmFPX (kingjones30 fork) llama.cpp Qwen3.8-Flash-Next ROCmFP4 STRIX 177B MoE (dankclaw, frank-large)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,23 +0,0 @@
|
||||
# Tunables for frank-npu (FastFlowLM NPU endpoint, Dankclaw)
|
||||
# Runtime: FLM 1.0.1 (/usr/bin/flm) on the XDNA2 NPU (50 TOPS). Boot tag is
|
||||
# nuextract3:4b (NuExtract3-4B-NPU2, Qwen3.5-4B skeleton, structured-extraction
|
||||
# model); serve routes every request by the exact FLM registry tag in the
|
||||
# request's model field, so any registered tag (gemma4-it:e4b, qwythos:9b, ...)
|
||||
# is reachable on this port regardless of the boot tag. The NPU is a
|
||||
# single-lane device: generations serialize FIFO, and the first request after a
|
||||
# model swap pays the ~6-11 s load cost. The NPU draws from the unified RAM
|
||||
# pool shared with frank-large (iGPU).
|
||||
FLM_MODEL=nuextract3:4b
|
||||
FLM_HOST=0.0.0.0
|
||||
FLM_PORT=7771
|
||||
# turbo = max NPU power mode (xrt-smi configure --pmode turbo at serve start).
|
||||
# No other tenant uses the NPU on this host, so the top mode costs nothing to
|
||||
# neighbours; drop to `performance` if iGPU power-budget contention ever shows.
|
||||
FLM_PMODE=turbo
|
||||
# FLM_ASR=1 additionally loads Whisper-V3-Turbo-NPU2 (+0.62 GB) and exposes
|
||||
# /v1/audio/transcriptions. NuExtract3 answers text-only; audio stays off.
|
||||
FLM_ASR=0
|
||||
# -1 = no --ctx-len flag: every pack loads with its own trained context
|
||||
# (nuextract3:4b -> 32768, gemma4-it:e4b -> 131072). A global override would
|
||||
# over-allocate KV for the small packs.
|
||||
FLM_CTX_LEN=-1
|
||||
@@ -1,17 +0,0 @@
|
||||
[Unit]
|
||||
Description=FastFlowLM NPU sight+sound inference (frank-npu) on Dankclaw
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
Environment="FRANK_NPU_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-npu.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-npu.sh
|
||||
LimitMEMLOCK=infinity
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,180 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX-GGUF as
|
||||
# frank-large via the kingjones30/ROCmFPX llama.cpp runtime on
|
||||
# Dankclaw (Strix Halo gfx1151, unified memory).
|
||||
#
|
||||
# Runtime: kingjones30/ROCmFPX (fork of charlie12345/ROCmFPX with the
|
||||
# model card's qwen4exp-on-rocmfpx patch) built at
|
||||
# /home/dank/GitPackages/kingjones-ROCmFPX/build (HIP, gfx1151,
|
||||
# ROCm 7.2.4, card recipe). The ROCmFP4 quant types plus the qwen4exp
|
||||
# architecture must be in one tree; stock llama.cpp cannot load this
|
||||
# GGUF.
|
||||
#
|
||||
# Model: Qwen3.8-Flash-Next base (qwen4exp MoE: 512 experts / 10 used,
|
||||
# 48 layers = 12 attention + 36 SSM, QSA indexer, hyper-connections, PLE
|
||||
# at layer 1) quantized Q4_0_ROCMFP4_STRIX, 113.47 GiB single file. The
|
||||
# 50.66 GiB PLE n-gram table stays OUT of GPU memory and streams via
|
||||
# mmap from the OS page cache — never add --no-mmap or -ot for it.
|
||||
#
|
||||
# Profile: card sampling table (temp 0.7, top_p 0.80, top_k 20, min_p 0.0,
|
||||
# presence 1.5 — non-thinking instruct) mixed with the validated serving
|
||||
# profile: ctx 401408 x 2 slots, b 4096 / ub 512, t 16 / tb 16, f16 KV,
|
||||
# fa on (if the graph build aborts on the QSA F16 mask, see the note in
|
||||
# the env file: port scripts/patches/qwen4exp-qsa-f32-mask.patch into
|
||||
# this tree and rebuild), cont-batching, prompt cache + 8 GiB cache-ram
|
||||
# + idle-slot cache, 16 ctx checkpoints, --fit off (auto-fit segfaults
|
||||
# on qwen4exp hyper-connection config). Speculative decoding off: the
|
||||
# Flash-Next MTP heads do not load on this loader; the card ships no
|
||||
# spec flags.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -------------------
|
||||
: "${LLAMA_BIN:=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server}"
|
||||
: "${MODEL_DIR:=/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX}"
|
||||
: "${MODEL_FILE:=Qwen3.8-Flash-Next-Q4_0-ROCmFP4-STRIX.gguf}"
|
||||
: "${MMPROJ_PATH:=/home/dank/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST/mmproj/mmproj-Qwen3.8-Flash-Next-f16.gguf}"
|
||||
: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
|
||||
: "${CHAT_TEMPLATE_KWARGS:={}}"
|
||||
: "${ALIAS:=frank-large}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7770}"
|
||||
: "${CTX_SIZE:=401408}"
|
||||
: "${PARALLEL:=2}"
|
||||
: "${BATCH_SIZE:=4096}"
|
||||
: "${UBATCH_SIZE:=512}"
|
||||
: "${NUM_THREADS:=16}"
|
||||
: "${THREAD_BATCH:=16}"
|
||||
: "${CACHE_TYPE_K:=f16}"
|
||||
: "${CACHE_TYPE_V:=f16}"
|
||||
: "${CTX_CHECKPOINTS:=16}"
|
||||
: "${CACHE_RAM:=8192}"
|
||||
: "${SLOT_SAVE_PATH:=$SCRIPT_DIR/slot-state}"
|
||||
: "${SPLIT_MODE:=none}"
|
||||
: "${FIT:=off}"
|
||||
: "${NGL:=999}"
|
||||
: "${FLASH_ATTN:=on}"
|
||||
: "${TEMPERATURE:=0.7}"
|
||||
: "${TOP_K:=20}"
|
||||
: "${TOP_P:=0.8}"
|
||||
: "${MIN_P:=0.0}"
|
||||
: "${PRESENCE_PENALTY:=1.5}"
|
||||
: "${REASONING:=on}"
|
||||
: "${REASONING_FORMAT:=deepseek}"
|
||||
: "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}"
|
||||
: "${SPEC_TYPE:=none}"
|
||||
: "${DRAFT_MODEL_DIR:=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST}"
|
||||
: "${DRAFT_MODEL_FILE:=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf}"
|
||||
: "${SPEC_DRAFT_N_MAX:=3}"
|
||||
: "${SPEC_DRAFT_N_MIN:=0}"
|
||||
: "${SPEC_DRAFT_P_MIN:=0.75}"
|
||||
: "${SPEC_DRAFT_NGL:=999}"
|
||||
: "${SPEC_DRAFT_DEVICE:=ROCm0}"
|
||||
: "${SPEC_DRAFT_TYPE_K:=f16}"
|
||||
: "${SPEC_DRAFT_TYPE_V:=f16}"
|
||||
|
||||
MODEL="$MODEL_DIR/$MODEL_FILE"
|
||||
for f in "$LLAMA_BIN" "$MODEL" "$CHAT_TEMPLATE_FILE"; do
|
||||
if [[ ! -r "$f" ]]; then
|
||||
echo "missing or unreadable required artifact: $f" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
if [[ ! -x "$LLAMA_BIN" ]]; then
|
||||
echo "missing executable runtime: $LLAMA_BIN" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- speculative decoding (MTP draft head) ---------------------------------
|
||||
# The MTP draft head is a separate GGUF sidecar loaded via --spec-draft-model
|
||||
# (alias --model-draft). ngram-only specs do not need a draft file.
|
||||
# Set SPEC_TYPE=none in the env to disable speculative decoding entirely.
|
||||
SPEC_ARGS=()
|
||||
if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then
|
||||
SPEC_ARGS=(
|
||||
--spec-type "$SPEC_TYPE"
|
||||
--spec-draft-n-max "$SPEC_DRAFT_N_MAX"
|
||||
--spec-draft-n-min "$SPEC_DRAFT_N_MIN"
|
||||
--spec-draft-p-min "$SPEC_DRAFT_P_MIN"
|
||||
)
|
||||
if [[ "$SPEC_TYPE" == *draft-mtp* ]]; then
|
||||
DRAFT_MODEL="$DRAFT_MODEL_DIR/$DRAFT_MODEL_FILE"
|
||||
if [[ ! -r "$DRAFT_MODEL" ]]; then
|
||||
echo "missing or unreadable draft model: $DRAFT_MODEL" >&2
|
||||
exit 1
|
||||
fi
|
||||
SPEC_ARGS+=(
|
||||
--spec-draft-model "$DRAFT_MODEL"
|
||||
--spec-draft-ngl "$SPEC_DRAFT_NGL"
|
||||
--spec-draft-device "$SPEC_DRAFT_DEVICE"
|
||||
--spec-draft-type-k "$SPEC_DRAFT_TYPE_K"
|
||||
--spec-draft-type-v "$SPEC_DRAFT_TYPE_V"
|
||||
)
|
||||
fi
|
||||
if [[ -n "${SPEC_DRAFT_P_SPLIT:-}" ]]; then
|
||||
SPEC_ARGS+=(--spec-draft-p-split "$SPEC_DRAFT_P_SPLIT")
|
||||
fi
|
||||
fi
|
||||
|
||||
# The STRIX repo ships no mmproj; this is the agentionai FAST package's
|
||||
# f16 vision tower for the same base model. Passed only when readable.
|
||||
MMPROJ_ARGS=()
|
||||
if [[ -n "${MMPROJ_PATH:-}" && -r "${MMPROJ_PATH:-}" ]]; then
|
||||
MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH")
|
||||
fi
|
||||
|
||||
SERVER_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
mkdir -p "$SLOT_SAVE_PATH"
|
||||
ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
|
||||
LIBRARY_PATH="$SERVER_DIR:$ROCM_PATH/lib:$ROCM_PATH/lib/rocm_sysdeps/lib:$ROCM_PATH/lib/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
|
||||
|
||||
exec /usr/bin/env \
|
||||
ROCM_PATH="$ROCM_PATH" \
|
||||
HIP_VISIBLE_DEVICES=0 \
|
||||
LD_LIBRARY_PATH="$LIBRARY_PATH" \
|
||||
HSA_OVERRIDE_GFX_VERSION="$HSA_OVERRIDE_GFX_VERSION" \
|
||||
"$LLAMA_BIN" \
|
||||
--model "$MODEL" \
|
||||
--alias "$ALIAS" \
|
||||
--chat-template-file "$CHAT_TEMPLATE_FILE" \
|
||||
--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS" \
|
||||
--host "$BIND_ADDR" \
|
||||
--port "$LISTEN_PORT" \
|
||||
--jinja \
|
||||
--device ROCm0 \
|
||||
--gpu-layers "$NGL" \
|
||||
--split-mode "$SPLIT_MODE" \
|
||||
--parallel "$PARALLEL" \
|
||||
--ctx-size "$CTX_SIZE" \
|
||||
--batch-size "$BATCH_SIZE" \
|
||||
--ubatch-size "$UBATCH_SIZE" \
|
||||
--fit "$FIT" \
|
||||
--flash-attn "$FLASH_ATTN" \
|
||||
--cache-type-k "$CACHE_TYPE_K" \
|
||||
--cache-type-v "$CACHE_TYPE_V" \
|
||||
--threads "$NUM_THREADS" \
|
||||
--threads-batch "$THREAD_BATCH" \
|
||||
--ctx-checkpoints "$CTX_CHECKPOINTS" \
|
||||
--cache-ram "$CACHE_RAM" \
|
||||
--slot-save-path "$SLOT_SAVE_PATH" \
|
||||
--cache-prompt \
|
||||
--cache-idle-slots \
|
||||
--metrics \
|
||||
--temp "$TEMPERATURE" \
|
||||
--top-p "$TOP_P" \
|
||||
--top-k "$TOP_K" \
|
||||
--min-p "$MIN_P" \
|
||||
--presence-penalty "$PRESENCE_PENALTY" \
|
||||
--reasoning "$REASONING" \
|
||||
--reasoning-format "$REASONING_FORMAT" \
|
||||
--reasoning-budget -1 \
|
||||
"${MMPROJ_ARGS[@]}" \
|
||||
"${SPEC_ARGS[@]}"
|
||||
@@ -1,62 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve the FastFlowLM (FLM) sight+sound endpoint — frank-npu.
|
||||
#
|
||||
# Runs `flm serve <tag>` on the XDNA2 NPU (50 TOPS, Dankclaw). The NPU is a
|
||||
# single-lane device: generations serialize in FIFO order, so this is a
|
||||
# background/triage endpoint, not an interactive multi-user chat. The NPU
|
||||
# draws from the unified RAM pool shared with frank-large (the iGPU
|
||||
# llama.cpp service); see serverAppConfigs-docs/LLM/frank-npu.md
|
||||
# for the memory budget and the model-selection tradeoff.
|
||||
#
|
||||
# NOTE: FLM requires unlimited memlock. The unit sets LimitMEMLOCK=infinity.
|
||||
# NOTE: an unknown model tag does NOT 404 — /v1/chat/completions answers HTTP 200 with a
|
||||
# flat {"error":"Failed to load <tag> model!"} body and no choices; /v1/completions returns a
|
||||
# nested OpenAI-shaped server_error (code 500) also under HTTP 200. Client model IDs must be the exact FLM registry
|
||||
# tag (e.g. gemma4-it:e4b), never a service alias. Custom tags only resolve
|
||||
# when FLM_CONFIG_PATH points at the user registry that carries them and their
|
||||
# kernels are symlinked into /home/dank/.config/flm/xclbins/<PackName>.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_NPU_ENV_FILE:-$SCRIPT_DIR/frank-npu.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -------------------
|
||||
: "${FLM_BIN:=/usr/bin/flm}"
|
||||
: "${FLM_MODEL:=gemma4-it:e4b}"
|
||||
: "${FLM_HOST:=0.0.0.0}"
|
||||
: "${FLM_PORT:=7771}"
|
||||
: "${FLM_PMODE:=turbo}"
|
||||
: "${FLM_ASR:=0}"
|
||||
: "${FLM_CTX_LEN:=131072}"
|
||||
|
||||
# The user registry is a superset of the system registry (it carries the
|
||||
# custom tags too), and its model_path resolves relative to the registry file,
|
||||
# so exports keep a tag swap in the env file from needing script edits.
|
||||
export FLM_CONFIG_PATH="${FLM_CONFIG_PATH:-/home/dank/.config/flm/model_list.json}"
|
||||
export FLM_XCLBIN_PATH="${FLM_XCLBIN_PATH:-/home/dank/.config/flm/xclbins}"
|
||||
|
||||
if [[ ! -x "$FLM_BIN" ]]; then
|
||||
echo "missing flm binary: $FLM_BIN" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- optional flags ----------------------------------------------------------
|
||||
OPT_ARGS=()
|
||||
if [[ "${FLM_ASR}" != "0" ]]; then
|
||||
OPT_ARGS+=(--asr "${FLM_ASR}")
|
||||
fi
|
||||
if [[ "${FLM_CTX_LEN}" != "-1" ]]; then
|
||||
OPT_ARGS+=(--ctx-len "${FLM_CTX_LEN}")
|
||||
fi
|
||||
|
||||
exec "$FLM_BIN" serve "$FLM_MODEL" \
|
||||
--host "$FLM_HOST" \
|
||||
-p "$FLM_PORT" \
|
||||
--pmode "$FLM_PMODE" \
|
||||
"${OPT_ARGS[@]}"
|
||||
@@ -1,87 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve peculiar-ragdoll/Tiel-Coder-35B-A3B (UD-Q6_K_XL) as
|
||||
# frank-default-small via stock llama.cpp on Dankserver (RTX 3090).
|
||||
#
|
||||
# Runtime: upstream llama.cpp (latest master, CUDA build with
|
||||
# -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86).
|
||||
# The model is a 35B-total / 8-active MoE hybrid (qwen35moe arch) with
|
||||
# SSM/Mamba layers (full_attention_interval=4: 10 attention + 30 SSM
|
||||
# layers out of 40). 256 experts, 8 used per token.
|
||||
#
|
||||
# Expert offload: --cpu-moe keeps all 256 expert weights in CPU RAM.
|
||||
# Non-expert weights (attention, SSM, shared expert, embeddings, output)
|
||||
# are ~1.6 GB and fit entirely on the 24 GB RTX 3090.
|
||||
#
|
||||
# KV cache: Q8_0 for both K and V. With CTX_SIZE=200700 (66 900 per
|
||||
# slot × 3 parallel) and 10 attention layers (2 KV heads × 256 dim),
|
||||
# the KV cache is ~2 GB — the requested 2 GB VRAM buffer.
|
||||
#
|
||||
# Model is loaded with mmap (default). The 30 GB Q6_K_XL file is
|
||||
# memory-mapped; only accessed pages enter RAM.
|
||||
#
|
||||
# Tunables come from frank-default-small.env unless FRANK_SMALL_ENV_FILE
|
||||
# overrides that path.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-default-small.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -----------------
|
||||
: "${MODEL_DIR:=/home/dank/models/tiel-coder-35b}"
|
||||
: "${MODEL_FILE:=Tiel-Coder-35B-A3B-UD-Q6_K_XL.gguf}"
|
||||
: "${LLAMA_SERVER:=/home/dank/llama.cpp/build/bin/llama-server}"
|
||||
: "${ALIAS:=frank-default-small}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7770}"
|
||||
: "${CTX_SIZE:=200700}"
|
||||
: "${PARALLEL:=3}"
|
||||
: "${CACHE_TYPE_K:=q8_0}"
|
||||
: "${CACHE_TYPE_V:=q8_0}"
|
||||
: "${N_GPU_LAYERS:=999}"
|
||||
: "${BATCH_SIZE:=4096}"
|
||||
: "${UBATCH_SIZE:=1024}"
|
||||
: "${VERBOSITY:=2}"
|
||||
: "${CPU_MOE:=1}"
|
||||
: "${TEMPERATURE:=1.0}"
|
||||
: "${TOP_K:=20}"
|
||||
: "${TOP_P:=0.95}"
|
||||
: "${MIN_P:=0.0}"
|
||||
: "${THREADS:=7}"
|
||||
: "${THREADS_HTTP:=8}"
|
||||
|
||||
MODEL="$MODEL_DIR/$MODEL_FILE"
|
||||
|
||||
if [[ ! -f "$MODEL" ]]; then
|
||||
echo "missing required file: $MODEL" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -x "$LLAMA_SERVER" ]]; then
|
||||
echo "missing llama-server binary: $LLAMA_SERVER" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exec "$LLAMA_SERVER" \
|
||||
--model "$MODEL" \
|
||||
$([ "$CPU_MOE" = "1" ] && printf -- "--cpu-moe") \
|
||||
--cache-type-k "$CACHE_TYPE_K" \
|
||||
--cache-type-v "$CACHE_TYPE_V" \
|
||||
--ctx-size "$CTX_SIZE" \
|
||||
--parallel "$PARALLEL" \
|
||||
--gpu-layers "$N_GPU_LAYERS" \
|
||||
--batch-size "$BATCH_SIZE" \
|
||||
--ubatch-size "$UBATCH_SIZE" \
|
||||
--temp "$TEMPERATURE" \
|
||||
--top-k "$TOP_K" \
|
||||
--top-p "$TOP_P" \
|
||||
--min-p "$MIN_P" \
|
||||
--threads "$THREADS" \
|
||||
--threads-http "$THREADS_HTTP" \
|
||||
--alias "$ALIAS" \
|
||||
--host "$BIND_ADDR" \
|
||||
--port "$LISTEN_PORT"
|
||||
@@ -1,6 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Repository pre-commit hook: run only tests selected by staged paths.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(git rev-parse --show-toplevel)"
|
||||
exec "$REPO_ROOT/scripts/test-changed.sh"
|
||||
-34
@@ -1,40 +1,6 @@
|
||||
# Local environment files — never commit real values
|
||||
.env
|
||||
*.env
|
||||
!.env.example
|
||||
# LLM fleet tunables (paths and numbers, never credentials)
|
||||
!scripts/*.env
|
||||
|
||||
# LLM fleet runtime state (slot save/restore files written by the server)
|
||||
scripts/slot-state/
|
||||
|
||||
# Playwright e2e: local creds, browser storage states, node artifacts
|
||||
tests/e2e/.env.local
|
||||
tests/e2e/.auth/
|
||||
tests/e2e/node_modules/
|
||||
tests/e2e/test-results/
|
||||
tests/e2e/playwright-report/
|
||||
|
||||
# pi subagent artifacts
|
||||
.pi-subagents/
|
||||
|
||||
# doc-audit scratch (transient, synthesized into serverAppConfigs-docs/ then trashed)
|
||||
.docaudit/
|
||||
|
||||
# Agent trash
|
||||
.AGENTTRASH/
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Virtual environments
|
||||
.venv/
|
||||
|
||||
# Obsidian editor state
|
||||
.obsidian/
|
||||
|
||||
# Editor/OS noise
|
||||
*.swp
|
||||
.DS_Store
|
||||
third_party/
|
||||
|
||||
@@ -1,908 +1,103 @@
|
||||
# AGENTS.md — serverAppConfigs Agent Rules
|
||||
# vLLM Scripts — Agent Instructions
|
||||
|
||||
## Overview
|
||||
## Project Overview
|
||||
Scripts for running large language models via vLLM on two local machines:
|
||||
- **dankserver**: NVIDIA GPU + System RAM (frank-default-small)
|
||||
- **dankclaw**: AMD Strix Halo Shared Memory (frank-default-large)
|
||||
|
||||
This repository is the single source of truth for the two homelab servers and
|
||||
the FrankBook-Pro Mac mini. It tracks the Podman container stacks and the LLM
|
||||
inference fleet. One repo, one checkout per host, one deploy script. All
|
||||
services run from `/home/dank/serverAppConfigs` on each host (the 2026-08-22
|
||||
merge absorbed the former `vllmScripts` repo, history preserved).
|
||||
## Directory Structure
|
||||
```
|
||||
vllmScripts/
|
||||
├── AGENTS.md # This file — agent instructions
|
||||
├── README.md # Human-readable project overview
|
||||
├── deploy.sh # Automated deployment script
|
||||
├── scripts/ # Executable vLLM launch scripts
|
||||
│ ├── validate-vllm.sh # Validation tool
|
||||
│ └── *.sh # Inference server scripts
|
||||
├── tests/ # Test suite
|
||||
│ └── test_validator.sh
|
||||
├── docs/ # Knowledge base
|
||||
│ ├── README.md
|
||||
│ └── *.md # Research notes
|
||||
└── failedAttempts/ # Archived scripts (for reference)
|
||||
```
|
||||
|
||||
- **Dankserver** (192.168.0.235) — media/library stack (RomM, Copyparty,
|
||||
Calibre-Web, Jellyfin, Plex) plus the `frank-primary` LLM service.
|
||||
- **Dankclaw** (192.168.0.143) — AI/search/dev/observability stack (SearXNG
|
||||
via Gluetun, ComfyUI, OpenWebUI, Hermes, Gitea, Grafana, Prometheus, Loki,
|
||||
Alloy, Pi-hole) plus TWO LLM services: `frank-large` (stock llama.cpp on
|
||||
the gfx1151 iGPU) and `frank-small` (llama.cpp on the RTX 3090). Caddy on
|
||||
Dankclaw reverse-proxies both hosts.
|
||||
- **FrankBook-Pro** (192.168.0.112) — macOS mini running the *arr stack
|
||||
(Prowlarr, Sonarr, Radarr, Lidarr, Readarr, qBittorrent) as quadlets inside
|
||||
its podman machine; all egress rides the Gluetun WireGuard netns. Deployed
|
||||
via the `frankbook-pro` handler.
|
||||
## Running Tests
|
||||
```bash
|
||||
# Test validator on dankserver
|
||||
bash tests/test_validator.sh dankserver
|
||||
|
||||
## Fleet topology (ACTIVE)
|
||||
# Test validator on dankclaw
|
||||
bash tests/test_validator.sh dankclaw
|
||||
|
||||
| Target | Service | Runtime | Model | Port |
|
||||
|--------|---------|---------|-------|------|
|
||||
| Dankserver | `frank-primary` | R9V radiance-vLLM fork `drwolfen/radiance-vllm-qwen4exp` (docker-pinned image bundle `2dac17a2`, docker→podman shim, root systemd unit) | unsloth Qwen3.8-Flash-Next-GGUF Q8_0 (175.30 GiB, 6 GGUF shards) with the blockfp8 MTP4 head (depth 4) and mmproj, TP=2 tiered experts (80-slot LRU cache; PLE n-gram PINNED in host RAM 54.4 GB; quantized inject-shard dev-overlay adapter), CED off, 131072 ctx single stream, Qwen-Sharp template, tool parser `qwen3_coder`, reasoning channel `reasoning` | 7770 |
|
||||
| Dankclaw | `frank-large` | gufo `gufo-org/gufo` v0.1.1, the Strix Halo engine, BUILT FROM SOURCE (root systemd unit; links the host ROCm 7.2.4) | Swift-1.5-Qwen3.8-27B-Uncensored UD-Q8_K_XL (27B dense, ABLITERATED/uncensored, 29.3 GiB, SWIFT OPEN LICENSE 1.0) with the z-lab DFlash2 Q4_K_M drafter (adaptive block policy, cap 7) on the gfx1151 iGPU (Strix Halo LPDDR, 114 GiB GTT), 262144 ctx single session, greedy, gufo's compiled-in Qwen renderer (NO Qwen-Sharp), text only | 7770 |
|
||||
| Dankclaw | `frank-small` | mainline llama.cpp master 37b53fd (root systemd unit) | HauhauCS Gemma4-12B-QAT-Uncensored-Balanced Q4_K_M (BF16 mmproj + MTP assistant sidecar, `draft-mtp,ngram-mod`) on the RTX 3090, 1048576 pool / 4 slots of 262,144, q8_0 KV, the fleet's fastest vision endpoint | 7771 |
|
||||
# Run all tests
|
||||
bash tests/test_validator.sh dankserver && bash tests/test_validator.sh dankclaw
|
||||
```
|
||||
|
||||
Retired endpoints (`frank-npu`, and every prior runtime) are in
|
||||
the [runbook index](#runbook-index). The full swap history lives in
|
||||
`serverAppConfigs-docs/LLM/index.md` — do not grow it here.
|
||||
## Deployment
|
||||
```bash
|
||||
# Deploy all scripts to all targets
|
||||
bash deploy.sh all
|
||||
|
||||
## Repository Layout
|
||||
# Deploy specific script to dankserver
|
||||
bash deploy.sh dankserver vllm-qwen36.sh
|
||||
|
||||
- `dankserver/` — compose.yaml + Caddyfile + .env.example for the media stack
|
||||
- `dankclaw/` — compose.yaml + Caddyfile + .env.example for the AI stack
|
||||
- `shared/` — homepage, Prometheus, Loki, Alloy, SearXNG configs
|
||||
- `scripts/` — `deploy.sh` / `deploy-plan.py` / `deploy-remote.sh`, LLM serve
|
||||
scripts, systemd units, env tunables, bench harnesses
|
||||
- `old/` — retired LLM scripts (reference only)
|
||||
- `serverAppConfigs-docs/` — knowledge base (Obsidian vault, OKF format)
|
||||
- `tests/` — bats suite + Playwright e2e
|
||||
# Deploy specific script to dankclaw
|
||||
bash deploy.sh dankclaw vllm-thinkingcap.sh
|
||||
```
|
||||
|
||||
## Git Workflow
|
||||
|
||||
All edits flow through git. Never edit files on a remote machine directly.
|
||||
### Branching Strategy
|
||||
- `main`: Stable, working scripts only
|
||||
- `dankserver/*`: Server-specific development
|
||||
- `dankserver/qwen36-awq`: Qwen3.6 AWQ experiments
|
||||
- `dankserver/*`: Feature branches for dankserver
|
||||
- `dankclaw/*`: Claw-specific development
|
||||
- `dankclaw/*`: Feature branches for dankclaw
|
||||
|
||||
1. **Source** the live config from the target machine
|
||||
2. **Commit** the fetched version here (redact secrets into `.env`)
|
||||
3. **Edit** locally
|
||||
4. **Test** — `./tests/run.sh` must pass
|
||||
5. **Commit and push** — origin is Gitea on Dankclaw
|
||||
6. **Pull** on the target (`deploy.sh` does this)
|
||||
7. **Deploy** — `scripts/deploy.sh` plans and dispatches only changed services
|
||||
### Worktree Pattern
|
||||
For parallel development:
|
||||
```bash
|
||||
# Create worktree for feature
|
||||
git worktree add ../vllm-feature feature/qwen36-awq
|
||||
|
||||
## Deployment
|
||||
|
||||
```
|
||||
./scripts/deploy.sh dankclaw
|
||||
./scripts/deploy.sh dankserver
|
||||
./scripts/deploy.sh dankclaw --service searxng
|
||||
./scripts/deploy.sh dankserver --service frank-primary
|
||||
./scripts/deploy.sh dankclaw --dry-run
|
||||
# When done, merge back
|
||||
git worktree remove ../vllm-feature
|
||||
```
|
||||
|
||||
The planner maps changed paths to services; the remote runner restarts only
|
||||
the selected service and records its revision outside the checkout.
|
||||
Documentation-only changes are a no-op. LLM services are selected only by
|
||||
their own files — every endpoint (`frank-large`/`frank-small`/`frank-primary`)
|
||||
by its unit/env/serve trio in `scripts/` — so
|
||||
unrelated
|
||||
container changes cannot interrupt inference. Set `FRANK_SUDO_PW` for headless
|
||||
runs on a host without passwordless sudo; the value never enters the repo.
|
||||
|
||||
### Deployment completeness (MANDATORY)
|
||||
|
||||
A deploy is **not complete** when `deploy.sh` exits green. It requires:
|
||||
|
||||
1. `./tests/run.sh` — all seams green, and
|
||||
2. the Playwright e2e suite (`tests/e2e/run.sh`) — runs **automatically** as
|
||||
the `[3/3]` gate at the end of `deploy.sh` against the LIVE fleet; a
|
||||
behavior-affecting deploy that fails it exits `DEPLOY INCOMPLETE`. Bypass
|
||||
only with `--skip-e2e` in an emergency.
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
A[Edit locally] --> B[Tests pass] --> C[Commit + push]
|
||||
C --> D[deploy.sh plan] --> E[Target pulls ff-only]
|
||||
E --> F[Dispatch selected service] --> G[Per-service health]
|
||||
G --> H[Playwright e2e gate] --> I{green?}
|
||||
I -->|yes| J[Deploy complete]
|
||||
I -->|no| K[DEPLOY INCOMPLETE]
|
||||
```
|
||||
|
||||
## Boot Persistence & Self-Healing (MANDATORY)
|
||||
|
||||
Every deployed, working Podman container MUST have a self-healing,
|
||||
boot-persistent Quadlet unit. A container that runs only from `compose.yaml`
|
||||
or a manual `podman run` dies silently at reboot — rootless Podman does not
|
||||
auto-start bare containers, and `restart: unless-stopped` does not survive the
|
||||
explicit stop that shutdown performs. Never report such a service as "up".
|
||||
|
||||
Unit requirements:
|
||||
|
||||
- **Boot persistence** — `[Install] WantedBy=default.target`, installed to
|
||||
`~/.config/containers/systemd/<name>.container`, enabled in the user
|
||||
manager, with linger on for the deploy user (`loginctl show-user dank -p
|
||||
Linger` → `Linger=yes`).
|
||||
- **Self-healing** — `Restart=always` (or `on-failure`) with a short
|
||||
`RestartSec`; quadlet health check where the service has one.
|
||||
- **Repo provenance** — the `.container` file lives in this repo
|
||||
(`<host>/<svc>/quadlet/`), is installed by the deployer, and is covered by
|
||||
the quadlet test seam. The host-side unit is generated state; the repo file
|
||||
is the source of truth.
|
||||
|
||||
Before declaring a service deployed: `systemctl --user is-enabled <name>` →
|
||||
`enabled`, `is-active` → `active`, and the container answers its health check.
|
||||
A running container with no unit is a gap: write the unit, test it, deploy it.
|
||||
|
||||
This rule exists because reboots found gaps twice (2026-08-26: Caddy as a
|
||||
bare compose container never returned; 2026-09-11: five observability
|
||||
services had no units). The observability services now have units under
|
||||
`dankclaw/observability/quadlet/` and their compose definitions were removed —
|
||||
one owner per container name. The stack is **configured and delivering as of
|
||||
2026-09-17**: a Grafana datasource and the `llm-fleet` dashboard are
|
||||
provisioned from `shared/grafana/provisioning/`, `shared/prometheus/rules/
|
||||
llm-fleet.yml` loads 5 alert rules + 1 recording rule, and firing rules are
|
||||
delivered: Prometheus → Alertmanager (`dankclaw/observability/quadlet/
|
||||
alertmanager.container`, loopback :9093) → host-side Signal relay
|
||||
(`scripts/signal-relay.py`, user unit, 192.168.0.143:18093 — the host LAN
|
||||
IP, the one bind a container can reach as `host.containers.internal`;
|
||||
iptables admits loopback, the host and the podman bridge subnet only) → signal-cli account 318311.
|
||||
Prometheus targets read down for structural reasons (HTML-scraped "probes",
|
||||
Gitea metrics off); the remedy is a blackbox exporter. See
|
||||
`serverAppConfigs-docs/Infrastructure/Observability.md`.
|
||||
|
||||
## Testing
|
||||
|
||||
Tests are mandatory before commit (`bats` required on the dev machine):
|
||||
|
||||
```
|
||||
./tests/run.sh
|
||||
```
|
||||
|
||||
Seams: compose validity, env consistency (no leaked secrets), Caddyfile
|
||||
validity, quadlet units, docs completeness **including vault link integrity**,
|
||||
script syntax (shell + python), LLM unit integrity, LLM secret hygiene,
|
||||
deployment flow, service-scoped planning, homepage config/pages/deploy/checker,
|
||||
and `e2e_scaffold` — which fails if a Playwright spec file's number prefix is in
|
||||
no project, because such a spec silently never runs.
|
||||
See `serverAppConfigs-docs/Architecture/Testing Policy.md`.
|
||||
|
||||
## Secret Rules
|
||||
|
||||
- Never commit real secrets. The live Gitea DB password and sudo password are
|
||||
test-enforced leak checks.
|
||||
- `.env` files are git-ignored; commit only `.env.example` with placeholders.
|
||||
Exception: `scripts/*.env` LLM tunables (paths and numbers, never
|
||||
credentials).
|
||||
- Do not document passwords anywhere in the repo. SSH keys are the auth
|
||||
mechanism; `FRANK_SUDO_PW` covers headless sudo and never enters the repo.
|
||||
- **A token passed as `curl -H "Authorization: Bearer …"` is world-readable.**
|
||||
It lands in `/proc/<pid>/cmdline`, so any `ps`/`pgrep -a` on the host prints
|
||||
it — that leaked an HF token into a session transcript on 2026-09-16, while
|
||||
merely checking whether a download was still running. Pass secrets out of
|
||||
band (`curl -K <mode-600 file>` carrying the header, or `HF_TOKEN` in the
|
||||
environment for `hf`/`hf_transfer`) and never `pgrep -a` a fetch command that
|
||||
carries an auth header.
|
||||
|
||||
## External Exposure Policy (MANDATORY)
|
||||
|
||||
Every Caddy-proxied service is either behind the auth chain (Anubis :8923 →
|
||||
Authentik embedded outpost) or explicitly internal-only — no public DNS name,
|
||||
no vhost. New services default to internal-only until guarded.
|
||||
|
||||
When guarding a new vhost — provider first, then route:
|
||||
|
||||
1. Create the outpost proxy provider FIRST (`external_host` = exact vhost URL
|
||||
with trailing slash; `internal_host = http://caddy:8017`). If the vhost
|
||||
flips first, authenticated traffic hits the outpost error page.
|
||||
2. Flip the vhost to `reverse_proxy anubis:8923` with
|
||||
`header_up X-Real-Ip {remote_host}`, and add the post-auth
|
||||
`@<svc> host <vhost>` handler on the `http://:8017` listener routing to
|
||||
the real upstream.
|
||||
3. Anubis guard set, Caddy TLS edge, and outpost provider list change
|
||||
together in one deploy.
|
||||
4. Service-to-service traffic targets internal addresses (`loki:3100`), never
|
||||
the public hostname — non-browser clients cannot solve challenges.
|
||||
|
||||
Exempt categories (document the reason in the vhost comment): the IdP itself
|
||||
(`auth.`), git smart-HTTP endpoints, services with their own complete auth.
|
||||
|
||||
## Deletion & Trash Policy (MANDATORY)
|
||||
|
||||
Never `rm`. Move deletions into trash so they stay recoverable:
|
||||
|
||||
1. In-repo files → `.AGENTTRASH/` beside the item's directory.
|
||||
2. Elsewhere → dated staging dir `~/.AGENTTRASH/YYYYMMDD-topic`.
|
||||
3. List staged items and sizes; wait for explicit user approval to purge.
|
||||
|
||||
`rm`, `rm -rf`, and `git clean -f` are forbidden without explicit permission.
|
||||
`git * --no-verify` is forbidden. The trash is a temporary hold, not an
|
||||
archive.
|
||||
|
||||
## Benchmarking Policy (MANDATORY)
|
||||
|
||||
**STOP the production service BEFORE benchmarking** — these servers serve
|
||||
real users, and a bench beside a live service starves the GPU and corrupts
|
||||
results. Use the runtime's own harness: `llama-bench` for the llama.cpp
|
||||
endpoints, `gufo bench` for frank-large. Rolling your own bench
|
||||
requires asking first.
|
||||
Scratch ports are not used on LLM hosts — a
|
||||
test process cannot share the single GPU anyway; validate against the service
|
||||
itself.
|
||||
|
||||
**`gufo bench` cannot rank speculative-decode settings, and will invert the
|
||||
ranking.** Measured 2026-09-28 on frank-large at depth 32768: its corpus
|
||||
reports `acceptance=0.933 ± 0.000` with an **identical trace sha256** across
|
||||
draft widths, i.e. the same highly-predictable text every run, and 93 % of
|
||||
every proposal accepted. It reported 29–37 t/s decode where real traffic
|
||||
gives ~15 t/s, and it made `--draft-tokens 16` look 6 % faster than 7 when
|
||||
real text made it 14 % *slower*. Use it for shape, not for A/B. For an A/B,
|
||||
replay **real** text through the live server and read the `Server-Timing`
|
||||
header — but note the unit rides in the **name** half (`ttft;dur=402.485`),
|
||||
so splitting on `=` alone keys the dict `ttft;dur` and every lookup silently
|
||||
returns nothing. Generate **at least 128 tokens** for the measurement: at 8
|
||||
tokens, `inter_token` is first-token noise and the run-to-run spread swamps
|
||||
every real difference.
|
||||
|
||||
This covers **any GPU experiment, not just benchmarks** — including a bare
|
||||
2-rank RCCL/`torch.distributed` probe. On 2026-09-16 one such probe was run
|
||||
against the live service "because it only needs 200 MB of VRAM"; the host went
|
||||
hard down ~20 minutes later (no ARP reply, 100% ICMP loss from two subnets, no
|
||||
recovery — a power cycle was required). A hung AQL dispatch wedging the GPU
|
||||
driver is the plausible mechanism, and this host has precedent for going
|
||||
unreachable until a power cycle. If an experiment needs GPUs, stop and mask
|
||||
first, or use a host that serves nobody. Telling an off machine from a wedged
|
||||
one: `ip neigh show <ip>` from a peer — FAILED means off/NIC dead, while a
|
||||
merely wedged kernel usually still answers ARP.
|
||||
|
||||
1. Stop and mask (self-healing would restart it):
|
||||
- Dankclaw `frank-large` (root unit, gufo on the iGPU):
|
||||
`sudo systemctl stop frank-large && sudo systemctl mask frank-large`
|
||||
- Dankclaw `frank-small` (root unit, llama.cpp on the 3090):
|
||||
`sudo systemctl stop frank-small && sudo systemctl mask frank-small`
|
||||
- Dankserver `frank-primary` (root unit, R9V radiance on the 2x R9700):
|
||||
`sudo systemctl stop frank-primary && sudo systemctl mask frank-primary`
|
||||
2. Bench.
|
||||
NOTE: `systemctl mask` FAILS on a relinked unit (the unit file in
|
||||
`/etc/systemd/system/` is itself a symlink into the checkout — mask
|
||||
refuses to replace it). Stop is enough for the bench window; the deploy
|
||||
restores the unit, so re-deploy after benching instead of unmasking.
|
||||
3. Unmask and start; verify `curl http://192.168.0.<host>:7770/health`.
|
||||
|
||||
**CRITICAL:** never `systemctl disable` a system unit on these hosts — it
|
||||
deletes the unit file from `/etc/systemd/system/`, not just the symlink.
|
||||
`mask` symlinks to `/dev/null` and is safe/reversible.
|
||||
|
||||
## Model Config (harness)
|
||||
|
||||
The harness reaches the fleet through `.omp/agent/models.yml` in the
|
||||
operator's home — `/home/blank/.omp/agent/models.yml` on the dev box, which is
|
||||
where the harness runs; there is no copy on the servers. Each provider is one
|
||||
HOST:PORT; each model `id` is the served alias: `frank-primary` →
|
||||
`192.168.0.235:7770`, `frank-large` → `192.168.0.143:7770`, `frank-small` →
|
||||
`modelRoles`.
|
||||
|
||||
Resolution rules: the default model resolves as a `defaultProvider` +
|
||||
`defaultModel` pair (the provider must serve the model); a subagent override
|
||||
uses a bare model `id`, which resolves only when unique across providers. Two
|
||||
ports on one host need two providers — the file cannot express "same host,
|
||||
different port".
|
||||
|
||||
When renaming a service, retiring an endpoint, or moving a model: update the
|
||||
provider block, model `id`, `contextWindow`, `maxTokens`, `input` in
|
||||
`models.yml`, then fix every `modelRoles` entry that named the old endpoint —
|
||||
a role pointing at a provider the file does not define fails at call time. As
|
||||
of 2026-09-26 both `frank-small` and `frank-primary` accept images
|
||||
(`input: [text, image]`) — frank-primary loads the BF16 mmproj explicitly
|
||||
(this build never autoloads projectors); `frank-large` remains text-only
|
||||
because its stock llama.cpp deployment loads no
|
||||
vision projector (the mmproj is not loaded). Note also that
|
||||
the reasoning channel differs by runtime: llama.cpp endpoints emit
|
||||
`reasoning_content`, vLLM emits `reasoning` — academic since the
|
||||
2026-09-26 swap, as every endpoint in the fleet is llama.cpp now.
|
||||
There is **no audio**
|
||||
input anywhere in the fleet except frank-small: that capability left with the
|
||||
FLM NPU endpoint on 2026-09-11 and returned on 2026-09-17 with the Gemma4
|
||||
projector's audio encoder (frank-small transcribes WAV verbatim even though
|
||||
`audio` is omitted from its `input` list — OMP's schema rejects it).
|
||||
|
||||
### frank-large — gufo, Swift-1.5-Qwen3.8-27B-Uncensored UD-Q8_K_XL (Dankclaw, ACTIVE)
|
||||
|
||||
**gufo** (`gufo-org/gufo`) v0.1.1 — "the Strix Halo inference engine", a
|
||||
vertical engine built for exactly this hardware (Ryzen AI MAX+ 395 /
|
||||
Radeon 8060S, `gfx1151`) — as a **root systemd unit**
|
||||
(`scripts/frank-large.service`, `User=dank`), serving
|
||||
`ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF`'s `UD-Q8_K_XL`
|
||||
(27B dense, 29.3 GiB, sha256-verified; **abliterated/uncensored**, and under
|
||||
the *non-OSI* SWIFT OPEN LICENSE 1.0) on port 7770 from the gfx1151 iGPU.
|
||||
One session, full 262,144 ctx (the model's native window, `SESSIONS=1`),
|
||||
DFlash2 spec decode (the z-lab `Qwen3.8-27B-DFlash2-Q4_K_M` drafter, gufo's
|
||||
**adaptive** block policy, `--draft-tokens 7`), greedy sampling, reasoning
|
||||
extracted to `reasoning_content`, and **no vision projector**.
|
||||
|
||||
**Two swaps landed on 2026-09-28**, both same day: the engine (from the
|
||||
unsloth llama.cpp b11030 prebuilt, which had replaced Kairic Edge ROCmFPX)
|
||||
and then the payload (from `unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL`, still on
|
||||
disk and the payload-only rollback). The two targets are structurally
|
||||
interchangeable for gufo — same `qwen35` architecture, the same 866 tensors
|
||||
in the same order, the same per-tensor Q8_0/F32/BF16 layout, the same MTP
|
||||
head at `blk.64` — verified by parsing and diffing both GGUF headers before
|
||||
either was loaded. **The DFlash2 drafter did not move**, so it has its own
|
||||
`DRAFTER_DIR` and the deployer's stray-run sweep keys on the drafter's
|
||||
filename, not the target's.
|
||||
|
||||
**The runtime is built from source on the host** —
|
||||
`/home/dank/GitPackages/gufo-v0.1.1/`, reproducible with
|
||||
`scripts/build-gufo.sh` (`--verify` re-checks an installed tree). gufo ships
|
||||
no prebuilt Linux binaries and the v0.1.0/v0.1.1 release assets are empty.
|
||||
Unlike the retired llama.cpp prebuilt, which bundled its own ROCm, **gufo
|
||||
links the host ROCm 7.2.4 through its RPATH**, so the HIP dev packages must
|
||||
stay installed. Two host workarounds the build script encodes:
|
||||
|
||||
- **Ubuntu's `librocwmma-dev` must be absent.** It installs a truncated
|
||||
rocWMMA header set into `/usr/include` that shadows AMD's complete one
|
||||
and breaks the build with `'internal/accessors.hpp' file not found`. A
|
||||
`-I` cannot fix it — in HIP mode the host pass searches the system
|
||||
directories first. The package has no reverse dependencies.
|
||||
- **`--gcc-install-dir` must name GCC 15.** Ubuntu 26.04 ships
|
||||
`/usr/include/c++/16`, which the ROCm clang prefers; its `<atomic_wait.h>`
|
||||
needs a `std::__detail::__notify_impl` symbol version the installed
|
||||
`libstdc++.so.6` (GLIBCXX_3.4.35) does not export, and the link fails.
|
||||
|
||||
`gufo diagnose` is the preflight and must report `[PASS]` with `gfx1151`.
|
||||
It also self-caps compute queues (`GPU_MAX_HW_QUEUES`, 2 resident for
|
||||
`serve llm`) after reading `/sys/class/kfd/kfd/proc/*/queues/*/type`, which
|
||||
matters because ComfyUI holds queues on the same device; four HIP processes
|
||||
is the device ceiling.
|
||||
|
||||
**The client contract changed, and the chat template is the sharp edge.**
|
||||
**gufo has no Jinja engine at all** — `src/models/qwen/chat_template.cpp` is
|
||||
587 lines of hand-written C++ that appends strings, with no
|
||||
`--chat-template-file`, no env override, and no template body to point at.
|
||||
`scripts/qwen-sharp-v22.4.0.jinja` therefore **cannot** be supplied to
|
||||
frank-large: that is a port, not a config line. Given the choice on
|
||||
2026-09-28 the decision was to ship on gufo's stock renderer and accept the
|
||||
loss — the `terse` system-prompt injection, the in-band `<|think_*|>`
|
||||
control, the `json|xml` `tool_call_format` switch, the tool-argument
|
||||
truncation and `add_vision_id` prefixes are all gone. gufo's own renderer
|
||||
does keep the effort instruction strings and `preserve_thinking`, which are
|
||||
what the harness depends on. **This is the thing to watch**: the endpoint
|
||||
now runs an *abliterated* model without the prompt that tells it to stop
|
||||
waffling. The cheap fix, if verbosity shows up, is to send the `terse` text
|
||||
as an ordinary system message from `models.yml` — not a template change.
|
||||
`reasoning_effort` accepts the full OpenAI set (all six levels verified
|
||||
200), so `models.yml`'s `reasoningEffortMap` fold is no longer *required* —
|
||||
it is kept deliberately, so that removing it does not silently move every
|
||||
`high` role from `xhigh` to `high` effort inside a swap. And there is no
|
||||
KV-quantization or prompt-cache-RAM flag; prompt reuse is on by default and
|
||||
`--cache-disk` is opt-in and off.
|
||||
|
||||
**Read the perf trade before assuming a win.** gufo's own Q8_K_XL tables
|
||||
(vs llama.cpp b11069, DFlash2) show prefill +37 % to +80 %, cold load 4.5×
|
||||
faster, decode +7 % at zero depth and +40 % at 4 users — but **−14 % at
|
||||
65,536 tokens of depth and −24 % at 131,072**. On the 8-bit target the
|
||||
crossover is ~32k of cached context, and frank-large runs one 262,144-token
|
||||
conversation, so long agent sessions sit on the losing side of that line.
|
||||
The Q4_K_XL target shows no such regression; revisiting the quantization is
|
||||
a separate decision from the runtime. Peak memory is 50.58 GiB at the full
|
||||
262,144 context against the iGPU's 114 GiB GTT aperture.
|
||||
|
||||
The serve script (`scripts/serve-frank-large.sh`) validates runtime → target
|
||||
GGUF → drafter and exits with a pointer message if any is missing, then
|
||||
`exec`s `gufo serve --host 0.0.0.0 --port 7770 --sessions 1 llm --model
|
||||
…--served-model-name frank-large --context 262144 --speculative dflash2 …`.
|
||||
The deployer's `deploy_llm_large` gates on the **runtime binary** as well
|
||||
as both payload paths, and returns before it stops anything.
|
||||
|
||||
Fetch: `scripts/download-frank-large-swift.sh` (the target; per-file byte
|
||||
count + sha256 against the Hub LFS oid, no auth header — the repo is public
|
||||
and a token on the curl command line is world-readable) and
|
||||
`scripts/download-frank-large.sh` (the DFlash2 drafter, which did not move).
|
||||
Both take `--verify-only`.
|
||||
|
||||
Runbook: `serverAppConfigs-docs/LLM/frank-large-gufo-qwen38-27b.md`.
|
||||
Rollback, two ways: **payload only** — re-fetch with
|
||||
`scripts/download-frank-large.sh` and repoint `MODEL_FILE` at
|
||||
`/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-UD-Q8_K_XL.gguf`; that
|
||||
29.3 GiB target was **reclaimed on 2026-09-28** (staged to
|
||||
`~/.AGENTTRASH/20260928-retired-models/`, restorable from there), so it is
|
||||
no longer a one-minute flip. **Engine** —
|
||||
[[frank-large-unsloth-qwen38-27b-q8xl]], the b11030 prebuilt at
|
||||
`/home/dank/GitPackages/unsloth-llama-b11030/`; the only way to get
|
||||
Qwen-Sharp back. The DFlash2 drafter was never reclaimed and still lives in
|
||||
`Qwen3.8-27B-UD-Q8_K_XL/`.
|
||||
|
||||
### frank-small — llama.cpp on the RTX 3090 (Dankclaw, ACTIVE)
|
||||
|
||||
Mainline llama.cpp master `37b53fd` (CUDA build with
|
||||
`-DGGML_CUDA_FA_ALL_QUANTS=ON`) as a **root systemd unit** (repo
|
||||
`scripts/frank-small.service`, `relink_unit`ed into `/etc/systemd/system/`),
|
||||
serving `HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced` at Q4_K_M.
|
||||
The MTP head is a SEPARATE assistant file (`mtp-gemma-4-12B-it.gguf`, 4
|
||||
nextn layers) loaded via `--model-draft`; the draftless **ngram-mod**
|
||||
matcher chains behind it in the same `--spec-type` value. BF16 mmproj,
|
||||
port 7771, q8_0 KV, `CTX_SIZE=1048576` / `PARALLEL=4`
|
||||
(four FULL 262,144-token slots — the measured card ceiling), batch
|
||||
**8192/1024**, greedy default. Deployed flags are in
|
||||
`scripts/frank-small.env`; the serve script refuses to start if the
|
||||
weights, the projector, the sidecar or the binary are missing.
|
||||
|
||||
Verified at deploy (2026-09-17): `/v1/models` reports `frank-small`; a
|
||||
chat completion returns content plus `reasoning_content`; a tool call
|
||||
returns `tool_calls`; an image request is answered from the projector;
|
||||
an `input_audio` WAV transcribes verbatim. Post-tuning live numbers at
|
||||
max length: pp@254k ~965 t/s, tg@253k 226–302 t/s with ~94% draft
|
||||
acceptance; idle 23,300 MiB of 24,576. The throughput sweep (and why
|
||||
ubatch 2048 OOMs the draft's pp buffer, and why draft depth 4 + poll
|
||||
100 is WORSE at long ctx) is tabled in the runbook.
|
||||
Runbook: `serverAppConfigs-docs/LLM/frank-small-gemma4-12b.md`.
|
||||
Rollback: [[frank-small-qwen38-gsq-rco]].
|
||||
|
||||
### frank-primary — R9V radiance-vLLM Qwen3.8-Flash-Next Q8_0 (Dankserver, ACTIVE)
|
||||
|
||||
The **R9V radiance-vLLM fork** (`drwolfen/radiance-vllm-qwen4exp`, docker-pinned
|
||||
prebuilt image bundle `r9v-qwen38-flash-next-wmma@sha256:2dac17a2…`) serving
|
||||
the **unsloth Qwen3.8-Flash-Next-GGUF Q8_0** (175.30 GiB, 6 GGUF shards)
|
||||
with the blockfp8 MTP4 head (depth 4) and the mmproj on **both** Radeon AI
|
||||
PRO R9700s (gfx1201, BDFs `0000:03:00.0` / `0000:07:00.0`), one stream at
|
||||
**131,072 ctx**, **TP=2 tiered experts** (80-slot LRU expert cache; PLE
|
||||
n-gram table PINNED in host RAM, 54.4 GB), the **Qwen-Sharp v22.4.0
|
||||
template** (via the launcher's `R9V_CHAT_TEMPLATE_FILE` passthrough, fork
|
||||
`30d4a32`), tool parser `qwen3_coder`, reasoning channel `reasoning`. Cut
|
||||
over 2026-09-27 from the uncensored IQ4_XS payload (itself cut over from
|
||||
llama.cpp b11030 the same day) at Frank's direction.
|
||||
|
||||
**The tuned shape lives on-host, not in the repo.** The q8 profile's setup
|
||||
run qualified the `R9V_*` config into
|
||||
`/home/dank/r9v-state/q8xl/setup.json` (`ready: true`); the repo env
|
||||
file `scripts/frank-primary.env` carries ONLY production overrides
|
||||
(`R9V_HOST_BIND=0.0.0.0`, `R9V_HOST_PORT=7770`,
|
||||
`R9V_SERVED_MODEL_NAME=frank-primary`, `R9V_CONTAINER_NAME=frank-primary-r9v`,
|
||||
`R9V_EXPECTED_GPU_BDFS`) plus two q8-only keys: the dev-overlay adapter
|
||||
(`R9V_DEV_QWEN4_EXP_ADAPTER_PY`, fork commit `363d606` — the Q8_0 payload
|
||||
quantizes BOTH fused hyper-connection shards, so the shard-uniformity
|
||||
check needs the in-stream dequant) and `R9V_FULL_MUTABLE_CACHE=0` (the
|
||||
overlay's env.always injects the uncensored 160-slot cache geometry, which
|
||||
the q8 80-slot tiered cache rejects). Never copy tuned values from the
|
||||
state into the repo — it would drift from the qualification.
|
||||
|
||||
**Spec decoding is MTP4 only.** The fork's `speculative_config` is a
|
||||
single dict — one method per engine, NO chaining — so the llama.cpp
|
||||
`draft-mtp,ngram-mod` pattern is NOT expressible here. MTP4 is wired
|
||||
(`R9V_MTP_SPEC_TOKENS=4`, draft TP=2); the PLE n-gram rides along
|
||||
architecturally (`ngram_embedding`); `ngram_gpu` exists as an alternative
|
||||
single method, not a chainable one.
|
||||
|
||||
**Serving = the fork's own launcher through a docker→podman shim.** The R9V
|
||||
tooling is docker-pinned; the shim at
|
||||
`/home/dank/GitPackages/radiance-vllm-qwen4exp/.podman-shim/docker` carries
|
||||
three laws: (1) the bundle is a containerd/OCI docker-save, so docker keys
|
||||
the image by its OCI manifest digest while podman keys by config digest but
|
||||
preserves the manifest digest in `RepoDigests` — the shim answers
|
||||
`image inspect sha256:X` from RepoDigests and rewrites bare `sha256:X` run
|
||||
refs to `repo@sha256:X`; (2) podman inspect answers
|
||||
`HostConfig.LogConfig.Config: null` where docker returns the log-opts map
|
||||
(the fork's doctor crashes on it) — the shim records `run --log-opt` pairs
|
||||
and normalizes inspect output; (3) podman lacks docker's `SecurityOptions`
|
||||
/`ServerVersion` info fields — the shim answers them.
|
||||
|
||||
**Unit and self-healing.** Root systemd unit (`scripts/frank-primary.service`,
|
||||
`User=dank`, `LimitMEMLOCK=infinity`) — the PLE pinned residency alone is
|
||||
54.4 GB, and a root unit grants the pin budget via PID1 (a rootless
|
||||
quadlet would be capped by the user manager's limit).
|
||||
The serve script (`scripts/serve-frank-primary.sh`) gates the setup state
|
||||
(ready + paths), the pinned image (RepoDigests), the PLE byte count
|
||||
(`stat -c %s`), every GGUF shard + the mmproj and the GPU device nodes,
|
||||
clears a stale container, then execs the fork's `scripts/launch.sh` through
|
||||
the shim and **blocks on `docker wait`** — the launcher `--detach`es, so
|
||||
without the wait a crashed engine is a green unit; `docker wait` cannot stop
|
||||
the container child, hence `ExecStop=podman stop -t 120
|
||||
frank-primary-r9v`. The deployer's `deploy_llm_primary` gates state/image/
|
||||
PLE/shards, `podman rm -f`s the container, waits out :7770, relinks/enables/
|
||||
restarts the unit and health-gates with the MainPID check (the false-green
|
||||
stray law).
|
||||
|
||||
Verified at cutover (2026-09-27): q8 setup state `ready: true`, all 20
|
||||
payload artifacts sha256-verified (175.30 GiB); the scratch-port bring-up
|
||||
battery answered vision ("Red" for a 64×64 red PNG), MTP4 mean acceptance
|
||||
length 3.19–5.00, and the PLE pinned proof (`mapping_bytes =
|
||||
mapping_rss_bytes = 54,400,261,120`, fully RSS-resident); wire + tool
|
||||
round-trip + template probe re-verified on :7770 after the deploy. No
|
||||
throughput bench was run (Frank declined); the probe-era 11 tok/s decode
|
||||
was residency-bound (halved expert headroom at double the bytes/expert),
|
||||
not the production number.
|
||||
|
||||
Runbook: `serverAppConfigs-docs/LLM/frank-primary-r9v-radiance-qwen38-q8.md`.
|
||||
Rollback: the uncensored payload — flip `R9V_STATE_FILE` back to
|
||||
`/home/dank/r9v-state/uncensored/setup.json`, drop the two q8-only keys
|
||||
and the template keys, redeploy (state + `restore-config-20260927.env` +
|
||||
payload intact on disk) — see
|
||||
[[frank-primary-r9v-radiance-qwen38-uncensored]].
|
||||
|
||||
## Key Lessons Learned
|
||||
Durable operational lessons from the fleet's history. Full detail in the
|
||||
linked runbooks.
|
||||
|
||||
**PCIe / multi-GPU**
|
||||
|
||||
- **RCCL collectives need PCIe atomics on every participating GPU.** RCCL's
|
||||
kernels carry a hidden hostcall-buffer parameter; ROCr refuses to dispatch
|
||||
them when the device has no atomics, and the failure surfaces as
|
||||
`hipErrorIllegalState` ("the operation cannot be performed in the present
|
||||
state") from `hipLaunchKernel`, not as anything that names atomics. The
|
||||
value is consumed at driver probe time, so writing the AtomicOp Requester
|
||||
bit with `setpci` at runtime changes nothing. Check the *chain*:
|
||||
`lspci -vv -s <gpu>` and every upstream bridge must show
|
||||
`AtomicOpsCap: Routing+` (a `Routing-` bridge is a hardware wall), and the
|
||||
endpoint's Device Control 2 (`setpci -s <gpu> CAP_EXP+28.w`) must have bit
|
||||
6 set. AMD desktop platforms put one card on CPU root ports (atomics work)
|
||||
and the rest behind the chipset (usually not) — that asymmetry is what
|
||||
blocked frank-medium's TP=2. A one-card collective proves nothing: NCCL
|
||||
short-circuits at `world_size=1`.
|
||||
- **That wall was physical, and it was fixed on 2026-09-20 by a board swap.**
|
||||
Both paths that put two cards on CPU root ports were: an M.2→PCIe x4 riser
|
||||
into a *CPU-attached* M.2 slot, or a board whose CPU lanes bifurcate to
|
||||
x8/x8. dankserver took the second: **ASUS ProArt B850-Creator WiFi NEO**,
|
||||
`PCIEX16_1` + `PCIEX16_2` at x8/x8 Gen5 on root ports `00:01.1` / `00:01.3`,
|
||||
every bridge now `Routing+` and **both endpoints `ReqEn+`** — the exact
|
||||
precondition the old test plan named. Bandwidth was never the constraint (a
|
||||
27B TP=2 allreduce at batch 4 is ≈10 MB per token-step, ≈1.4 ms at Gen4 x4
|
||||
and under 0.4 ms at Gen5 x8).
|
||||
**Proven on the same day.** vLLM's R4D one-shot allreduce runs TP=2 here and
|
||||
frank-medium serves on it; llama.cpp's RCCL path was built too
|
||||
(`GGML_HIP_RCCL=ON`) and its tensor mode measured — real collective, slower
|
||||
shape. Note the collective choice matters: TheRock's RCCL builds lack gfx1201
|
||||
device code (`invalid device function`), which is why R4D is the path vLLM
|
||||
uses at all. Evidence matrix and the measured three-arm verdict:
|
||||
[[Dual-R9700-Cooperation]].
|
||||
- **When a GPU collective dies, read the runtime, not the framework.** The
|
||||
message that identified the cause came from `AMD_LOG_LEVEL=3`
|
||||
(`Pcie atomics not enabled, hostcall not supported` → `AQL dispatch
|
||||
failed!`), not from vLLM or RCCL. `AMD_LOG_LEVEL` costs one env var and
|
||||
prints the failing HIP call with its return code.
|
||||
- **The software path that puts both cards on ONE model without a collective:
|
||||
llama.cpp's layer split.** On the old board vLLM could not do it in any of its
|
||||
parallelisms — TP needs an allreduce, PP needs the same p2p primitive, and both
|
||||
died on the card that could not dispatch an RCCL kernel. `--split-mode
|
||||
layer` in llama.cpp needs no
|
||||
collective: layers are distributed in order and only the boundary activation
|
||||
crosses PCIe, once per forward pass. Measured here: 1991 t/s prefill /
|
||||
18.8 tok/s decode layer-split, against 272 / 26.3 tensor (butterfly
|
||||
fallback).
|
||||
This was what `frank-medium` ran until 2026-09-20, when the board swap made
|
||||
collectives legal and radiance TP=2 beat it on every axis (95.8 vs 45.7 t/s
|
||||
decode, 0.21 vs 0.72 s TTFT, 259 vs 72 t/s burst-4) — see
|
||||
[[frank-medium-radiance-mxfp4]]; the llama shape is the rollback target
|
||||
[[frank-medium-llamacpp-dflash2]].
|
||||
- **Layer split is also the cross-vendor answer — and `-ot` is what cannot be
|
||||
combined with it.** On Dankclaw (2026-09-16) a CUDA 3090 + Vulkan iGPU layer
|
||||
split ran a real two-device graph: `-dev CUDA0/Vulkan1 -sm layer -ts 0.15/0.85`
|
||||
held 14,994 MiB on the 3090 and gave 27.2 tok/s with 268.5 t/s prefill at 64k
|
||||
depth (iGPU alone: 25.8 / 231.8). What does NOT work is per-tensor placement
|
||||
across devices: `--override-tensor <group>=<device>` sets
|
||||
`has_tensor_overrides()`, a REQUIRED term of the scheduler's
|
||||
`pipeline_parallel` flag (llama-context.cpp:433 — the only place in the tree
|
||||
that reads it), and that flag is what sets `n_copies` in
|
||||
`ggml_backend_sched_new`; with `n_copies == 1` a pre-allocated weight on a
|
||||
non-primary device has no backend and the graph aborts (`ggml-backend.cpp:941`,
|
||||
reported with the leaf's own op name `(NONE)`). `=CPU` is the only override
|
||||
target that works, because the CPU backend is last in the list. Pure Vulkan
|
||||
changes nothing here — the gate is the override, not the driver mix. See
|
||||
[[x4-pair-experiment]].
|
||||
- **A host↔3090 link at x4 measures 6.72 GB/s, and that is a design
|
||||
constraint, not a detail.** `pcie-link-test.cu` on Dankclaw (2026-09-16,
|
||||
services stopped): pinned H2D 6.72 GB/s, D2H 6.60 GB/s — 85 % of the Gen4 x4
|
||||
ceiling — with async no better than sync and pageable within 1 % of pinned,
|
||||
i.e. the copies are link-limited rather than staging-limited (the usual
|
||||
"pageable costs 2x" rule does not apply here). The card is x16-capable and the
|
||||
Strix Halo port beneath it is x8-capable (`LnkCap: 16GT/s, Width x8` on
|
||||
`00:03.1`), so the x4 lives in the adapter or cable; the link idles at
|
||||
2.5 GT/s x4 and trains up under load. Consequence for any engine that
|
||||
*streams* weights across it: a 3 MiB slab costs 474 us, and a 111 GiB A3B
|
||||
model needs ~1.70 GiB of active experts per token ⇒ **3.7 tok/s** (4.6 with a
|
||||
20 % resident cache), against **20.46 tok/s** for the same model with the
|
||||
experts *computed* in host RAM by llama.cpp (`-ot exps=CPU`, whence the link
|
||||
carries activations only) and 25–26.5 on the iGPU. Streaming over PCIe loses;
|
||||
computing in RAM does not care. FreeToken — expert LRU in host RAM, exactly
|
||||
this streaming shape — also loads only modelopt block-FP8 safetensors for
|
||||
`qwen3_5_moe`, so a Q4_K_M GGUF is unloadable there before bandwidth is even
|
||||
the question. See [[x4-pair-experiment]].
|
||||
|
||||
|
||||
**ROCm / GPU**
|
||||
|
||||
- **Layer split was the deployed shape on both multi-device llama.cpp services,
|
||||
for two different reasons.** On Dankserver it was the *only* way both R9700s
|
||||
served one model (no collective could be dispatched on card #2) until the
|
||||
2026-09-20 board swap; frank-medium now runs radiance TP=2 instead. On a
|
||||
RCCL-less fork build it is also the fast one: tensor parallel falls back to
|
||||
a butterfly allreduce and collapses prefill ~7x (measured 271.8 vs 1991 t/s).
|
||||
Pin the spec-decode drafter to one device — an unpinned draft under tensor
|
||||
mode aborts the KV build, and on a layer split the natural home is the card
|
||||
that ends up holding the target's last layers.
|
||||
- **Bundling a ROCm runtime next to a llama.cpp binary: `cp -r SRC DST/` is a
|
||||
trap.** With a missing `DST`, cp creates `DST` as a copy of `SRC` and drops
|
||||
one directory level. rocBLAS and hipBLASLt look for their device code at a
|
||||
fixed depth (`../lib/rocblas/library`, `../lib/hipblaslt/library/<arch>`),
|
||||
and the flattened copy fails at *request* time, not load time: the engine
|
||||
reads the weights, answers `/health` and dies on the first completion with
|
||||
`Cannot read .../TensileLibrary.dat: No such file or directory for GPU arch
|
||||
: gfx1201`. Declare the destination, then copy the contents.
|
||||
- **mmap must stay on** for the big-PLE models: `--no-mmap` forces the
|
||||
~50 GiB PLE table into anonymous memory → silent cgroup kill, dmesg is the
|
||||
only evidence. mmap pages ARE page cache. Never pin the PLE tensor with
|
||||
`-ot` (23 → 13.4 t/s).
|
||||
- **HIP reads bare env vars** `TEMP`, `HOST`, `GPU_MAX_HEAP_SIZE`. Serve
|
||||
scripts export env files with `set -a`, so a tunable named `TEMP` leaks
|
||||
into HIP (`TEMP=1.0` sent the JIT cache to path `1.0` and segfaulted every
|
||||
load, fixed 2026-08-13 by renaming to `TEMPERATURE`). Never name a tunable
|
||||
these; check with
|
||||
`strings /opt/rocm-*/lib/libamdhip64.so.* | grep -xE "TEMP|HOST|GPU_MAX_HEAP_SIZE"`.
|
||||
- **`-DGGML_CUDA_FA_ALL_QUANTS=ON` is required** for q8_0 KV + flash
|
||||
attention on CUDA builds; without it prefill collapses ~12x (95 vs 1300+
|
||||
t/s, measured 2026-08-17).
|
||||
- **Flash-Next MTP heads do not load on any current loader** — they need full
|
||||
per-layer `blk.<il>.hc_attn_norm.weight` and ship only `blk.48` (verified
|
||||
3x). Don't re-attempt without new loader support.
|
||||
- **QSA flash-attn assert:** `GGML_ASSERT(a-type == GGML_TYPE_F32)` on
|
||||
qwen4exp needs `scripts/patches/qwen4exp-qsa-f32-mask.patch` — but only on
|
||||
the agentionai FAST layer set; the kingjones STRIX set works unpatched.
|
||||
- **GPU budget on Dankclaw is the GTT aperture (~114 GiB), not the
|
||||
rocm-smi "dedicated" figure.** The Radeon 8060S (gfx1151) reports
|
||||
116,732 MiB (`llama-server --list-devices`;
|
||||
`gtt_total=122402963456` in `/sys/class/drm/card1/device/`). The older
|
||||
"64 GiB, SVM-backed" lesson was already stale — the CIRU Orca predecessor
|
||||
held 92.6 GiB of weights on the iGPU. `free`/`dmidecode` show the
|
||||
CPU-side carve-out — intentional, do not change.
|
||||
|
||||
**Load / crash diagnosis**
|
||||
|
||||
- Model load is slow (minutes cold); health returns 503 while loading; do
|
||||
not kill the process. `deploy.sh` waits up to 20 minutes (`LLM_WAIT`). On
|
||||
halogen, the FIRST request after a (re)start can go silent ~45–60 s
|
||||
(health `engine_unresponsive`, 503s) and then self-recovers — do not
|
||||
restart over it.
|
||||
- Crash-loop procedure: (1) `journalctl -u <svc> -n 100` + kernel line via
|
||||
`journalctl -k` (segfault at address 8 in `libamdhip64` = NULL unref in
|
||||
HIP); (2) run serve script, then raw binary — different outcome isolates
|
||||
the env file; (3) `gdb -batch -ex run -ex "bt 25" --args bash
|
||||
scripts/serve-<svc>.sh` — llama-server buffers stdout and loses the log
|
||||
tail on SIGSEGV; trust gdb, not the log; (4) `amdgpu: SVM mapping failed`
|
||||
spam is a symptom, not a root cause.
|
||||
- Journal signatures: client abort logs `W srv stop: cancel task, id_task =
|
||||
N`; natural stop logs only `slot release` with `truncated = 0`. The server
|
||||
never preempts a running generation; excess tasks queue when slots are
|
||||
busy.
|
||||
|
||||
**Host memory / swap**
|
||||
|
||||
- Swap resize ordering: create and `swapon` the replacement BEFORE `swapoff`
|
||||
of the old one, or live pages must land in RAM that `MemAvailable`
|
||||
overstates. Dankclaw swap (64 GiB at `/swap.img`, fstab-driven) protects
|
||||
the co-tenants, not frank-large — pinned weights don't swap, and paging
|
||||
the KV pool is worse than the OOM it prevents.
|
||||
- Host swap state is invisible to `deploy.sh` (no repo file) — check
|
||||
`swapon --show` before reasoning about headroom.
|
||||
|
||||
**Runtime-specific guards**
|
||||
|
||||
- **Kairic Edge** (if ever re-deployed): the pinned CK commit needs
|
||||
`third_party/composable_kernel/include/ck/config.h` generated manually
|
||||
(untracked; `git clean` removes it), and its greedy-argmax fastpath rejects
|
||||
non-greedy requests and reasoning budgets. Re-test chat before production.
|
||||
- **FastMTP sidecar**: build must be commit `4df29be4` + the HauhauCS patch;
|
||||
stock binaries fail the draft load with a tensor-shape mismatch.
|
||||
|
||||
**llama.cpp serving**
|
||||
|
||||
- **MTP is a property of the FILE, not of the flags.** The `-mtp` GGUF build
|
||||
carries the head as `blk.<n>.nextn.*` (`qwen35.nextn_predict_layers = 1`);
|
||||
`--spec-type draft-mtp` then sets `load_mtp`, and the draft context runs on
|
||||
the MAIN model's weights (`shares_model`), so there is no second copy of a
|
||||
12 GiB model in VRAM and no `-md` argument. The plain build has no MTP
|
||||
tensors at all, and pointing `--model-draft` at the file would load it twice.
|
||||
- **What can be chained, and what cannot.** `--spec-type a,b` builds every
|
||||
enabled implementation and chains drafting through them, in a fixed internal
|
||||
order (ngram types first, then the drafters — CLI order is irrelevant). The
|
||||
gate is `ctx_dft`: **draftless** types (the ngram family) are added
|
||||
unconditionally, while `draft-mtp`, `draft-dflash`, `draft-dspark` and
|
||||
`draft-eagle3` all require a draft CONTEXT. So a chain may hold one
|
||||
context-backed drafter plus any number of ngram types — `draft-dflash,ngram-mod`
|
||||
is legal and measured (2.6x the drafter alone on prompt-echo traffic, byte-
|
||||
identical to it on prose at the fleet's 48/64 window) — but
|
||||
`draft-mtp,draft-dflash` dies at load: *"context type MTP requested but model
|
||||
doesn't contain MTP layers"*, because MTP builds its context against whatever
|
||||
model the draft context holds. Two context-backed types cannot share one
|
||||
process, and MTP's head only loads when MTP **is** the draft model.
|
||||
- **A drafter's KV scales with the TARGET's context, and its window policy
|
||||
decides how much that costs.** The draft context is built with
|
||||
`cparams.n_ctx = llama_n_ctx(ctx_tgt)` (`common/speculative.cpp`) — there is
|
||||
no flag to cap it — so a drafter that declares NO sliding window caches the
|
||||
whole window on whatever device `--spec-draft-device` names. Swapping DFlash2
|
||||
(`dflash.attention.sliding_window = 2048`, ~40 MiB of draft KV) for DSpark
|
||||
(no window; 5 layers x 2 x 8 kv heads x 128 dim x 2 B = 20 KiB per position,
|
||||
so **5.0 GiB at 262144**) put ROCm1 at 33.9 GB of 34.2 GB and killed the
|
||||
FIRST draft graph capture with `ROCm error: out of memory` — after the model
|
||||
had loaded and `/health` had answered. Read `sliding_window` out of the
|
||||
drafter's GGUF header before sizing any spec-decode deployment, and prefer a
|
||||
skewed `-ts` over a context cut when the full window matters. The pair was
|
||||
swapped DFlash2 -> DSpark -> DFlash2 in one day (2026-09-16) and the round
|
||||
trip moved the balance point FOUR layers (~4.3 GiB of headroom, 40:25 ->
|
||||
36:29), which is the practical size of the effect. See
|
||||
[[frank-medium-llamacpp-dflash2]].
|
||||
- **`--ubatch-size` has an OPTIMUM on this stack, not a plateau — and its
|
||||
compute buffers land about TWICE as heavy on the card that holds the output
|
||||
layers and the drafter.** Measured on frank-medium 2026-09-16 at the deployed
|
||||
shape (262144 over two slots, a 64k prefill measured at 64k depth,
|
||||
`scripts/bench-batch-ubatch.sh`): prefill runs 1046 tok/s at ubatch 256, 1068
|
||||
at 384, **1193 at 512 (the peak)**, 1135 at 768, 1152 at 1024, 1151 at 1280,
|
||||
1165 at 1536, 1148 at 1792 and 1078 at 2048. Both ends lose — below 512 the
|
||||
SMs are not saturated, and 2048 is 10 % off the peak — and 512 -> 1536 costs
|
||||
ROCm0 1.30 GiB and ROCm1 2.41 GiB to go 2.4 % SLOWER. The logical
|
||||
`--batch-size` does not matter: 2048 through 8192 land within 2 % of each
|
||||
other at a fixed ubatch and use identical VRAM. Decode never moves at all
|
||||
(15.88 tok/s at depth 65536 on every cell). Because the buffers are
|
||||
asymmetric, a split balanced at one batch is unbalanced at another: 512 ->
|
||||
1536 shifted frank-medium's balance by a layer, so 36:29 became 35:30.
|
||||
Re-measure the balance after changing the batch. (8192/2048 failed to allocate
|
||||
its 2020 MiB pp buffer here under the DSpark drafter, when ROCm1 had 0.08 GiB
|
||||
free. That was a CONTIGUITY failure, not a total-bytes one, and no totals
|
||||
model — the sweep's or the seam's — can see it.)
|
||||
- **One `-ts` step is one LAYER, and a hybrid stack's layers are not the same
|
||||
size.** qwen35 is 16 x (3 Gated DeltaNet + 1 Gated Attention), so a step is
|
||||
worth ~0.43 GiB on a DeltaNet boundary and ~1.43 GiB on a full-attention one
|
||||
(0.45 GiB of weights plus ~1.0 GiB of its own f16 KV at 262144) — both
|
||||
measured on frank-medium. Balance by measurement, never by interpolating
|
||||
between two splits; an affine fit reproduces the deployed point and overstates
|
||||
the gaps in between. `tests/frank_medium_unit.bats` encodes that fit as a
|
||||
gate. See [[frank-medium-llamacpp-dflash2]].
|
||||
- **ngram-mod prints no startup banner.** The load log shows the drafter's
|
||||
`adding speculative implementation 'draft-dflash'` and nothing at all for the
|
||||
matcher, so a chain can look unengaged while it is working. Identify it
|
||||
behaviourally (`draft_n` > 0 only where text repeats) or by the per-task
|
||||
`draft acceptance = ... mean len = ...` line. It costs no VRAM (a 64 MiB
|
||||
host-RAM hash table) and its window decides which traffic it can see: too
|
||||
short and it proposes on ordinary prose fragments, which costs verification
|
||||
without acceptance.
|
||||
- **`--jinja` is what `tools` needs.** On 2026-09-16 a subagent died with
|
||||
`500 tools param requires --jinja flag`, from the ik_llama.cpp runtime that
|
||||
then served frank-large: every tool-using agent on that endpoint was broken
|
||||
and the failure read like a model fault. Mainline llama.cpp 0.4.1 defaults
|
||||
`--jinja` on and the serve script pins it; halogen also accepts `tools` and
|
||||
returns `tool_calls`. Check tool support before routing a role at an endpoint.
|
||||
- **Quantized KV on CUDA needs `GGML_CUDA_FA_ALL_QUANTS=ON`** — the same
|
||||
option and the same ~12x prefill collapse described in the ROCm note above.
|
||||
f16 KV needs nothing; frank-small ran f16 for that reason until
|
||||
2026-09-17, when `build-cuda` was rebuilt with the option ON so the
|
||||
endpoint could take q8_0 KV.
|
||||
- **llama-bench separates a LIST with `/` and SWEEP values with `,`.** Both
|
||||
`-dev` and `-ts` follow this: `-dev CUDA0,Vulkan1` is two *single-device*
|
||||
configurations, never one two-device run, and `-ts 0.85,0.15` is two sweep
|
||||
values each becoming `[0.85, 0, …]` / `[0.15, 0, …]`, i.e. device 0 takes
|
||||
everything. Write `-dev A/B -ts x/y`. This artifact produced a wrong fleet
|
||||
conclusion on 2026-09-16 — "CUDA + Vulkan cannot serve one model", with two
|
||||
bogus OOMs that were really "all 111 GiB on the 24 GiB card" — and the bench
|
||||
table's own `dev` column was printing the truth the whole time. Any
|
||||
placement-dependent number needs an independent check: sample the other
|
||||
device's memory (`nvidia-smi`) while the run is loaded.
|
||||
- **A model's own template can REJECT the harness's thinking level.** Qwen3.8's
|
||||
template accepts only `xhigh`, `medium` and `low` for `reasoning_effort` and
|
||||
raises on anything else — HTTP 500, not a clamp — so the harness's levels
|
||||
must be folded onto that set in `models.yml` (`compat.reasoningEffortMap`).
|
||||
A role whose level is unmapped fails on its FIRST request, and the failure
|
||||
names Jinja, not the model: every subagent on `frank-small` died with
|
||||
`Unexpected reasoning effort high` until `high: xhigh` was added on
|
||||
2026-09-16. Halogen accepts the wider set, so the two entries legitimately
|
||||
differ.
|
||||
|
||||
**Ops**
|
||||
|
||||
- Never `pkill -f` a pattern that appears in your own command line — it kills
|
||||
your ssh session. Use `pkill -x <exact-name>`.
|
||||
- Both hosts allow passwordless sudo for the deploy user. Use plain `sudo` in
|
||||
committed scripts (no `echo pw | sudo -S`); `FRANK_SUDO_PW` covers hosts
|
||||
without it.
|
||||
- ComfyUI on Dankclaw runs with **GPU access** (`/dev/kfd` + `/dev/dri` via
|
||||
`AddDevice`, `GroupAdd=keep-groups`, plus the udev rule that makes those nodes
|
||||
world-writable — user-unit spawns carry no supplementary groups). It must NOT
|
||||
pass `--cpu`: the gfx1151 ROCm wheels exist and the unit's `--cpu` era ended
|
||||
2026-09-16, when a stale device-less container was found restarted 9,146
|
||||
times (`RuntimeError: No HIP GPUs are available`).
|
||||
- SearXNG outbound traffic must stay on the Gluetun tunnel; do not move its
|
||||
ports off the gluetun service.
|
||||
- `amd_iommu=off` on Dankclaw (bought 13–16 % prefill for frank-large,
|
||||
2026-09-11) permanently retired the XDNA2 NPU: the driver's SVA/PASID bind
|
||||
needs an enabled IOMMU (`/dev/accel/accel0` fails `err=-19`). Revert path
|
||||
in `serverAppConfigs-docs/LLM/dankclaw-iommu-translation-off.md`.
|
||||
- Rootless Quadlet containers run with `--replace --rm`: `podman logs` covers
|
||||
only the CURRENT instance; history across recreations lives in
|
||||
`journalctl --user -u <name>.service`. Read rootless storage WITHOUT sudo
|
||||
(per-user storage — `sudo podman logs` finds nothing).
|
||||
- **A dead rootless podman pause process breaks ALL container management on the
|
||||
host, and `podman system migrate` panics in 5.7.0.** Symptoms: every `podman`
|
||||
command fails with `invalid internal status … could not find any running
|
||||
process`, every container start fails binding podman's bridge addresses
|
||||
(`Error starting server failed to bind udp listener on 10.89.x.1:53: Cannot
|
||||
assign requested address`), and every `Restart=always` quadlet loops. Container
|
||||
storage is intact; what is gone is `/run/user/<uid>/libpod/tmp/alive` (a UUID
|
||||
token, not a pid) and the rootless netns holding the bridges — deleting the
|
||||
stale token is not enough, and `podman system migrate` segfaults in
|
||||
`teardownNetworkBackend`. Repair: **reboot** (`/run/user` clears and all 28
|
||||
quadlets return on their own — check `Linger=yes` and `WantedBy=default.target`
|
||||
first). Seen 2026-09-16, hours into heavy GPU work on the same host — but the
|
||||
trigger is NOT established (`journalctl -k` recorded no OOM kill; the kernel
|
||||
had no other word on it). Treat the pause process as host state that can die
|
||||
for its own reasons, and reach for the reboot when it does.
|
||||
- **A container cannot reach a `Network=host` service at the host's LAN IP.**
|
||||
netavark DNATs *published* ports only: from a bridge container
|
||||
`192.168.0.143:<published>` answers while `192.168.0.143:<host bind>` and the
|
||||
gateway `10.89.1.1` are both refused. Use `host.containers.internal`
|
||||
(169.254.1.2). This is what 502'd `hermes.franksdank.cloud` until the Caddyfile
|
||||
upstream was changed on 2026-09-16 (its vhost comment had claimed the LAN IP
|
||||
was open from that bridge — it never was).
|
||||
- **A `*.gguf` filter misses mradermacher's payloads: large quants ship as
|
||||
`.partNofM` fragments.** `mradermacher/Qwen3.8-Flash-Next-Uncensored-i1-GGUF`
|
||||
carries Q4_K_M (37.00 + 37.00 + 36.97 = 110.97 GiB) and Q4_K_S (35.00 + 35.00
|
||||
+ 34.09 = 104.09 GiB) as parts, while the smaller rungs (Q2_K, Q3_K_*, IQ3_*,
|
||||
IQ4_XS) are single files. Fetch the parts, verify each against the Hub's LFS
|
||||
sha256, concatenate in numeric order — that is how the deployed 111 GiB file
|
||||
was assembled. Listing the Hub tree and filtering `endswith(".gguf")` makes
|
||||
Q4_K_S look like it does not exist.
|
||||
- **A fetch script must `-f` and verify, or it reports success on an access
|
||||
error.** `curl -o FILE URL` exits 0 on HTTP 401/403 and writes the error body
|
||||
as the "artifact": a gated repo yielded three 153-byte shards with identical
|
||||
sha256 values and a cheerful `DOWNLOAD_DONE` in 20 seconds for 104 GiB. Use
|
||||
`-f`/`--fail-with-body`, compare each file's byte count to the Hub's declared
|
||||
size, and check the LFS sha256 — `scripts/download-frank-medium-gguf.sh`
|
||||
already does all three; ad-hoc fetches tend not to. Gated repos (e.g.
|
||||
`orcarouter/Qwen3.8-Flash-Next-Uncensored-GGUF`, same 104.09 GiB Q4_K_S)
|
||||
return 403 even with the host's HF token until that account accepts the gate.
|
||||
|
||||
## Runbook Index
|
||||
|
||||
Active:
|
||||
|
||||
| Service | Runbook |
|
||||
|---------|---------|
|
||||
| frank-large (gufo Qwen3.8-27B UD-Q8_K_XL, Dankclaw) | `serverAppConfigs-docs/LLM/frank-large-gufo-qwen38-27b.md` |
|
||||
| frank-small (llama.cpp Gemma4-12B QAT, Dankclaw) | `serverAppConfigs-docs/LLM/frank-small-gemma4-12b.md` |
|
||||
| frank-primary (R9V radiance-vLLM Qwen3.8-Flash-Next Q8_0, Dankserver) | `serverAppConfigs-docs/LLM/frank-primary-r9v-radiance-qwen38-q8.md` |
|
||||
| Dankclaw kernel posture (`amd_iommu=off`) | `serverAppConfigs-docs/LLM/dankclaw-iommu-translation-off.md` |
|
||||
| Deployment flow + incidents | `serverAppConfigs-docs/LLM/deployment.md` |
|
||||
|
||||
Retired (rollback targets and history):
|
||||
|
||||
| Runtime | Host | Window | Runbook |
|
||||
|---------|------|--------|---------|
|
||||
| llama.cpp b11030 Qwen3.8-27B UD-Q8_K_XL + DFlash2 (`frank-large`) | Dankclaw | 2026-09-21 → 09-28 | `frank-large-unsloth-qwen38-27b-q8xl.md` |
|
||||
| R9V radiance-vLLM Qwen3.8-Flash-Next-Uncensored IQ4_XS (`frank-primary`) | Dankserver | 2026-09-27 (same-day swap to Q8_0) | `frank-primary-r9v-radiance-qwen38-uncensored.md` |
|
||||
| llama.cpp b11030 Qwen3.8-Flash-Next Q8_0 + shared MTP (`frank-primary`) | Dankserver | 2026-09-26 → 09-27 | `frank-primary-llamacpp-qwen38-flash-next-q8.md` |
|
||||
| ik_llama.cpp Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M (`frank-large`) | Dankclaw | 2026-09-15 → 09-16 | `frank-large-ik-uncensored.md` |
|
||||
| vllm-radiance TP=2 MXFP4 + DFlash2 (`frank-medium`) | Dankserver | 2026-09-20 → 09-26 | `frank-medium-radiance-mxfp4.md` |
|
||||
| Lucebox DS4.1 DeepSeek-V4.1-Flash ROCmFP23 (`frank-primary`) | Dankserver | 2026-09-26 (same-day swap) | `frank-primary-lucebox-ds4.md` |
|
||||
| kingjones STRIX Qwen3.8-Flash-Next ROCmFP4 | Dankclaw | 2026-08-30 → 09-11 | `kingjones-strix-rocmfp4.md` |
|
||||
| halogen-flash-server Qwen3.8-Flash-Next w4b (`frank-large`) | Dankclaw | 2026-09-16 → 09-19 | `halogen-flash-server.md` |
|
||||
| CIRU v4.4.1 Orca Qwen3.8-Flash (`frank-large`) | Dankclaw | 2026-09-19 → 09-21 | `frank-large-ciru-orca.md` |
|
||||
| unsloth llama.cpp b11030 Qwen3.8-Flash-Next UD-IQ4_XS (`frank-large`) | Dankclaw | 2026-09-21 (same-day swap) | `frank-large-unsloth-qwen38-flash-next.md` |
|
||||
| Kairic Edge Qwen3.8-27B IU4 W4A4 (`frank-large`) | Dankclaw | 2026-09-21 (same-day swap) | `frank-large-kairic-edge-qwen38-27b.md` |
|
||||
| ISTA-DASLab Qwen3.8-27B-GSQ-RCO IQ3_S (`frank-small`) | Dankclaw | 2026-09-16 → 09-17 | `frank-small-qwen38-gsq-rco.md` |
|
||||
| vLLM Gemma4-12B w4a16 (`frank-small`) | Dankclaw | 2026-09-19 (same-day revert) | `frank-small-vllm-gemma4.md` |
|
||||
| agentionai ROCmFP4-FAST imatrix | Dankclaw | 2026-08-29 → 08-30 | `agentionai-fast-imatrix.md` |
|
||||
| eemin Qwen3.8-27B ROCmI4 W4A4 MTP | Dankclaw | 2026-08-18 → 08-25 | `eemin-rocmi4-mtp.md` |
|
||||
| Kairic Edge IU4 | Dankclaw | 2026-08-22 (same-day revert) | `kairic-edge.md` |
|
||||
| CIRU ActiveFPX v3 / PromptForge | Dankclaw | through 2026-08-22 | `ciru-activefpx-promptforge.md` |
|
||||
| DFlash2 Vulkan | Dankclaw | 2026-08-21 (same-day revert) | `dflash2-large-vulkan.md` |
|
||||
| FastFlowLM NPU (`frank-npu`) | Dankclaw | twice, retired 2026-09-11 | `frank-npu.md`, `FastFlowLM NPU Inference.md` |
|
||||
| FreeToken (official + Huihui FP8, `frank-small`) | Dankserver | 2026-09-02 → 09-15 | `freetoken-frank-small.md` |
|
||||
| FastMTP patched llama.cpp | Dankserver | retired 2026-08-18 | `qwen3.8-spec-matrix.md` |
|
||||
| ik_llama.cpp Ornith CRACK | Dankserver | retired 2026-09-02 | `ikllama-ornith-dankserver.md` |
|
||||
| llama.cpp layer split + DFlash2 (`frank-medium`) | Dankserver | 2026-09-16 → 09-20 | `frank-medium-llamacpp-dflash2.md` |
|
||||
|
||||
Retired runbooks are under `serverAppConfigs-docs/LLM/`. Retired scripts live
|
||||
in `old/` and `.AGENTTRASH/`.
|
||||
|
||||
## Why
|
||||
|
||||
Direct edits on the remote drift from version control, and future agents
|
||||
won't know what changed. Git is the single source of truth. The two workloads
|
||||
share two hosts, one Caddy, and one Gitea; one repo keeps them in one
|
||||
history.
|
||||
|
||||
## Summary
|
||||
|
||||
| Context | Action |
|
||||
|---------|--------|
|
||||
| Editing a compose/Caddyfile/serve script | **Through git** — fetch, commit, edit, test, push, pull, deploy. Never edit on the remote. |
|
||||
| Config exists only on remote | **Pull it into the repo first** — commit the fetched version (redacted) before editing. |
|
||||
| Unstaged local changes | **Commit or discard** — check `git status` before starting work; ask if unsure. |
|
||||
| Deploying a committed change | **`scripts/deploy.sh <target>`** — pushes, pulls ff-only, plans, dispatches, health-checks. |
|
||||
| Container without a Quadlet unit | **Give it one** — boot-persistent, self-healing `.container` via git. |
|
||||
| Benchmarking | **Stop and mask the service first**, bench, unmask, restart, verify health. |
|
||||
| Deleting a file | **Move it to trash** — `.AGENTTRASH/` (in-repo) or `~/.AGENTTRASH/YYYYMMDD-topic`. Never `rm`. |
|
||||
| Tests exist | **Run them** — verify tests pass after every change. Do not commit failing tests. |
|
||||
| Real secrets | **Never commit** — host `.env` files; `.env.example` holds placeholders only. |
|
||||
### Commit Guidelines
|
||||
- Use descriptive commit messages: `git commit -m "Add AWQ DFlash script"`
|
||||
- Commit before deploying: `git add -A && git commit -m "Update for deployment"`
|
||||
- Push to remote: `git push origin <branch>`
|
||||
|
||||
### Deployment Flow
|
||||
1. Make changes locally
|
||||
2. Run tests: `bash tests/test_validator.sh dankserver`
|
||||
3. Commit: `git add -A && git commit -m "Description"`
|
||||
4. Deploy: `bash deploy.sh dankserver`
|
||||
5. Verify: `bash scripts/validate-vllm.sh -t dankserver`
|
||||
|
||||
## Remote Servers
|
||||
|
||||
### dankserver
|
||||
- SSH: `dank@dankserver`
|
||||
- Scripts: `~/vllm/`
|
||||
- Service: `vllm-awq-qwen36.service`
|
||||
- Model: frank-default-small (AWQ 4bit)
|
||||
- Port: 34401
|
||||
- Status: Disabled (pending bug fixes)
|
||||
|
||||
### dankclaw
|
||||
- SSH: `dank@dankclaw`
|
||||
- Scripts: `~/vllm/`
|
||||
- Service: `vllm-thinkingcap.service`
|
||||
- Model: frank-default-large (FP8)
|
||||
- Port: 3090
|
||||
- Status: Running (you are the deployed inference)
|
||||
|
||||
## Troubleshooting
|
||||
- Service failed: `systemctl status <service>`
|
||||
- Container stuck: `docker stop <name> && docker rm <name>`
|
||||
- OOM error: Reduce `--max-model-len` or increase `--cpu-offload-gb`
|
||||
- Prefix caching conflict: Remove `--enable-prefix-caching` from script
|
||||
|
||||
@@ -1,63 +1,105 @@
|
||||
# Server App Configs
|
||||
# vLLM Scripts
|
||||
|
||||
The single repository for the two homelab servers: the Podman container
|
||||
stacks **and** the LLM inference fleet. One repo, one checkout per host,
|
||||
one deploy script.
|
||||
|
||||
> 2026-08-22: the former `vllmScripts` repository (LLM fleet) is merged
|
||||
> into this repo with its full git history. See AGENTS.md for the merge
|
||||
> note.
|
||||
Scripts for running large language models via vLLM on two local machines.
|
||||
|
||||
## Targets
|
||||
|
||||
| Host | User | IP | Podman stack | LLM service | LLM port |
|
||||
|------|------|----|--------------|-------------|----------|
|
||||
| dankserver | dank | 192.168.0.235 | media and library | frank-medium (ROCmFPX llama.cpp, Qwen3.8-27B AGENT) | 7770 |
|
||||
| dankclaw | dank | 192.168.0.143 | AI, search, development, observability | frank-large (gufo, Qwen3.8-27B UD-Q8_K_XL) | 7770 |
|
||||
| Host | User | Codename | Architecture | Service Status |
|
||||
|------|------|----------|-------------|----------------|
|
||||
| dankserver | dank | DankServer | NVIDIA GPU + System RAM | `vllm-awq-qwen36.service` (failed) |
|
||||
| dankclaw | dank | DankClaw | AMD Strix Halo (Shared Memory) | `vllm-thinkingcap.service` (failed) |
|
||||
|
||||
## Active Services
|
||||
## File Locations
|
||||
|
||||
| Service | Host | Hostname | Port |
|
||||
|---------|------|----------|------|
|
||||
| RomM | dankserver | romm.dankserver.local | 8085 |
|
||||
| Copyparty | dankserver | files.dankserver.local | 3923 |
|
||||
| Calibre-Web | dankserver | books.dankserver.local | 8083 |
|
||||
| frank-medium | dankserver | — (bare endpoint) | 7770 |
|
||||
| SearXNG | dankclaw | search.dankclaw.local | 8081 |
|
||||
| ComfyUI | dankclaw | comfyui.dankclaw.local | 8188 |
|
||||
| OpenWebUI | dankclaw | chat.dankclaw.local | 3000 |
|
||||
| Hermes | dankclaw | hermes.dankclaw.local | 8090 |
|
||||
| Gitea | dankclaw | gitea.dankclaw.local | 3001 |
|
||||
| Grafana | dankclaw | grafana.dankclaw.local | 3002 |
|
||||
| Prometheus | dankclaw | prometheus.dankclaw.local | 9090 |
|
||||
| frank-large | dankclaw | — (bare endpoint) | 7770 |
|
||||
### Local (vllmScripts/)
|
||||
- `scripts/` — Active inference scripts
|
||||
- `tests/` — Test suite
|
||||
- `docs/` — Knowledge base
|
||||
- `failedAttempts/` — Archived scripts
|
||||
|
||||
### Remote (dankserver)
|
||||
- `~/vllm/` — Deployed inference scripts
|
||||
- `~/vllm/failedAttempts/` — Archived scripts
|
||||
- `~/vLLM/` — Legacy project directory (vllm-qwen36.sh)
|
||||
|
||||
### Remote (dankclaw)
|
||||
- `~/vllm/` — vLLM source code (not inference scripts)
|
||||
- `~/vLLM/` — Inference scripts (if deployed)
|
||||
|
||||
## Models
|
||||
|
||||
| Model | Target | Format |
|
||||
|-------|--------|--------|
|
||||
| Qwen3.6 35B A3B FP8 | dankserver | FP8 |
|
||||
| Qwen3.6 27B FP8 | dankclaw | FP8 |
|
||||
| Qwen3.6 35B A3B AWQ 4bit | dankserver | AWQ 4bit |
|
||||
| Qwen3.6 35B A3B DFlash | dankserver | Draft model |
|
||||
|
||||
## Structure
|
||||
|
||||
- `dankserver/` — compose.yaml + Caddyfile + .env.example for the Dankserver stack
|
||||
- `dankclaw/` — compose.yaml + Caddyfile + .env.example for the Dankclaw stack
|
||||
- `shared/` — homepage, prometheus config, deploy helpers
|
||||
- `scripts/` — `deploy.sh` and `deploy-remote.sh` (service-scoped deployment), the planner, LLM serve scripts, systemd units, env tunables, and bench harnesses
|
||||
- `old/` — retired PP/NPU-era LLM scripts (reference only)
|
||||
- `serverAppConfigs-docs/` — project knowledge base (Obsidian vault, wikilinks). `serverAppConfigs-docs/LLM/` is the LLM fleet
|
||||
- `tests/` — bats test suite (see [[Testing Policy]])
|
||||
|
||||
## Deployment
|
||||
|
||||
See [serverAppConfigs-docs/Architecture/Deployment Flow.md](serverAppConfigs-docs/Architecture/Deployment%20Flow.md)
|
||||
and [serverAppConfigs-docs/LLM/deployment.md](serverAppConfigs-docs/LLM/deployment.md). Run
|
||||
`./scripts/deploy.sh <dankclaw|dankserver>` after committing changes. It
|
||||
plans and deploys only the changed services. Use `--service NAME` for an
|
||||
explicit single-service retry; inference is restarted only when its own files
|
||||
(or the shared chat template) changed.
|
||||
- `docs/` — Project knowledge base with research notes on vLLM configuration, hardware specifics, and deployment notes
|
||||
- `scripts/` — Executable vLLM launch scripts
|
||||
- `vllm-qwen36.sh` — FP8 inference script for dankserver
|
||||
- `vllm-awq-qwen36-dflash.sh` — AWQ 4bit with DFlash speculative decoding (docker)
|
||||
- `vllm-awq-qwen36-dflash-ampere.sh` — AWQ 4bit with DFlash speculative decoding (Ampere-optimized, systemd)
|
||||
|
||||
## Knowledge Base
|
||||
|
||||
The `serverAppConfigs-docs/` directory is an Obsidian vault. Start at
|
||||
[serverAppConfigs-docs/index.md](serverAppConfigs-docs/index.md). The LLM fleet knowledge base lives at
|
||||
[serverAppConfigs-docs/LLM/index.md](serverAppConfigs-docs/LLM/index.md).
|
||||
See [[docs/README.md]] for the project knowledge base index.
|
||||
|
||||
## Testing
|
||||
## Lessons Learned
|
||||
|
||||
Run `./tests/run.sh` before every push. See
|
||||
[serverAppConfigs-docs/Architecture/Testing Policy.md](serverAppConfigs-docs/Architecture/Testing%20Policy.md).
|
||||
### AWQ + DFlash on Ampere Hardware (2026-08-08)
|
||||
|
||||
**What worked:**
|
||||
- `cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit` loads successfully on Ampere GPUs
|
||||
- AWQ Marlin kernel engaged automatically for weight conversion
|
||||
- DFlash draft model `z-lab/Qwen3.6-35B-A3B-DFlash` detected as EAGLE model
|
||||
- torch.compile with cudagraphs works with `--max-num-batched-tokens 1536`
|
||||
|
||||
**What didn't work:**
|
||||
- `--enable-prefix-caching` conflicts with Mamba cache "align" mode — causes `AssertionError` in `unify_kv_cache_spec_page_size`
|
||||
- `--max-model-len 262144` causes OOM during AWQ→Marlin weight conversion on 24GB GPU
|
||||
- Nightly vLLM `--speculative-config` requires `num_speculative_tokens` when model is provided
|
||||
- `--speculative-model` flag removed in nightly; use `--speculative-config` JSON instead
|
||||
|
||||
**Working configuration:**
|
||||
- `--max-model-len 32768` (32k context)
|
||||
- `--cpu-offload-gb 40` with `--cpu-offload-params "experts"`
|
||||
- `--gpu-memory-utilization 0.95`
|
||||
- `--performance-mode throughput`
|
||||
- `--enable-chunked-prefill`
|
||||
- `--max-num-batched-tokens 1536`
|
||||
- Thread binding: `OMP_NUM_THREADS=8`, `VLLM_CPU_OMP_THREADS_BIND=8`
|
||||
|
||||
**Performance observations (no speculative decoding):**
|
||||
| Streams | Prompt Speed | Completion Speed | Total Time |
|
||||
|---------|-------------|-----------------|------------|
|
||||
| 1 | 4.27 tok/s | 21.42 tok/s | 11,951ms |
|
||||
| 2 | 2.44 tok/s | 12.25 tok/s | 20,891ms |
|
||||
| 3 | 1.35 tok/s | 6.76 tok/s | 37,849ms |
|
||||
| 4 | 1.43 tok/s | 7.19 tok/s | 35,606ms |
|
||||
|
||||
Note: Speeds decrease with more streams due to KV cache contention.
|
||||
|
||||
**Still needs work:**
|
||||
- DFlash speculative decoding integration (KV cache assertion bug in nightly)
|
||||
- Full context length support (32k is temporary workaround for OOM)
|
||||
|
||||
## Validation
|
||||
|
||||
Use `scripts/validate-vllm.sh` to test inference servers:
|
||||
|
||||
```bash
|
||||
# Test dankserver (default)
|
||||
bash scripts/validate-vllm.sh -t dankserver
|
||||
|
||||
# Test dankclaw
|
||||
bash scripts/validate-vllm.sh -t dankclaw
|
||||
|
||||
# Custom host/port
|
||||
bash scripts/validate-vllm.sh -H 192.168.1.100 -P 8001 -M my-model
|
||||
|
||||
# Run tests
|
||||
bash tests/test_validator.sh dankserver
|
||||
```
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
# Dankclaw stack secrets and configuration.
|
||||
# Copy to .env and fill real values. Never commit real secrets.
|
||||
|
||||
# SearXNG
|
||||
SEARXNG_SECRET=changeme
|
||||
SEARXNG_BASE_URL=http://search.franksdank.cloud/
|
||||
# Kagi search API key (https://help.kagi.com/kagi/api/). Powers the paid
|
||||
# `kagi` engine in shared/searxng/settings.yml; deploy-remote.sh bakes it into
|
||||
# the live settings.yml via scripts/searxng-render-settings.py. The quadlet
|
||||
# does NOT pass it into the container environment.
|
||||
KAGI_API_KEY=changeme
|
||||
|
||||
# Gluetun VPN (see https://github.com/qdm12/gluetun-wiki)
|
||||
# Single-peer WireGuard model. Fill from your provider's WireGuard config.
|
||||
VPN_SERVICE_PROVIDER=protonvpn
|
||||
VPN_TYPE=wireguard
|
||||
WIREGUARD_PRIVATE_KEY=changeme
|
||||
WIREGUARD_ADDRESSES=0.0.0.0/32
|
||||
WIREGUARD_PUBLIC_KEY=changeme
|
||||
WIREGUARD_ENDPOINT_IP=0.0.0.0
|
||||
WIREGUARD_ENDPOINT_PORT=51820
|
||||
WIREGUARD_ALLOWED_IPS=0.0.0.0/0,::/0
|
||||
WIREGUARD_PERSISTENT_KEEPALIVE_INTERVAL=25s
|
||||
|
||||
# Gitea database
|
||||
GITEA_DB_PASSWORD=changeme
|
||||
|
||||
# Grafana OAuth2 client secret (authentik OAuth2Provider "grafana", pk 9).
|
||||
# Created via ak shell on authentik-server; the live .env must hold the
|
||||
# provider's generated client_secret verbatim. Rotation = rotate the provider
|
||||
# secret in authentik, restart grafana after updating the live .env value.
|
||||
GRAFANA_OAUTH_CLIENT_SECRET=changeme
|
||||
|
||||
# Caddy (the homelab's single reverse proxy, on this host)
|
||||
# bcrypt hash (caddy hash-password). deploy.sh renders it into
|
||||
# /home/dank/Containers/caddy/Caddyfile, which the container mounts.
|
||||
# CADDY_BASICAUTH_HASH: the shared family password, now used only by the
|
||||
# pihole vhost (grafana moved behind the Authentik guard chain 2026-09-01).
|
||||
# Username: dankuser.
|
||||
CADDY_BASICAUTH_HASH=changeme
|
||||
# (The homepage's per-user hashes were retired: the homepage and its /admin/
|
||||
# page are now behind Authentik forward_auth, not basicauth.)
|
||||
|
||||
# Hermes -> Signal (signal-cli daemon on the host, port 8080)
|
||||
# E.164 number of the linked Signal account (your primary number, for Note-to-Self).
|
||||
SIGNAL_ACCOUNT=+15551234567
|
||||
# Comma-separated E.164 numbers allowed to message the agent.
|
||||
SIGNAL_ALLOWED_USERS=+15551234567
|
||||
|
||||
# Hermes web UI (Hermex server) login password. Required: the webui binds to
|
||||
# loopback and is reached only through the SSH tunnel, but Hermex still signs
|
||||
# in with this password. Set a strong value in the live .env.
|
||||
HERMES_WEBUI_PASSWORD=changeme
|
||||
|
||||
# Porkbun API (franksdank.cloud + franksdank.casa zones). Caddy uses these
|
||||
# for DNS-01 Let's Encrypt wildcard cert issuance. Porkbun API keys are
|
||||
# account-wide; rotate them here rather than in the dashboard.
|
||||
# PORKBUN_API_SECRET_KEY is the name the caddy-dns/porkbun module expects.
|
||||
PORKBUN_API_KEY=changeme
|
||||
PORKBUN_API_SECRET_KEY=changeme
|
||||
|
||||
# Authentik (SSO / OIDC provider)
|
||||
# AUTHENTIK_PG_PASS: the PostgreSQL password for authentik-postgres.
|
||||
# Generate: openssl rand -base64 32
|
||||
AUTHENTIK_PG_PASS=changeme
|
||||
# AUTHENTIK_SECRET_KEY: the server's secret key (64+ chars).
|
||||
# Generate: openssl rand -base64 48
|
||||
AUTHENTIK_SECRET_KEY=changeme
|
||||
|
||||
# Marreta (content-cleaning proxy, behind the Authentik guard chain).
|
||||
# deploy-remote.sh seeds /home/dank/Containers/marreta/env from these three
|
||||
# values (only if that file does not already exist) and the marreta quadlet
|
||||
# maps them to the app's APP_KEY / ADMIN_EMAIL / ADMIN_PASSWORD.
|
||||
# MARRETA_APP_KEY: Laravel encryption key. Generate:
|
||||
# openssl rand -base64 32
|
||||
# Changing it after first boot invalidates encrypted session/cookie data.
|
||||
MARRETA_APP_KEY=changeme
|
||||
# Admin panel login (marreta.franksdank.cloud/admin). The app creates this
|
||||
# account on first boot from these values.
|
||||
MARRETA_ADMIN_EMAIL=admin@franksdank.cloud
|
||||
MARRETA_ADMIN_PASSWORD=changeme
|
||||
@@ -1,424 +0,0 @@
|
||||
# Homelab Caddyfile — the single Caddy for both hosts.
|
||||
#
|
||||
# Caddy runs as a container on dankclaw (192.168.0.143, this repo's
|
||||
# compose.yaml service 'caddy') and fronts every service on both hosts:
|
||||
# - dankclaw services: by container name (compose network) or by the
|
||||
# published LAN port (quadlet services on isolated networks)
|
||||
# - dankserver services (192.168.0.235): by published LAN port
|
||||
#
|
||||
# Naming: flat franksdank.cloud scheme (see serverAppConfigs-docs/Infrastructure/LAN-DNS.md).
|
||||
# DNS for these names is served by the Pi-hole quadlet on this host
|
||||
# (dankclaw/pihole/addn-hosts — every name resolves to 192.168.0.143)
|
||||
# and handed to LAN clients by the SURFboard G36 router.
|
||||
#
|
||||
# Authentication chain (2026-08-29): the guarded vhosts (homepage, SearXNG,
|
||||
# Prometheus, RomM, Copyparty) proxy EVERYTHING through the Anubis
|
||||
# bot-challenge proxy first, then the whole request is proxied THROUGH
|
||||
# Authentik (2026.8 proxy-provider mode: this build serves no forward_auth
|
||||
# endpoint; the authentication flow runs inline — unauthenticated clients get
|
||||
# the login UI served in place, and once a session exists Authentik proxies
|
||||
# the request on to the app via the provider's internal_host). Chain: client →
|
||||
# Caddy (TLS, public listener) → Anubis :8923 (verify: DENY known AI scrapers,
|
||||
# CHALLENGE everyone else) → Caddy's internal listener http://:8016 (auth
|
||||
# decision; Host headers survive every hop) → authenticated traffic is
|
||||
# forwarded back to http://:8017 and routed straight to the app.
|
||||
# Apps with their own auth (Gitea, OpenWebUI, Jellyfin, *arr, Pi-hole) and the
|
||||
# Authentik IdP itself (auth.franksdank.cloud — its login flow must never sit
|
||||
# behind a challenge) are exempt by construction: their vhosts never route
|
||||
# through Anubis. ComfyUI left this exempt set on 2026-08-31 and Grafana left
|
||||
# it on 2026-09-01 (OAuth2 relying-party mode against authentik): both now sit
|
||||
# behind the guard chain; Grafana authenticates users itself via the OAuth2
|
||||
# code flow, so no identity header is forwarded to it.
|
||||
# The pihole vhost keeps its basicauth edge; the hash placeholder below is
|
||||
# rendered by deploy.sh from CADDY_BASICAUTH_HASH (host .env, git-ignored) into
|
||||
# /home/dank/Containers/caddy/Caddyfile, which the container mounts.
|
||||
#
|
||||
# TLS: real Let's Encrypt certs. Site addresses carry no http:// prefix,
|
||||
# so Caddy serves HTTPS (and auto-redirects HTTP). Certs are issued via
|
||||
# dns-01 through the Porkbun module built into the franksdank/caddy-porkbun
|
||||
# image (caddy/Containerfile). That needs no inbound ports and works from the
|
||||
# LAN, where Pi-hole resolves these names to 192.168.0.143.
|
||||
|
||||
{
|
||||
# Admin stays ON: it binds to 127.0.0.1:2019 inside the container and the
|
||||
# port is not published, so nothing outside the container can reach it.
|
||||
# It powers the config file watcher, so a rendered-config change
|
||||
# hot-reloads without a container restart. 'admin off' once left a broken
|
||||
# basicauth hash sitting on disk unapplied for hours (2026-08-22).
|
||||
email admin@franksdank.cloud
|
||||
|
||||
# Solve ACME challenges for every site via Porkbun DNS (no port 80/443
|
||||
# reachability needed). Credentials come from the container environment
|
||||
# (compose.yaml, from the host .env — never committed).
|
||||
acme_dns porkbun {
|
||||
api_key {env.PORKBUN_API_KEY}
|
||||
api_secret_key {env.PORKBUN_API_SECRET_KEY}
|
||||
}
|
||||
}
|
||||
|
||||
# --- Bare-IP access (health checks, convenience) ---
|
||||
:80 {
|
||||
redir https://www.franksdank.cloud/ 308
|
||||
}
|
||||
|
||||
# --- Homepage (Anubis → Authentik SSO) ---
|
||||
# www holds the homepage; the naked domain redirects onto it at the TLS edge.
|
||||
# A redirect needs no challenge, so unauthenticated naked traffic never enters
|
||||
# the auth chain. The proxy provider's external_host must match the host that
|
||||
# actually carries traffic (https://www.franksdank.cloud) or Authentik matches
|
||||
# zero providers and answers its error page instead.
|
||||
franksdank.cloud {
|
||||
redir https://www.franksdank.cloud/ 308
|
||||
}
|
||||
|
||||
www.franksdank.cloud {
|
||||
reverse_proxy anubis:8923 {
|
||||
header_up X-Real-Ip {remote_host}
|
||||
}
|
||||
}
|
||||
|
||||
# --- Legacy name: the old homepage name redirects straight onto www ---
|
||||
dankclaw.franksdank.cloud {
|
||||
redir https://www.franksdank.cloud/ 308
|
||||
}
|
||||
|
||||
# --- Legacy name: the media host has no Caddy of its own anymore ---
|
||||
dankserver.franksdank.cloud {
|
||||
redir https://www.franksdank.cloud/ 308
|
||||
}
|
||||
|
||||
# --- Authentik (SSO / OIDC provider) ---
|
||||
# Exempt by construction: the IdP login flow must be reachable without a
|
||||
# challenge, so this vhost never routes through Anubis.
|
||||
auth.franksdank.cloud {
|
||||
reverse_proxy authentik-server:9000
|
||||
}
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# Services on this host (dankclaw, 192.168.0.143)
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
# --- SearXNG (quadlet; behind the Gluetun tunnel, published 8081) ---
|
||||
search.franksdank.cloud {
|
||||
reverse_proxy anubis:8923 {
|
||||
header_up X-Real-Ip {remote_host}
|
||||
}
|
||||
}
|
||||
|
||||
# --- OpenWebUI (quadlet on slirp4netns; published 3005) ---
|
||||
ai.franksdank.cloud {
|
||||
reverse_proxy 192.168.0.143:3005
|
||||
}
|
||||
|
||||
# --- Gitea (quadlet; published 3000) ---
|
||||
# Exempt by construction: Gitea has its own login, and git smart-HTTP
|
||||
# clients (git push/fetch over HTTPS) cannot solve JS challenges. The
|
||||
# policy would need a bypass for /info/refs + /git-*-pack paths if this
|
||||
# ever moves behind Anubis.
|
||||
gitea.franksdank.cloud {
|
||||
reverse_proxy 192.168.0.143:3000
|
||||
}
|
||||
|
||||
# --- Grafana (guarded: Anubis → Authentik outpost gate; login inside Grafana) ---
|
||||
# Auth model = OAuth2 relying party against authentik (OAuth2Provider "grafana",
|
||||
# confidential client). The guard chain gates unauthenticated traffic; the
|
||||
# outpost carries no provider token in this deployment, so identity headers
|
||||
# never reach Grafana — RP mode needs none and works regardless of topology.
|
||||
# The basicauth edge this replaces (CADDY_BASICAUTH_HASH) is retired.
|
||||
grafana.franksdank.cloud {
|
||||
reverse_proxy anubis:8923 {
|
||||
header_up X-Real-Ip {remote_host}
|
||||
}
|
||||
}
|
||||
|
||||
# --- Prometheus ---
|
||||
prometheus.franksdank.cloud {
|
||||
reverse_proxy anubis:8923 {
|
||||
header_up X-Real-Ip {remote_host}
|
||||
}
|
||||
}
|
||||
|
||||
# --- Loki (guarded: Anubis → Authentik; loki-proxy provider, internal_host caddy:8017) ---
|
||||
loki.franksdank.cloud {
|
||||
reverse_proxy anubis:8923 {
|
||||
header_up X-Real-Ip {remote_host}
|
||||
}
|
||||
}
|
||||
|
||||
# --- Seerr (guarded: Anubis → Authentik; seerr provider, internal_host caddy:8017) ---
|
||||
seerr.franksdank.cloud {
|
||||
reverse_proxy anubis:8923 {
|
||||
header_up X-Real-Ip {remote_host}
|
||||
}
|
||||
}
|
||||
|
||||
# --- Marreta (guarded: Anubis → Authentik; marreta provider, internal_host caddy:8017) ---
|
||||
# Content-cleaning proxy. Any authentik-authenticated user may use it (no
|
||||
# group gate). Egress rides the searxng gluetun tunnel; reached via the
|
||||
# host-published port 8082 (gluetun netns), not by container name.
|
||||
marreta.franksdank.cloud {
|
||||
reverse_proxy anubis:8923 {
|
||||
header_up X-Real-Ip {remote_host}
|
||||
}
|
||||
}
|
||||
|
||||
# --- Pi-hole (quadlet on slirp4netns; published 8083) ---
|
||||
pihole.franksdank.cloud {
|
||||
basicauth {
|
||||
dankuser REPLACE_WITH_HASH
|
||||
}
|
||||
reverse_proxy 192.168.0.143:8083
|
||||
}
|
||||
|
||||
# --- Hermes WebUI (Hermex webui; host-netns container, published 8787) ------
|
||||
# Exempt by construction: the webui authenticates itself (HERMES_WEBUI_PASSWORD),
|
||||
# so no Anubis challenge and no Authentik proxy provider is required — nothing
|
||||
# here enters the http://:8016 auth chain. Internal-only by DNS absence: there is
|
||||
# no public Porkbun A record for this name, so only LAN clients (Pi-hole
|
||||
# addn-hosts → 192.168.0.143) can resolve it at all; DNS-01 issuance via the
|
||||
# Porkbun module needs no public record. Upstream is `host.containers.internal`
|
||||
# — netavark's host address, 169.254.1.2 — NOT the host LAN IP: hermes-webui is
|
||||
# `Network=host`, so 8787 is a host-netns bind and not a published port, and
|
||||
# netavark DNATs only published ports (192.168.0.143:9119, the dashboard below,
|
||||
# answers from this bridge; 192.168.0.143:8787 does not). Measured live
|
||||
# 2026-09-16 inside the Caddy container: 192.168.0.143:8787 and 10.89.1.1:8787
|
||||
# both REFUSED, host.containers.internal:8787 → 200.
|
||||
hermes.franksdank.cloud {
|
||||
reverse_proxy host.containers.internal:8787
|
||||
}
|
||||
|
||||
# --- Hermes dashboard (admin UI; published 9119 on dankclaw_default) --------
|
||||
# Exempt by construction: the app enforces its own complete auth — verified live
|
||||
# from Caddy's own network, an unauthenticated GET / returns 302 → /login?next=%2F
|
||||
# (uvicorn), and basic_auth.username + a scrypt password_hash are set in the
|
||||
# dashboard's /data/config.yaml. No outpost provider needed.
|
||||
hermesadmin.franksdank.cloud {
|
||||
reverse_proxy 192.168.0.143:9119
|
||||
}
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# Services on dankserver (192.168.0.235) — published LAN ports
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
# --- Plex (existing, kept) ---
|
||||
# http:// on purpose: franksdank.casa is not hosted in the same Porkbun
|
||||
# account as franksdank.cloud, so the DNS-01 provider cannot issue a
|
||||
# Let's Encrypt cert for it. It stays plain-HTTP legacy; the canonical
|
||||
# name is plex.franksdank.cloud (HTTPS, below).
|
||||
http://plex.franksdank.casa {
|
||||
reverse_proxy 192.168.0.235:32400 {
|
||||
transport http {
|
||||
tls_insecure_skip_verify
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# PMS requires TLS from every non-loopback client: plain-HTTP off-loopback
|
||||
# is read and silently closed (diagnosed 2026-08-26), which surfaced as an
|
||||
# empty-body 502 here. Dial the upstream over TLS; PMS's self-signed cert is
|
||||
# skipped on this pinned-LAN hop only (client-facing LE TLS is untouched).
|
||||
# The web UI lives at /web/; / returns the XML MediaContainer index.
|
||||
plex.franksdank.cloud {
|
||||
reverse_proxy 192.168.0.235:32400 {
|
||||
transport http {
|
||||
tls_insecure_skip_verify
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# --- Calibre-Web ---
|
||||
books.franksdank.cloud {
|
||||
reverse_proxy 192.168.0.235:8083
|
||||
}
|
||||
|
||||
# --- RomM ---
|
||||
romm.franksdank.cloud {
|
||||
reverse_proxy anubis:8923 {
|
||||
header_up X-Real-Ip {remote_host}
|
||||
}
|
||||
}
|
||||
|
||||
# --- Copyparty ---
|
||||
files.franksdank.cloud {
|
||||
reverse_proxy anubis:8923 {
|
||||
header_up X-Real-Ip {remote_host}
|
||||
}
|
||||
}
|
||||
|
||||
# --- Jellyfin ---
|
||||
jelly.franksdank.cloud {
|
||||
reverse_proxy 192.168.0.235:8096
|
||||
}
|
||||
|
||||
|
||||
# --- *arr Stack (exempt by construction: own auth) ---
|
||||
|
||||
prowlarr.franksdank.cloud {
|
||||
reverse_proxy 192.168.0.112:9696
|
||||
}
|
||||
|
||||
radarr.franksdank.cloud {
|
||||
reverse_proxy 192.168.0.112:7878
|
||||
}
|
||||
|
||||
# --- Sonarr: internal-only by construction (retired 2026-09-04) -----------
|
||||
# No vhost and no public DNS record for sonarr.franksdank.cloud: the TV-library
|
||||
# admin UI is reachable only on the LAN at 192.168.0.112:8989 (users request TV
|
||||
# through Seerr, which sits behind the guard chain; admins reach Sonarr's own
|
||||
# UI directly). A brief unguarded public vhost lived here and served the
|
||||
# zero-auth UI to the internet via manual DNS — retired.
|
||||
|
||||
lidarr.franksdank.cloud {
|
||||
reverse_proxy 192.168.0.112:8686
|
||||
}
|
||||
|
||||
readarr.franksdank.cloud {
|
||||
reverse_proxy 192.168.0.112:8787
|
||||
}
|
||||
|
||||
qbittorrent.franksdank.cloud {
|
||||
reverse_proxy 192.168.0.112:8080
|
||||
}
|
||||
|
||||
# --- Internal listener: auth decision layer ---------------------------------
|
||||
# Anubis dials http://caddy:8016 after verifying a client (TARGET in
|
||||
# dankclaw/anubis/quadlet/anubis.container). Host headers survive the hop.
|
||||
# Every request here is proxied whole-request to Authentik with Host
|
||||
# preserved. This build's embedded outpost routes ONLY on the ORIGINAL Host
|
||||
# against each proxy provider's external_host — meta headers are ignored and
|
||||
# cross-host absolute next= redirects are rejected by the flow's open-redirect
|
||||
# guard ("Request has been denied: Invalid next URL"), verified live.
|
||||
# Unauthenticated clients get the login flow served in place; authenticated
|
||||
# sessions are forwarded by the provider's internal_host back to
|
||||
# http://:8017 and routed straight to the app. Every host that may enter this
|
||||
# listener MUST have a proxy provider registered with the embedded outpost —
|
||||
# the Anubis guard set, the Caddy TLS edge, and the provider list stay in
|
||||
# lockstep. Exempt vhosts never enter this block.
|
||||
http://:8016 {
|
||||
reverse_proxy authentik-server:9000 {
|
||||
header_up X-Forwarded-Proto https
|
||||
}
|
||||
}
|
||||
|
||||
# --- Post-auth listener: where Authentik forwards authenticated traffic ----
|
||||
# The proxy provider's internal_host is http://caddy:8017, so every request
|
||||
# that passes authentication lands here (Host header preserved) and routes
|
||||
# straight to its app. This is the end of the chain — nothing dials back
|
||||
# into Authentik from here, so the chain cannot loop.
|
||||
http://:8017 {
|
||||
@home host www.franksdank.cloud
|
||||
handle @home {
|
||||
root * /srv/homepage
|
||||
# Deploys overwrite /srv/homepage in place; without revalidation a
|
||||
# browser keeps serving the pre-deploy CSS/JS/HTML. no-cache forces an
|
||||
# ETag round-trip on every load (a cheap 304 when unchanged).
|
||||
header Cache-Control "no-cache"
|
||||
# RBAC (2026-09-01): /admin requires membership in the homepage-admins
|
||||
# authentik group. In proxy-provider mode the outpost injects
|
||||
# X-Authentik-Groups on its upstream request to this listener and strips
|
||||
# any client-supplied X-Authentik-* headers first, so browsers cannot
|
||||
# spoof it. Regex is a substring match, insensitive to the group-list
|
||||
# separator.
|
||||
handle /admin* {
|
||||
@adminok header_regexp X-Authentik-Groups homepage-admins
|
||||
handle @adminok {
|
||||
try_files /admin.html
|
||||
file_server
|
||||
}
|
||||
respond "Forbidden: homepage admins only" 403
|
||||
}
|
||||
# User docs page (static). Anchors (#models etc.) are client-side,
|
||||
# so both /docs and /docs/ map onto the single docs.html.
|
||||
handle /docs {
|
||||
try_files /docs.html
|
||||
file_server
|
||||
}
|
||||
handle /docs/ {
|
||||
try_files /docs.html
|
||||
file_server
|
||||
}
|
||||
@data path /services.json /status.json /app.js /style.css /deployed.js
|
||||
handle @data {
|
||||
file_server
|
||||
}
|
||||
handle {
|
||||
try_files /index.html
|
||||
file_server
|
||||
}
|
||||
}
|
||||
|
||||
@search host search.franksdank.cloud
|
||||
handle @search {
|
||||
reverse_proxy 192.168.0.143:8081
|
||||
}
|
||||
|
||||
@loki host loki.franksdank.cloud
|
||||
handle @loki {
|
||||
reverse_proxy loki:3100
|
||||
}
|
||||
|
||||
@seerr host seerr.franksdank.cloud
|
||||
handle @seerr {
|
||||
reverse_proxy seerr:5055
|
||||
}
|
||||
|
||||
@grafana host grafana.franksdank.cloud
|
||||
handle @grafana {
|
||||
reverse_proxy grafana:3000
|
||||
}
|
||||
|
||||
@prometheus host prometheus.franksdank.cloud
|
||||
handle @prometheus {
|
||||
reverse_proxy prometheus:9090
|
||||
}
|
||||
|
||||
@romm host romm.franksdank.cloud
|
||||
handle @romm {
|
||||
reverse_proxy 192.168.0.235:8085
|
||||
}
|
||||
|
||||
@files host files.franksdank.cloud
|
||||
handle @files {
|
||||
reverse_proxy 192.168.0.235:3923
|
||||
}
|
||||
|
||||
@prowlarr host prowlarr.franksdank.cloud
|
||||
handle @prowlarr {
|
||||
reverse_proxy 192.168.0.112:9696
|
||||
}
|
||||
|
||||
@radarr host radarr.franksdank.cloud
|
||||
handle @radarr {
|
||||
reverse_proxy 192.168.0.112:7878
|
||||
}
|
||||
|
||||
@lidarr host lidarr.franksdank.cloud
|
||||
handle @lidarr {
|
||||
reverse_proxy 192.168.0.112:8686
|
||||
}
|
||||
|
||||
@readarr host readarr.franksdank.cloud
|
||||
handle @readarr {
|
||||
reverse_proxy 192.168.0.112:8787
|
||||
}
|
||||
|
||||
@qbittorrent host qbittorrent.franksdank.cloud
|
||||
handle @qbittorrent {
|
||||
reverse_proxy 192.168.0.112:8080
|
||||
}
|
||||
|
||||
@marreta host marreta.franksdank.cloud
|
||||
handle @marreta {
|
||||
reverse_proxy 192.168.0.143:8082
|
||||
}
|
||||
}
|
||||
|
||||
# --- Optional: Jackett ---
|
||||
# jackett.franksdank.cloud {
|
||||
# reverse_proxy 192.168.0.235:9117
|
||||
# }
|
||||
|
||||
# --- Optional: Whisparr ---
|
||||
# whisparr.franksdank.cloud {
|
||||
# reverse_proxy 192.168.0.235:6969
|
||||
# }
|
||||
@@ -1,40 +0,0 @@
|
||||
# ComfyUI on the gfx1151 (AMD Strix Halo / Radeon 8060S) ROCm torch build.
|
||||
#
|
||||
# Base image: AMD's gfx1151-tagged vLLM image. It is the only verified build
|
||||
# where torch.cuda.is_available() is True on this APU (device capability
|
||||
# (11,5) = gfx1151). It already ships the matching ROCm torch family
|
||||
# (torch 2.9.1+rocm, torchvision 0.24.0+rocm, torchaudio 2.9.0+rocm), so we
|
||||
# must NOT let pip reinstall them: ComfyUI's requirements.txt lists an unpinned
|
||||
# `torch`, which would otherwise clobber the gfx1151 kernels with a default build.
|
||||
FROM docker.io/rocm/vllm:rocm7.12.0_gfx1151_ubuntu24.04_py3.12_pytorch_2.9.1_vllm_0.16.0
|
||||
|
||||
# Bake the ComfyUI code into the image (rebuild the image to update the code).
|
||||
RUN git clone --depth 1 https://github.com/comfyanonymous/ComfyUI /opt/ComfyUI
|
||||
WORKDIR /opt/ComfyUI
|
||||
|
||||
# Pre-create the user-data dirs. The host mounts its collections over them
|
||||
# (see the comfyui service in compose.yaml).
|
||||
RUN mkdir -p /opt/ComfyUI/models /opt/ComfyUI/input /opt/ComfyUI/output /opt/ComfyUI/custom_nodes
|
||||
|
||||
# Install ComfyUI's dependencies, but skip the torch family (already present
|
||||
# above as the gfx1151 ROCm builds). torchsde and the rest are kept.
|
||||
RUN grep -viE '^(torch|torchvision|torchaudio)([^a-z]|$)' requirements.txt > /tmp/req-notorch.txt \
|
||||
&& pip install --no-cache-dir -r /tmp/req-notorch.txt
|
||||
|
||||
# The city96/ComfyUI-GGUF custom node (loaded from the host's custom_nodes
|
||||
# volume) needs the gguf library to decode GGUF-quantized diffusion models.
|
||||
# sentencepiece and protobuf are its optional tokenizer deps. Baking them into
|
||||
# the image keeps the dependency across container recreations — the node itself
|
||||
# lives in the host volume, which persists, but the Python env does not.
|
||||
RUN pip install --no-cache-dir "gguf>=0.13.0" sentencepiece protobuf
|
||||
|
||||
# Bake the qwen_image21 patch script into the image. It is run by the
|
||||
# startup wrapper before ComfyUI starts, so the patch survives container
|
||||
COPY dankclaw/apply_qwen21_patch.py /opt/ComfyUI/apply_qwen21_patch.py
|
||||
COPY dankclaw/comfyui-start.sh /opt/ComfyUI/comfyui-start.sh
|
||||
RUN chmod +x /opt/ComfyUI/comfyui-start.sh
|
||||
|
||||
CMD ["/opt/ComfyUI/comfyui-start.sh"]
|
||||
EXPOSE 8188
|
||||
# v2: VAE 4D-to-5D latent expansion for video VAEs (qwen_image21)
|
||||
# v3: fix in_channels to use shape[0] (latent input dim) not shape[1] (hidden dim)
|
||||
@@ -1,18 +0,0 @@
|
||||
# Alloy + a static busybox.
|
||||
#
|
||||
# grafana/alloy has a shell but no wget/curl, so the compose healthcheck
|
||||
# cannot run anything inside the container. This image drops a static
|
||||
# busybox into it for the healthcheck command only. All Alloy behavior is
|
||||
# unchanged.
|
||||
#
|
||||
# Build on dankclaw:
|
||||
# podman build -t franksdank/alloy:latest -f dankclaw/alloy/Containerfile .
|
||||
|
||||
FROM docker.io/library/alpine:latest AS tools
|
||||
# busybox-static is statically linked: the target images have no dynamic
|
||||
# loader (loki is distroless), so the alpine default /bin/busybox (musl
|
||||
# dynamic) would fail with "no such file or directory".
|
||||
RUN apk add --no-cache busybox-static
|
||||
|
||||
FROM docker.io/grafana/alloy:latest
|
||||
COPY --from=tools /bin/busybox.static /usr/local/bin/busybox
|
||||
@@ -1,25 +0,0 @@
|
||||
{
|
||||
"bots": [
|
||||
{
|
||||
"name": "deny AI scrapers",
|
||||
"user_agent_regex": "(?i)^(GPTBot|ChatGPT-User|OAI-SearchBot|CCBot|anthropic-ai|ClaudeBot|Claude-Web|PerplexityBot|PerplexityUser|Google-Extended|Applebot-Extended|Bytespider|Amazonbot|meta-externalagent|DuckAssistBot|cohere-ai|YouBot|KimiBot)",
|
||||
"action": "DENY"
|
||||
},
|
||||
{
|
||||
"name": "serve anubis own endpoints",
|
||||
"path_regex": "^/\\.within\\.website",
|
||||
"action": "ALLOW"
|
||||
},
|
||||
{
|
||||
"name": "serve authentik outpost transport",
|
||||
"path_regex": "^/outpost\\.goauthentik\\.io/",
|
||||
"action": "ALLOW"
|
||||
},
|
||||
{
|
||||
"name": "challenge everything else",
|
||||
"path_regex": ".",
|
||||
"action": "CHALLENGE"
|
||||
}
|
||||
],
|
||||
"dnsbl": false
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
# Quadlet unit: Anubis bot-challenge proxy (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/anubis.container
|
||||
# Unit name: anubis.service (container name: anubis)
|
||||
#
|
||||
# Two-tier chain: the public Caddy listeners for the guarded vhosts proxy
|
||||
# everything to this container (port 8923, internal only — no published port,
|
||||
# so metrics on :9090 stay reachable only inside dankclaw_default); Anubis
|
||||
# verifies the client (challenge/DENY per policy) and forwards accepted
|
||||
# traffic back to Caddy's internal listener (TARGET below), where the normal
|
||||
# per-vhost routing — including Authentik forward_auth — resumes. Host headers
|
||||
# survive both hops, so vhost matching on the internal listener still works.
|
||||
#
|
||||
# Image pin: the image is pulled from ghcr.io/techarohq/anubis (source of
|
||||
# truth), but the unit runs it under a localhost/-aliased tag: on this host's
|
||||
# podman 5.7 quadlet path, registry-prefixed refs fail to resolve from local
|
||||
# storage at container create time ("image not known") even when present; the
|
||||
# localhost alias resolves (verified live 2026-08-28). v1.27.0 pinned 2026-09-02: --help
|
||||
# verified on the image itself (cookie-domain, ed25519-private-key-hex,
|
||||
# cookie-expiration-time default 168h, difficulty default 4); JSON policy
|
||||
# schema unchanged; X-Real-Ip is now REQUIRED (missing header => HTTP 500) —
|
||||
# the Caddy chain always sets it (bats seam enforces the header_up line).
|
||||
# Upgrading = pull new tag, re-tag the localhost alias, restart the unit.
|
||||
#
|
||||
# No secrets in this repo. COOKIE_DOMAIN makes the challenge cookie valid for
|
||||
# all *.franksdank.cloud vhosts: solve the proof-of-work once (e.g. on www)
|
||||
# and every guarded vhost honors it until expiry (default 168h, aligned with
|
||||
# authentik's 7-day session) or re-login. ED25519_PRIVATE_KEY_HEX pins the
|
||||
# JWT signing key (host file /home/dank/Containers/anubis/env, mode 0600,
|
||||
# gitea-style ${VAR} expansion via the [Service] EnvironmentFile): without
|
||||
# it every restart invalidates everyone's solved challenges. See
|
||||
# serverAppConfigs-docs/Infrastructure/Anubis.md.
|
||||
|
||||
[Unit]
|
||||
Description=Anubis bot-challenge proxy container
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Container]
|
||||
ContainerName=anubis
|
||||
Image=localhost/techarohq/anubis:v1.27.0
|
||||
Pull=never
|
||||
Network=dankclaw_default
|
||||
# flagenv maps env names to flag names (TARGET -> --target, POLICY_FNAME ->
|
||||
# --policy-fname). TARGET points back at Caddy's internal listener; the
|
||||
# container DNS name resolves on dankclaw_default.
|
||||
Environment=TARGET=http://caddy:8016
|
||||
Environment=POLICY_FNAME=/policy/policy.json
|
||||
Environment=COOKIE_DOMAIN=franksdank.cloud
|
||||
EnvironmentFile=/home/dank/Containers/anubis/env
|
||||
Environment=ED25519_PRIVATE_KEY_HEX=${ED25519_PRIVATE_KEY_HEX}
|
||||
Volume=/home/dank/serverAppConfigs/dankclaw/anubis/policy.json:/policy/policy.json:ro
|
||||
# The image ships no HEALTHCHECK (distroless ko-build: entrypoint is
|
||||
# /ko-app/anubis, no shell in the image), so the probe is declared here.
|
||||
# JSON exec form on purpose: this quadlet generator joins HealthCmd into a
|
||||
# single \x20-escaped argv element and podman then runs it as shell-form —
|
||||
# which needs a shell the distroless image does not have. A JSON array is
|
||||
# passed through verbatim and executes /ko-app/anubis directly.
|
||||
HealthCmd=["/ko-app/anubis","--healthcheck"]
|
||||
HealthInterval=15s
|
||||
HealthTimeout=5s
|
||||
HealthRetries=3
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=/home/dank/Containers/anubis/env
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,52 +0,0 @@
|
||||
import os
|
||||
|
||||
# Apply qwen_image21 detection patch
|
||||
f = "/opt/ComfyUI/comfy/model_detection.py"
|
||||
with open(f) as fh:
|
||||
code = fh.read()
|
||||
if "qwen_image21" not in code:
|
||||
qwen21_check = (
|
||||
" if '{}txt_in.text_norm.weight'.format(key_prefix) in state_dict_keys: # Qwen Image 2.1\n"
|
||||
" dit_config = {}\n"
|
||||
' dit_config["image_model"] = "qwen_image21"\n'
|
||||
" dit_config[\"in_channels\"] = state_dict['{}img_in.weight'.format(key_prefix)].shape[0]\n"
|
||||
" dit_config[\"num_layers\"] = count_blocks(state_dict_keys, '{}transformer_blocks.'.format(key_prefix) + '{}.')\n"
|
||||
" return dit_config\n"
|
||||
)
|
||||
anchor = " if '{}txt_norm.weight'.format(key_prefix) in state_dict_keys: # Qwen Image"
|
||||
code = code.replace(anchor, qwen21_check + "\n" + anchor)
|
||||
with open(f, "w") as fh:
|
||||
fh.write(code)
|
||||
print("Applied qwen_image21 detection patch")
|
||||
|
||||
# Add qwen_image21 to GGUF node arch list
|
||||
loader = "/opt/ComfyUI/custom_nodes/ComfyUI-GGUF/loader.py"
|
||||
if os.path.exists(loader):
|
||||
with open(loader) as fh:
|
||||
lcode = fh.read()
|
||||
if '"qwen_image21"' not in lcode:
|
||||
old = '"lumina2", "qwen_image"}'
|
||||
new = '"lumina2", "qwen_image", "qwen_image21"}'
|
||||
if old in lcode:
|
||||
lcode = lcode.replace(old, new)
|
||||
with open(loader, "w") as fh:
|
||||
fh.write(lcode)
|
||||
print("Applied qwen_image21 arch list patch")
|
||||
|
||||
# Patch VAE decode to expand 4D latents to 5D for video VAEs (latent_dim=3)
|
||||
sd = "/opt/ComfyUI/comfy/sd.py"
|
||||
with open(sd) as fh:
|
||||
sdc = fh.read()
|
||||
if "self.latent_dim == 3 and samples_in.ndim == 4" not in sdc:
|
||||
anchor = " if self.latent_dim == 2 and samples_in.ndim == 5:\n samples_in = samples_in[:, :, 0]"
|
||||
replacement = (
|
||||
" if self.latent_dim == 2 and samples_in.ndim == 5:\n"
|
||||
" samples_in = samples_in[:, :, 0]\n"
|
||||
" elif self.latent_dim == 3 and samples_in.ndim == 4:\n"
|
||||
" samples_in = samples_in.unsqueeze(2)"
|
||||
)
|
||||
if anchor in sdc:
|
||||
sdc = sdc.replace(anchor, replacement)
|
||||
with open(sd, "w") as fh:
|
||||
fh.write(sdc)
|
||||
print("Applied VAE 4D-to-5D latent expansion patch")
|
||||
@@ -1,40 +0,0 @@
|
||||
# Quadlet unit: Authentik LDAP outpost (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/authentik-outpost-ldap.container
|
||||
# Unit name: authentik-outpost-ldap.service (podman 5.x drops the container- prefix)
|
||||
#
|
||||
# Serves the authentik LDAP provider ("Jellyfin LDAP", Base DN
|
||||
# dc=franksdank,dc=cloud) so dankserver's Jellyfin can authenticate its users
|
||||
# via the LDAP-Auth plugin. The type-specific ghcr.io/goauthentik/ldap image
|
||||
# listens on 3389 (ldap) / 6636 (ldaps) inside the container; published to
|
||||
# host 389/636 for cross-host binds from dankserver. The outpost token lives
|
||||
# in the host .env (AUTHENTIK_LDAP_OUTPOST_TOKEN), never in this repo. The
|
||||
# outpost dials core over websocket at AUTHENTIK_HOST; auth. is guard-chain
|
||||
# exempt (IdP).
|
||||
|
||||
[Unit]
|
||||
Description=Authentik LDAP outpost container
|
||||
After=network.target authentik-server.service
|
||||
Wants=authentik-server.service
|
||||
|
||||
[Container]
|
||||
ContainerName=authentik-outpost-ldap
|
||||
Network=dankclaw_default
|
||||
# Image=ghcr.io/goauthentik/ldap:2026.8.3 (amd64 digest; podman 5.7.0 refuses
|
||||
# the tag's nested manifest list, so pin the single-platform digest).
|
||||
EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env
|
||||
Image=ghcr.io/goauthentik/ldap@sha256:2e7b259eb8c37faac916946ddf48ec681ee9d3a9f724fe8521bf161973f13cf9
|
||||
Environment=AUTHENTIK_HOST=https://auth.franksdank.cloud
|
||||
Environment=AUTHENTIK_INSECURE=false
|
||||
Environment=AUTHENTIK_TOKEN=${AUTHENTIK_LDAP_OUTPOST_TOKEN}
|
||||
PublishPort=389:3389
|
||||
PublishPort=636:6636
|
||||
User=0:0
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,44 +0,0 @@
|
||||
# Quadlet unit: Authentik outpost (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/authentik-outpost.container
|
||||
# Unit name: authentik-outpost.service (podman 5.x drops the container- prefix)
|
||||
#
|
||||
# In authentik 2026.8.x the proxy-provider URL space (/outpost.goauthentik.io/*)
|
||||
# is served by the Rust worker binary, not the server — verified live: with a
|
||||
# tenant-matching Host header, /if/flow/... answers 200 on the server while
|
||||
# /auth/caddy still 404s. This container runs the same image as the server and
|
||||
# worker units; Exec=worker selects the Rust worker role, and the
|
||||
# AUTHENTIK_HOST/AUTHENTIK_TOKEN env flips it into outpost mode: it dials back
|
||||
# to core over websocket (ws/outpost/<uuid>/) and serves forward-auth for Caddy
|
||||
# on its own 9000. No published port — Caddy reaches it by container DNS on
|
||||
# dankclaw_default; the token lives in the host .env, never in this repo.
|
||||
|
||||
[Unit]
|
||||
Description=Authentik outpost container
|
||||
After=network.target authentik-postgres.service authentik-server.service
|
||||
Wants=authentik-server.service
|
||||
|
||||
[Container]
|
||||
ContainerName=authentik-outpost
|
||||
Image=ghcr.io/goauthentik/server:2026.8.3
|
||||
Exec=worker
|
||||
Network=dankclaw_default
|
||||
EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env
|
||||
Environment=AUTHENTIK_POSTGRESQL__HOST=authentik-postgres
|
||||
Environment=AUTHENTIK_POSTGRESQL__NAME=authentik
|
||||
Environment=AUTHENTIK_POSTGRESQL__USER=authentik
|
||||
Environment=AUTHENTIK_POSTGRESQL__PASSWORD=${AUTHENTIK_PG_PASS}
|
||||
Environment=AUTHENTIK_SECRET_KEY=${AUTHENTIK_SECRET_KEY}
|
||||
Environment=AUTHENTIK_HOST=https://auth.franksdank.cloud
|
||||
Environment=AUTHENTIK_INSECURE=false
|
||||
Environment=AUTHENTIK_TOKEN=${AUTHENTIK_OUTPOST_TOKEN}
|
||||
Volume=/home/dank/Containers/authentik/data:/data
|
||||
User=0:0
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,42 +0,0 @@
|
||||
# Quadlet unit: Authentik PostgreSQL 16 (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/authentik-postgres.container
|
||||
# Unit name: authentik-postgres.service (podman 5.x drops the container- prefix)
|
||||
#
|
||||
# Replaces the compose service retired when this stack went boot-persistent
|
||||
# (rootless Podman does not auto-start bare containers at boot). Secrets stay
|
||||
# out of this file: systemd expands ${AUTHENTIK_PG_PASS} from the git-ignored
|
||||
# host .env at start time (same pattern as gitea-postgres).
|
||||
|
||||
[Unit]
|
||||
Description=Authentik PostgreSQL container
|
||||
After=network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=authentik-postgres
|
||||
Image=docker.io/library/postgres:16-alpine
|
||||
Network=dankclaw_default
|
||||
EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env
|
||||
Environment=POSTGRES_DB=authentik
|
||||
Environment=POSTGRES_USER=authentik
|
||||
Environment=POSTGRES_PASSWORD=${AUTHENTIK_PG_PASS}
|
||||
Volume=/home/dank/Containers/authentik/postgres:/var/lib/postgresql/data
|
||||
# Unquoted argv: this quadlet generator space-splits HealthCmd into the
|
||||
# healthcheck's argv array, so a quoted probe would arrive mangled (the old
|
||||
# `/bin/sh -c "pg_isready..."` form arrived as /bin/sh + broken args and
|
||||
# failed with exit 1 forever — the health status was a false negative while
|
||||
# the DB itself accepted connections). Bare argv works under both exec forms.
|
||||
HealthCmd=pg_isready -d authentik -U authentik
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=5
|
||||
HealthStartPeriod=20s
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,45 +0,0 @@
|
||||
# Quadlet unit: Authentik core + embedded Caddy outpost (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/authentik-server.container
|
||||
# Unit name: authentik-server.service (podman 5.x drops the container- prefix)
|
||||
#
|
||||
# The embedded outpost serves /outpost.goauthentik.io/* including the
|
||||
# /auth/caddy forward_auth endpoint the Caddyfile chains to — no outpost token
|
||||
# exists for this deployment by design. Resolved by name (A2A DNS) from every
|
||||
# container on dankclaw_default, including caddy. The 9000 publish is
|
||||
# loopback-only on purpose: LAN clients reach Authentik through Caddy, never
|
||||
# around it; the loopback publish exists so host-side tooling can probe
|
||||
# /-/health/. Runs as the image default user (same userns mapping the compose
|
||||
# era used, so the existing /data ownership carries over unchanged).
|
||||
|
||||
[Unit]
|
||||
Description=Authentik server container
|
||||
After=network.target authentik-postgres.service
|
||||
Wants=authentik-postgres.service
|
||||
|
||||
[Container]
|
||||
ContainerName=authentik-server
|
||||
Image=ghcr.io/goauthentik/server:2026.8.3
|
||||
Exec=server
|
||||
Network=dankclaw_default
|
||||
EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env
|
||||
Environment=AUTHENTIK_POSTGRESQL__HOST=authentik-postgres
|
||||
Environment=AUTHENTIK_POSTGRESQL__NAME=authentik
|
||||
Environment=AUTHENTIK_POSTGRESQL__USER=authentik
|
||||
Environment=AUTHENTIK_POSTGRESQL__PASSWORD=${AUTHENTIK_PG_PASS}
|
||||
Environment=AUTHENTIK_SECRET_KEY=${AUTHENTIK_SECRET_KEY}
|
||||
PublishPort=127.0.0.1:9000:9000
|
||||
# Probe must contain no quotes: this host's quadlet v1 generator space-splits
|
||||
# HealthCmd into the healthcheck argv array, and quoted probes arrive mangled
|
||||
# (verified live 2026-08-28). `python /manage.py check --database default`
|
||||
# exits nonzero iff Django (incl. the DB connection) cannot serve.
|
||||
HealthCmd=python /manage.py check --database default
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,37 +0,0 @@
|
||||
# Quadlet unit: Authentik worker — background tasks, email, webhook/log outbox
|
||||
# (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/authentik-worker.container
|
||||
# Unit name: authentik-worker.service (podman 5.x drops the container- prefix)
|
||||
#
|
||||
# Same image as the server; the Exec= command selects the worker role. Runs as
|
||||
# user 0:0 (rootless: maps to the invoking host user, matching the compose-era
|
||||
# semantics). No published ports — the worker talks to PostgreSQL in-network
|
||||
# and has no listener of its own.
|
||||
|
||||
[Unit]
|
||||
Description=Authentik worker container
|
||||
After=network.target authentik-postgres.service authentik-server.service
|
||||
Wants=authentik-server.service
|
||||
|
||||
[Container]
|
||||
ContainerName=authentik-worker
|
||||
Image=ghcr.io/goauthentik/server:2026.8.3
|
||||
Exec=worker
|
||||
Network=dankclaw_default
|
||||
EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env
|
||||
Environment=AUTHENTIK_POSTGRESQL__HOST=authentik-postgres
|
||||
Environment=AUTHENTIK_POSTGRESQL__NAME=authentik
|
||||
Environment=AUTHENTIK_POSTGRESQL__USER=authentik
|
||||
Environment=AUTHENTIK_POSTGRESQL__PASSWORD=${AUTHENTIK_PG_PASS}
|
||||
Environment=AUTHENTIK_SECRET_KEY=${AUTHENTIK_SECRET_KEY}
|
||||
Volume=/home/dank/Containers/authentik/data:/data
|
||||
User=0:0
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,22 +0,0 @@
|
||||
# Caddy with the Porkbun DNS provider (github.com/caddy-dns/porkbun).
|
||||
#
|
||||
# The porkbun module lets Caddy solve Let's Encrypt DNS-01 challenges
|
||||
# directly through the Porkbun API, so the wildcard cert for
|
||||
# *.franksdank.cloud / *.franksdank.casa can be issued and renewed without
|
||||
# any port 80/443 reachability from the internet (see the Caddyfile in
|
||||
# ../Caddyfile and serverAppConfigs-docs/Infrastructure/LAN-DNS.md).
|
||||
#
|
||||
# Build on dankclaw (podman rootless, same as every other container here):
|
||||
# podman build -t franksdank/caddy-porkbun:latest -f dankclaw/caddy/Containerfile .
|
||||
#
|
||||
# The first build runs an xcaddy Go compilation and takes a few minutes.
|
||||
# Rebuild only when Caddy or the porkbun module must be updated.
|
||||
|
||||
FROM docker.io/library/caddy:2-builder AS builder
|
||||
|
||||
RUN xcaddy build \
|
||||
--with github.com/caddy-dns/porkbun \
|
||||
--output /usr/bin/caddy-porkbun
|
||||
|
||||
FROM docker.io/library/caddy:2
|
||||
COPY --from=builder /usr/bin/caddy-porkbun /usr/bin/caddy
|
||||
@@ -1,7 +0,0 @@
|
||||
# Caddy environment (placeholders only). The deploy writes the live file at
|
||||
# /home/dank/Containers/caddy/env from the checkout's .env; the quadlet reads
|
||||
# it via EnvironmentFile. Consumed by the Caddyfile's acme_dns block
|
||||
# (DNS-01 wildcard cert issuance). PORKBUN_API_SECRET_KEY is the name the
|
||||
# caddy-dns/porkbun module expects.
|
||||
PORKBUN_API_KEY=changeme
|
||||
PORKBUN_API_SECRET_KEY=changeme
|
||||
@@ -1,54 +0,0 @@
|
||||
# Quadlet unit: canonical Caddy reverse proxy + ACME (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/caddy.container
|
||||
# Unit name: caddy.service (container name: caddy)
|
||||
#
|
||||
# The homelab's single Caddy: it fronts every service on both hosts and issues
|
||||
# Let's Encrypt wildcard certs via DNS-01 (Porkbun). It left compose.yaml on
|
||||
# 2026-08-26 after a reboot exposed that the bare compose container never came
|
||||
# back: rootless Podman has no boot-time auto-start for bare containers, and
|
||||
# `restart: unless-stopped` does not survive the explicit stop that shutdown
|
||||
# performs. See AGENTS.md "Boot Persistence & Self-Healing" and
|
||||
# serverAppConfigs-docs/Infrastructure/Caddy.md.
|
||||
#
|
||||
# It joins the existing compose network (dankclaw_default) so the Caddyfile's
|
||||
# container-name upstreams (authentik-server, grafana, ...) still resolve; if
|
||||
# no compose service has created that network yet at boot, quadlet creates it
|
||||
# first and compose reuses it by name later. The image is built locally from
|
||||
# dankclaw/caddy/Containerfile and exists only on this host, so Pull=never: a
|
||||
# boot must not stall on a registry that does not have it. Porkbun credentials
|
||||
# come from /home/dank/Containers/caddy/env (deploy writes it from the
|
||||
# checkout's .env; secrets never enter the repo).
|
||||
|
||||
[Unit]
|
||||
Description=Caddy reverse proxy + ACME (Porkbun DNS-01) container
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Container]
|
||||
ContainerName=caddy
|
||||
Image=franksdank/caddy-porkbun:latest
|
||||
Pull=never
|
||||
Network=dankclaw_default
|
||||
PublishPort=0.0.0.0:80:80/tcp
|
||||
PublishPort=0.0.0.0:443:443/tcp
|
||||
EnvironmentFile=/home/dank/Containers/caddy/env
|
||||
Volume=/home/dank/Containers/caddy/Caddyfile:/etc/caddy/Caddyfile:ro
|
||||
Volume=/home/dank/Containers/caddy/data:/data
|
||||
Volume=/home/dank/Containers/caddy/config:/config
|
||||
# The homepage: the host's /srv/homepage (deploy-managed, not the repo's
|
||||
# shared/homepage).
|
||||
Volume=/srv/homepage:/srv/homepage:ro
|
||||
# The bare-IP catch-all answers 302 (curl -f passes on <400); this proves
|
||||
# Caddy itself is serving without touching any upstream.
|
||||
HealthCmd=curl -sf -o /dev/null http://localhost:80/
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=3
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,4 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Apply qwen_image21 patches before starting ComfyUI
|
||||
python3 /opt/ComfyUI/apply_qwen21_patch.py 2>/dev/null
|
||||
exec python main.py --listen 0.0.0.0 --port 8188
|
||||
@@ -1,21 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""ComfyUI container healthcheck: the UI must answer on :8188.
|
||||
|
||||
Runs INSIDE the container, where the UI listens (the unit publishes that port on
|
||||
the host loopback for the homepage status card). ComfyUI only answers once
|
||||
torch, the custom nodes and the graph are loaded, so a 200 here means the
|
||||
service is usable rather than merely a live process — the distinction that
|
||||
matters, because the failure this replaced was a container that stayed "up"
|
||||
while dying on every start.
|
||||
|
||||
HealthCmd in the quadlet splits on spaces into an argv array (podman 5.x v1
|
||||
generator), which is why this is a file and not an inline `python3 -c`.
|
||||
"""
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen("http://127.0.0.1:8188/", timeout=10) as resp:
|
||||
sys.exit(0 if resp.status == 200 else 1)
|
||||
except Exception:
|
||||
sys.exit(1)
|
||||
@@ -1,70 +0,0 @@
|
||||
# Quadlet unit: ComfyUI (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/comfyui.container
|
||||
# Unit name: comfyui.service (container name: comfyui)
|
||||
#
|
||||
# WHY THIS FILE EXISTS. ComfyUI ran from `dankclaw/compose.yaml` through a
|
||||
# hand-written `~/.config/systemd/user/comfyui.service` that called
|
||||
# `podman compose up -d`. Its container was created 2026-09-15 14:12, BEFORE the
|
||||
# GPU declarations were added to the compose service, and `restart:
|
||||
# unless-stopped` then kept restarting that stale container — so it started
|
||||
# without /dev/kfd, without /dev/dri and without keep-groups, died on
|
||||
# `RuntimeError: No HIP GPUs are available`, and had restarted 9,146 times when
|
||||
# this was found. A recreate via compose would have fixed the container; a unit
|
||||
# fixes the class: one owner per container name, repo-tracked, boot-persistent
|
||||
# through the generator.
|
||||
#
|
||||
# GPU ACCESS. /dev/kfd + /dev/dri plus keep-groups. A literal `render` group
|
||||
# entry cannot work under rootless Podman (the group name does not resolve
|
||||
# inside the user namespace), and the companion udev rule
|
||||
# (dankclaw/udev/99-comfyui-gpu-access.rules) keeps those nodes MODE 0666 so a
|
||||
# user-unit spawn — which carries no supplementary groups — can still open them.
|
||||
# Both halves are asserted by tests/comfyui_rocm.bats.
|
||||
#
|
||||
# EXPOSURE. LAN-reachable, no auth. ComfyUI has no authentication and executes
|
||||
# arbitrary node graphs. It is internal-only (no public vhost, no guard chain);
|
||||
# it is reached on the LAN via the Pi-hole DNS record
|
||||
# (comfyui.franksdank.cloud -> 192.168.0.143, in dankclaw/pihole/addn-hosts),
|
||||
# which resolves to this host's IP — so the bind is widened from loopback to
|
||||
# 0.0.0.0:8188. The homepage admin card links to that URL and probes
|
||||
# 127.0.0.1:8188.
|
||||
#
|
||||
# DATA. Models, input, output and custom_nodes live under
|
||||
# /home/dank/comfyui-data (host state, not in the repo).
|
||||
[Unit]
|
||||
Description=ComfyUI (gfx1151 ROCm image generation)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Container]
|
||||
ContainerName=comfyui
|
||||
Image=localhost/comfyui-rocm:gfx1151
|
||||
Pull=never
|
||||
Network=dankclaw_default
|
||||
PublishPort=0.0.0.0:8188:8188
|
||||
AddDevice=/dev/kfd
|
||||
AddDevice=/dev/dri
|
||||
GroupAdd=keep-groups
|
||||
Volume=/home/dank/comfyui-data/models:/opt/ComfyUI/models
|
||||
Volume=/home/dank/comfyui-data/input:/opt/ComfyUI/input
|
||||
Volume=/home/dank/comfyui-data/output:/opt/ComfyUI/output
|
||||
Volume=/home/dank/comfyui-data/custom_nodes:/opt/ComfyUI/custom_nodes
|
||||
Volume=/home/dank/serverAppConfigs/dankclaw/comfyui/comfyui-healthcheck.py:/usr/local/bin/comfyui-healthcheck.py:ro
|
||||
Exec=/opt/ComfyUI/comfyui-start.sh
|
||||
# The probe lives in a file because the podman 5.x v1 generator splits
|
||||
# HealthCmd on spaces into an argv array; an inline `python3 -c "..."` becomes
|
||||
# a puzzle of quoting that fails silently.
|
||||
HealthCmd=python3 /usr/local/bin/comfyui-healthcheck.py
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=3
|
||||
# First start loads torch and the node graph; give it room before the probe
|
||||
# counts against it.
|
||||
HealthStartPeriod=300s
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,4 +0,0 @@
|
||||
# Gitea stack secrets. Copy to /opt/gitea/.env on dankclaw (mode 0600).
|
||||
# The live password never enters this repo.
|
||||
|
||||
GITEA_DB_PASSWORD=changeme
|
||||
@@ -1,31 +0,0 @@
|
||||
# Quadlet unit: Gitea PostgreSQL (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/gitea-postgres.container
|
||||
# Unit name: gitea-postgres.service (podman 5.x drops the container- prefix)
|
||||
#
|
||||
# The DB password lives in /home/dank/Containers/gitea/env on the host (mode 0600).
|
||||
# This file holds no secrets. See serverAppConfigs-docs/Dankclaw/Gitea.md.
|
||||
# The [Service] EnvironmentFile= line lets systemd expand
|
||||
# ${GITEA_DB_PASSWORD} in the generated ExecStart.
|
||||
|
||||
[Unit]
|
||||
Description=Gitea PostgreSQL container
|
||||
After=network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=gitea-postgres
|
||||
Network=gitea.network
|
||||
Image=docker.io/library/postgres:16-alpine
|
||||
EnvironmentFile=/home/dank/Containers/gitea/env
|
||||
Environment=POSTGRES_USER=gitea
|
||||
Environment=POSTGRES_PASSWORD=${GITEA_DB_PASSWORD}
|
||||
Environment=POSTGRES_DB=gitea
|
||||
Volume=/home/dank/Containers/gitea/postgres:/var/lib/postgresql/data
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=/home/dank/Containers/gitea/env
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,44 +0,0 @@
|
||||
# Quadlet unit: Gitea web server (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/gitea-server.container
|
||||
# Unit name: gitea-server.service (podman 5.x drops the container- prefix)
|
||||
#
|
||||
# The [Service] EnvironmentFile= line lets systemd expand
|
||||
# ${GITEA_DB_PASSWORD} in the generated ExecStart.
|
||||
# The DB password lives in /home/dank/Containers/gitea/env on the host (mode 0600).
|
||||
# This file holds no secrets. See serverAppConfigs-docs/Dankclaw/Gitea.md.
|
||||
#
|
||||
# Ports (live): 3000 = HTTP UI/API, 2222 = git SSH. The docs table in
|
||||
# Dankclaw Stack.md historically said 3001; the live host port is 3000.
|
||||
|
||||
[Unit]
|
||||
Description=Gitea web server container
|
||||
Wants=gitea-postgres.service
|
||||
After=gitea-postgres.service network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=gitea-server
|
||||
Network=gitea.network
|
||||
Image=docker.io/gitea/gitea:latest
|
||||
EnvironmentFile=/home/dank/Containers/gitea/env
|
||||
Environment=DATABASE_TYPE=postgres
|
||||
Environment=DATABASE_HOST=gitea-postgres
|
||||
Environment=DATABASE_NAME=gitea
|
||||
Environment=DATABASE_USER=gitea
|
||||
Environment=DATABASE_PASSWD=${GITEA_DB_PASSWORD}
|
||||
Environment=DATABASE_PORT=5432
|
||||
Environment=USER_UID=1000
|
||||
Environment=USER_GID=1000
|
||||
PublishPort=0.0.0.0:3000:3000
|
||||
PublishPort=2222:22
|
||||
Volume=/home/dank/Containers/gitea/data:/data
|
||||
Volume=/etc/timezone:/etc/timezone:ro
|
||||
Volume=/etc/localtime:/etc/localtime:ro
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=/home/dank/Containers/gitea/env
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,10 +0,0 @@
|
||||
# Quadlet network: private bridge for the Gitea stack (rootless).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/gitea.network
|
||||
# Unit name: gitea-network.service (podman network name: systemd-gitea)
|
||||
#
|
||||
# DNS is enabled by default for quadlet networks. This matters: the
|
||||
# automatic "podman" default network has dns_enabled=false, so the
|
||||
# gitea-postgres and gitea-server container names would not resolve.
|
||||
|
||||
[Network]
|
||||
@@ -1,100 +0,0 @@
|
||||
# Hermes configuration backup / non-secret baseline.
|
||||
#
|
||||
# Source on Dankclaw: /home/dank/Containers/hermes/data/config.yaml
|
||||
# This file is a sanitized Git backup, not a complete replacement for the
|
||||
# live Hermes state. Do not add dashboard password hashes, phone numbers,
|
||||
# OAuth credentials, API keys, sessions, memories, or generated state here.
|
||||
# The live directory is bind-mounted into the containers as /data.
|
||||
|
||||
model:
|
||||
provider: dankclaw-large
|
||||
base_url: http://192.168.0.143:7770/v1
|
||||
api_key: not-needed
|
||||
default: frank-large
|
||||
|
||||
providers:
|
||||
dankclaw-large:
|
||||
api: http://192.168.0.143:7770/v1
|
||||
discover_models: false
|
||||
models:
|
||||
frank-large:
|
||||
context_length: 114688
|
||||
dankserver-small:
|
||||
api: http://192.168.0.235:7770/v1
|
||||
discover_models: false
|
||||
models:
|
||||
frank-medium:
|
||||
context_length: 114688
|
||||
|
||||
# Auxiliary routing from the live configuration. Named providers route via
|
||||
# their `api:` URL; `custom` entries route by their explicit `base_url`.
|
||||
auxiliary:
|
||||
vision: {provider: dankclaw-large, model: frank-large}
|
||||
web_extract: {provider: dankserver-small, model: frank-medium}
|
||||
compression: {provider: dankserver-small, model: frank-medium}
|
||||
skills_hub: {provider: dankserver-small, model: frank-medium}
|
||||
approval: {provider: dankserver-small, model: frank-medium}
|
||||
mcp: {provider: dankserver-small, model: frank-medium}
|
||||
# Title generation goes out over the Responses transport on purpose: Hermes
|
||||
# hardcodes response_format=json_schema on this one call, no local back-end
|
||||
# implements constrained decoding (it is a 400), and this transport is the
|
||||
# only one that drops the field. reasoning.effort=none is required with it —
|
||||
# the call caps output at 64 tokens, which a thinking pass would consume
|
||||
# entirely, leaving an empty title.
|
||||
title_generation:
|
||||
provider: dankserver-small
|
||||
model: frank-medium
|
||||
api_mode: codex_responses
|
||||
extra_body:
|
||||
reasoning:
|
||||
effort: none
|
||||
memory_query_rewrite: {provider: custom, model: frank-medium, base_url: http://192.168.0.235:7770/v1}
|
||||
tts_audio_tags: {provider: custom, model: frank-medium, base_url: http://192.168.0.235:7770/v1}
|
||||
triage_specifier: {provider: dankclaw-large, model: frank-large}
|
||||
kanban_decomposer: {provider: dankclaw-large, model: frank-large}
|
||||
profile_describer: {provider: dankserver-small, model: frank-medium}
|
||||
curator: {provider: dankserver-small, model: frank-medium}
|
||||
|
||||
dashboard:
|
||||
# basic_auth intentionally omitted; its password_hash remains live-only.
|
||||
font: atkinson-hyperlegible
|
||||
moa:
|
||||
presets:
|
||||
default:
|
||||
reference_models:
|
||||
- provider: openrouter
|
||||
model: deepseek/deepseek-v4-pro
|
||||
enabled: false
|
||||
degraded_reference_policy: loud
|
||||
fanout: user_turn
|
||||
reference_models:
|
||||
- provider: openrouter
|
||||
model: deepseek/deepseek-v4-pro
|
||||
enabled: false
|
||||
aggregator:
|
||||
provider: openrouter
|
||||
model: anthropic/claude-opus-4.8
|
||||
degraded_reference_policy: loud
|
||||
max_tokens: 4096
|
||||
fanout: user_turn
|
||||
enabled: true
|
||||
skills:
|
||||
disabled: []
|
||||
onboarding:
|
||||
seen:
|
||||
busy_input_prompt: true
|
||||
_config_version: 34
|
||||
platforms:
|
||||
signal:
|
||||
# Live config has the Signal adapter disabled; http_url points at the
|
||||
# host signal-cli daemon. The account phone number stays live-only.
|
||||
enabled: false
|
||||
extra:
|
||||
http_url: http://192.168.0.143:8080
|
||||
mcp_servers:
|
||||
chrome-devtools:
|
||||
command: /usr/local/bin/npx
|
||||
args:
|
||||
- -y
|
||||
- chrome-devtools-mcp@latest
|
||||
- --browserUrl=http://127.0.0.1:9222
|
||||
@@ -1,38 +0,0 @@
|
||||
# Quadlet unit: Chrome DevTools Protocol daemon for the hermes agent (rootless).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/chrome-devtools-protocol.container
|
||||
# Unit name: chrome-devtools-protocol.service (container name: chrome-devtools-protocol)
|
||||
#
|
||||
# Headless Chrome exposing raw CDP over http://127.0.0.1:9222. The hermes
|
||||
# agent's chrome-devtools MCP client (mcp_servers.chrome-devtools in
|
||||
# /data/config.yaml) spawns npx chrome-devtools-mcp with --browserUrl pointing
|
||||
# at this endpoint and drives the browser over it. Both consumers — the Signal
|
||||
# gateway and the webui server — run in the host network namespace, so a
|
||||
# loopback-only bind is reachable by exactly the processes that may drive the
|
||||
# browser and by nobody else on the LAN or WAN.
|
||||
#
|
||||
# Entrypoint override drops the image's socat front (which would bind
|
||||
# 0.0.0.0:9222 in the host namespace and hand browser control to the whole
|
||||
# LAN). --no-sandbox is required because rootless podman grants Chrome no user
|
||||
# namespace for its sandbox; the content processes therefore run as the uid
|
||||
# that already owns this container, capping the blast radius of a hostile page.
|
||||
# Image pinned by digest: upgrades are a deliberate unit edit + redeploy, not a
|
||||
# tag drift at pull time.
|
||||
|
||||
[Unit]
|
||||
Description=Chrome DevTools Protocol daemon (headless-shell) for hermes
|
||||
|
||||
[Container]
|
||||
ContainerName=chrome-devtools-protocol
|
||||
Network=host
|
||||
Image=docker.io/chromedp/headless-shell@sha256:2d349b544a1ea6b5b5fd7c0fe99215ff662339c57407ee2e8c0a11af93516b04
|
||||
Entrypoint=/headless-shell/headless-shell
|
||||
Exec=--no-sandbox --disable-gpu --use-gl=angle --use-angle=swiftshader --remote-debugging-address=127.0.0.1 --remote-debugging-port=9222
|
||||
ShmSize=1073741824
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,33 +0,0 @@
|
||||
# Quadlet unit: Hermes web dashboard (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/hermes-dashboard.container
|
||||
# Unit name: hermes-dashboard.service (container name: hermes-dashboard)
|
||||
#
|
||||
# Management UI for the agent (config, API keys, sessions). Published on
|
||||
# 9119 on network dankclaw_default; Caddy's hermesadmin.franksdank.cloud vhost
|
||||
# reverse-proxies this port directly (no guard chain) because the dashboard
|
||||
# enforces its own basic-auth login from /data/config.yaml
|
||||
# (basic_auth.username + scrypt password_hash), making it exempt-by-construction.
|
||||
# The name has no public DNS record, so the vhost resolves on the LAN only.
|
||||
# Shares /data with the gateway: same agent state, two processes.
|
||||
|
||||
[Unit]
|
||||
Description=Hermes web dashboard container
|
||||
|
||||
[Container]
|
||||
ContainerName=hermes-dashboard
|
||||
Image=localhost/hermes-agent:latest
|
||||
Pull=never
|
||||
Network=dankclaw_default
|
||||
Environment=HERMES_DATA_DIR=/data
|
||||
Environment=HERMES_HOME=/data
|
||||
Volume=/home/dank/Containers/hermes/data:/data
|
||||
PublishPort=9119:9119
|
||||
Exec=bash -c 'hermes dashboard --host 0.0.0.0 --port 9119 --no-open --skip-build'
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,65 +0,0 @@
|
||||
# Quadlet unit: Hermex webui (Hermex) container (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/hermes-webui.container
|
||||
# Unit name: hermes-webui.service (container name: hermes-webui)
|
||||
#
|
||||
# Runs the open-source hermes-webui (github.com/nesquena/hermes-webui) in
|
||||
# foreground mode: bootstrap.py creates .venv inside the mounted checkout,
|
||||
# installs WebUI + agent deps, then execs server.py as a single foreground
|
||||
# process. It reads HERMES_HOME=/data, so it drives the SAME agent state
|
||||
# (memory, skills, sessions) as the gateway container.
|
||||
#
|
||||
# Host networking on purpose: the agent's chrome-devtools MCP client spawns
|
||||
# npx chrome-devtools-mcp with --browserUrl=http://127.0.0.1:9222, and the CDP
|
||||
# daemon binds loopback-only in the host network namespace (see
|
||||
# chrome-devtools-protocol.container). Only processes sharing that namespace
|
||||
# can drive the browser; a bridge-network client could not reach it at all.
|
||||
# The server binds 0.0.0.0:8787 of the host netns, and that is deliberate:
|
||||
# this unit is host-networked (the CDP daemon above needs a shared loopback), so
|
||||
# Caddy cannot reach it by container name and fronts the vhost
|
||||
# hermes.franksdank.cloud with `reverse_proxy host.containers.internal:8787`
|
||||
# (netavark's host address, 169.254.1.2) — not the host LAN IP, which Caddy's
|
||||
# bridge netns cannot reach for a bare host-netns bind; the measured asymmetry
|
||||
# with published ports is recorded in the hermes block of dankclaw/Caddyfile.
|
||||
# A loopback bind was tried on 2026-09-16 for a tighter posture and broke that
|
||||
# vhost, because a bind is also the only way an off-host process can reach a
|
||||
# host-networked container: the LAN address and the tunnel's localhost hop are
|
||||
# both served by the wildcard, and a loopback-only bind serves only the latter.
|
||||
# Controls are the auth chain in front of the vhost plus the password.
|
||||
|
||||
[Unit]
|
||||
Description=Hermes webui (Hermex) container
|
||||
|
||||
[Container]
|
||||
ContainerName=hermes-webui
|
||||
Image=localhost/hermes-agent:latest
|
||||
Pull=never
|
||||
Network=host
|
||||
EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env
|
||||
Environment=HERMES_WEBUI_HOST=0.0.0.0
|
||||
Environment=HERMES_WEBUI_PORT=8787
|
||||
Environment=HERMES_WEBUI_PASSWORD=${HERMES_WEBUI_PASSWORD}
|
||||
Environment=HERMES_HOME=/data
|
||||
# Headless host: never show the first-run wizard (a profile already
|
||||
# exists in /data from the gateway).
|
||||
Environment=HERMES_WEBUI_SKIP_ONBOARDING=1
|
||||
Volume=/home/dank/Containers/hermes/data:/data
|
||||
Volume=/home/dank/Containers/hermes/webui:/webui
|
||||
Exec=python3 /webui/bootstrap.py --foreground
|
||||
# JSON exec form on purpose: this quadlet generator joins a bare HealthCmd
|
||||
# into ONE argv element with \x20 escapes and then shell-executes it — the
|
||||
# python one-liner's quoting would be mangled. A JSON array is passed through
|
||||
# verbatim. Start period covers first-boot pip installs into /webui/.venv.
|
||||
HealthCmd=["python3","-c","import urllib.request;urllib.request.urlopen('http://127.0.0.1:8787/health')"]
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=5
|
||||
HealthStartPeriod=600s
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,45 +0,0 @@
|
||||
# Quadlet unit: Hermes agent gateway (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/hermes.container
|
||||
# Unit name: hermes.service (container name: hermes)
|
||||
#
|
||||
# The messaging gateway (Signal adapter -> the host signal-cli daemon). The
|
||||
# image's default s6 entrypoint would run the interactive TUI, which exits
|
||||
# without a TTY; Exec= below replaces it with the headless gateway command,
|
||||
# spliced by the v1 quadlet generator as trailing args after the image ref
|
||||
# (same splice pattern proven live on authentik-worker). The gateway drops to
|
||||
# the unprivileged hermes user (uid 10000) at runtime; /data is world-writable
|
||||
# on purpose so that unmapped uid can write agent.log and its state there.
|
||||
#
|
||||
# Host networking: the adapter dials the signal-cli daemon, a HOST process
|
||||
# listening on :8080. The compose era dialed the host LAN IP from a bridge
|
||||
# network, where only DNAT'd published ports accept connections to that
|
||||
# address — un-forwarded host ports refuse instantly (verified live
|
||||
# 2026-08-30). A host-network client dials 127.0.0.1:8080 with no such
|
||||
# dependency, and the gateway publishes nothing inbound.
|
||||
|
||||
[Unit]
|
||||
Description=Hermes agent gateway container
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Container]
|
||||
Network=host
|
||||
Image=localhost/hermes-agent:latest
|
||||
Pull=never
|
||||
EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env
|
||||
Environment=HERMES_DATA_DIR=/data
|
||||
Environment=HERMES_HOME=/data
|
||||
Environment=SIGNAL_HTTP_URL=http://127.0.0.1:8080
|
||||
Environment=SIGNAL_ACCOUNT=${SIGNAL_ACCOUNT}
|
||||
Environment=SIGNAL_ALLOWED_USERS=${SIGNAL_ALLOWED_USERS}
|
||||
Volume=/home/dank/Containers/hermes/data:/data
|
||||
Exec=bash -c 'hermes gateway run --force -v'
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,30 +0,0 @@
|
||||
# Loki + a static busybox.
|
||||
#
|
||||
# grafana/loki is distroless: no shell, no wget, so nothing can run inside the
|
||||
# container to check it. This image drops a static busybox into it for the
|
||||
# healthcheck. All Loki behavior is unchanged.
|
||||
#
|
||||
# The healthcheck lives HERE, in exec form, and not in the Quadlet unit: a
|
||||
# unit's HealthCmd reaches podman as CMD-SHELL, which needs /bin/sh — the one
|
||||
# thing this image does not have. Specifying it here is also what lets the
|
||||
# container report its own health instead of looking merely "Up" while wedged.
|
||||
#
|
||||
# Build on dankclaw:
|
||||
# podman build --format docker -t franksdank/loki:latest \
|
||||
# -f dankclaw/loki/Containerfile .
|
||||
#
|
||||
# --format docker is required: podman's default OCI image format has no
|
||||
# HEALTHCHECK field, so a default-format build warns and silently discards the
|
||||
# one below, leaving a container that reports no health at all.
|
||||
|
||||
FROM docker.io/library/alpine:latest AS tools
|
||||
# busybox-static is statically linked: the target images have no dynamic
|
||||
# loader (loki is distroless), so the alpine default /bin/busybox (musl
|
||||
# dynamic) would fail with "no such file or directory".
|
||||
RUN apk add --no-cache busybox-static
|
||||
|
||||
FROM docker.io/grafana/loki:latest
|
||||
COPY --from=tools /bin/busybox.static /usr/local/bin/busybox
|
||||
# Exec form (JSON array): no shell is involved.
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
|
||||
CMD ["/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:3100/ready"]
|
||||
@@ -1,34 +0,0 @@
|
||||
# Quadlet unit: Marreta browser companion — Lightpanda (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/marreta-browser.container
|
||||
# Unit name: marreta-browser.service (container name: marreta_browser)
|
||||
#
|
||||
# Lightpanda is a headless browser that exposes the Chrome DevTools Protocol
|
||||
# over a WebSocket on port 9222. Marreta drives it to render JS-heavy pages
|
||||
# the plain curl fetch cannot. It shares gluetun's network namespace so the
|
||||
# browser's OWN outbound fetches also exit the WireGuard tunnel — this is
|
||||
# required because marreta's PROXY_URL only covers the curl strategy; the
|
||||
# CDP session passes no proxy to the browser, so without netns sharing the
|
||||
# browser would leak the host's real egress IP.
|
||||
#
|
||||
# Marreta reaches it on loopback (same netns): ws://127.0.0.1:9222. No
|
||||
# published port — nothing outside the netns talks to it directly.
|
||||
# See serverAppConfigs-docs/Dankclaw/Marreta.md.
|
||||
|
||||
[Unit]
|
||||
Description=Marreta Lightpanda browser companion container
|
||||
Wants=gluetun.service
|
||||
After=gluetun.service network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=marreta_browser
|
||||
Network=gluetun.container
|
||||
Image=docker.io/lightpanda/browser:nightly
|
||||
|
||||
[Service]
|
||||
TimeoutStartSec=300
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,66 +0,0 @@
|
||||
# Quadlet unit: Marreta content-cleaning proxy (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/marreta.container
|
||||
# Unit name: marreta.service (container name: marreta)
|
||||
#
|
||||
# Marreta is a Laravel app (serversideup/php:8.4-fpm-nginx) that strips
|
||||
# paywalls/tracking from a fetched page. It reuses gluetun's network
|
||||
# namespace (Network=gluetun.container) so ALL of its outbound fetches exit
|
||||
# the same WireGuard tunnel as SearXNG — the curl strategy AND, because the
|
||||
# browser companion shares the same netns, the Lightpanda fetches too
|
||||
# (marreta's PROXY_URL does NOT cover the browser: the CDP connection passes
|
||||
# no proxy, so netns sharing is the only way to keep the browser's egress on
|
||||
# the tunnel).
|
||||
#
|
||||
# Port: the shared netns already has SearXNG on 8080, so marreta must NOT
|
||||
# bind 8080. NGINX_HTTP_PORT=8085 relocates it (the fpm-nginx image does
|
||||
# not bake conf.d/*.conf; the site config is envsubst'd from
|
||||
# ${NGINX_HTTP_PORT} at container start). Gluetun publishes host 8082 ->
|
||||
# 8085 so Caddy can reach it; see dankclaw/searxng/quadlet/gluetun.container.
|
||||
# The app reaches its browser companion over loopback (same netns):
|
||||
# ws://127.0.0.1:9222.
|
||||
#
|
||||
# Secrets (APP_KEY, admin creds) live in /home/dank/Containers/marreta/env
|
||||
# (mode 0600, dank-owned so the rootless unit can read it); this file holds
|
||||
# none. See serverAppConfigs-docs/Dankclaw/Marreta.md.
|
||||
|
||||
[Unit]
|
||||
Description=Marreta content-cleaning proxy container
|
||||
Wants=gluetun.service marreta-browser.service
|
||||
After=gluetun.service marreta-browser.service network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=marreta
|
||||
Network=gluetun.container
|
||||
Image=ghcr.io/manualdousuario/marreta:latest
|
||||
# Relocate off SearXNG's 8080 inside the shared netns.
|
||||
Environment=NGINX_HTTP_PORT=8085
|
||||
Environment=APP_NAME=Marreta
|
||||
Environment=APP_ENV=production
|
||||
Environment=APP_DEBUG=false
|
||||
Environment=APP_URL=https://marreta.franksdank.cloud
|
||||
Environment=APP_LOCALE=en
|
||||
Environment="APP_DESCRIPTION=Paywall-free reading"
|
||||
Environment=BROWSER_WS_ENDPOINT=ws://127.0.0.1:9222
|
||||
Environment=DISABLE_CACHE=false
|
||||
# Secrets expanded from the EnvironmentFile below.
|
||||
Environment=APP_KEY=${MARRETA_APP_KEY}
|
||||
Environment=ADMIN_EMAIL=${MARRETA_ADMIN_EMAIL}
|
||||
Environment=ADMIN_PASSWORD=${MARRETA_ADMIN_PASSWORD}
|
||||
# Named volume: podman auto-creates it and copyups the image's storage/app
|
||||
# ownership (www-data), so the app can write its SQLite DB + cache.
|
||||
Volume=marreta_storage:/var/www/html/storage/app
|
||||
HealthCmd=["curl","-fsS","-o","/dev/null","http://127.0.0.1:8085/up"]
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=5
|
||||
HealthStartPeriod=120s
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=/home/dank/Containers/marreta/env
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,54 +0,0 @@
|
||||
# Quadlet unit: alertmanager (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/alertmanager.container
|
||||
# Unit name: alertmanager.service (container name: alertmanager)
|
||||
#
|
||||
# The delivery half of the LLM alert rules. Until 2026-09-17 the rules in
|
||||
# shared/prometheus/rules/llm-fleet.yml evaluated and reached nobody — a
|
||||
# firing alert was visible in Prometheus and on the Grafana dashboard and
|
||||
# nowhere else, and an undelivered alert looks identical to a quiet one.
|
||||
# This unit closes that gap: Prometheus pushes to alertmanager:9093 over the
|
||||
# dankclaw_default network, and the single receiver webhooks the host-side
|
||||
# signal-relay user unit, which hands the text to signal-cli under Frank's
|
||||
# existing Signal account.
|
||||
#
|
||||
# PUBLISHING. 127.0.0.1 only, like every other observability unit: the
|
||||
# unauthenticated Alertmanager API (lifecycle endpoints are always on in
|
||||
# this binary — there is no flag to gate them) has no business on a host
|
||||
# interface, and no off-host consumer needs it. Caddy gains no vhost for
|
||||
# it; the homepage timer probes loopback.
|
||||
#
|
||||
# CONFIG. /opt/alertmanager/config/alertmanager.yml is installed from
|
||||
# shared/prometheus/alertmanager.yml by deploy-remote.sh on every deploy, so
|
||||
# the repo copy is the source of truth. STATE lives in /opt/alertmanager/data.
|
||||
#
|
||||
# Rootless: container root maps to the invoking user (dank) in this user
|
||||
# namespace and dank owns /opt/alertmanager/data. The image's default user
|
||||
# (nobody) maps to a subuid that cannot write it — same User=0:0 as prometheus.
|
||||
|
||||
[Unit]
|
||||
Description=Alertmanager alert delivery (Dankclaw)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Container]
|
||||
ContainerName=alertmanager
|
||||
Image=docker.io/prom/alertmanager:latest
|
||||
Network=dankclaw_default
|
||||
User=0:0
|
||||
Exec=--config.file=/etc/alertmanager/alertmanager.yml --storage.path=/alertmanager
|
||||
PublishPort=127.0.0.1:9093:9093
|
||||
Volume=/opt/alertmanager/config:/etc/alertmanager
|
||||
Volume=/opt/alertmanager/data:/alertmanager
|
||||
HealthCmd=wget --spider -q http://localhost:9093/-/healthy
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=30s
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,63 +0,0 @@
|
||||
# Quadlet unit: alloy (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/alloy.container
|
||||
# Unit name: alloy.service (container name: alloy)
|
||||
#
|
||||
# Log shipper. This replaces the compose service of the same name in
|
||||
# dankclaw/compose.yaml ("Observability" block).
|
||||
#
|
||||
# IMAGE. franksdank/alloy is built locally by deploy-remote.sh
|
||||
# (build_compose_image) from dankclaw/alloy/Containerfile: upstream
|
||||
# grafana/alloy plus a static busybox for the healthcheck. Pull=never keeps a
|
||||
# boot from reaching for that local tag at a registry.
|
||||
#
|
||||
# NETWORK. It joins dankclaw_default so its config's loki.write endpoint
|
||||
# resolves the container name loki:3100.
|
||||
#
|
||||
# THE UI PORT EXISTS ON PURPOSE. Alloy's UI listens on the container's own
|
||||
# 127.0.0.1 by default and cannot be moved from the config file (alloy 1.x
|
||||
# refuses the ui block), so the compose service had no reachable health
|
||||
# endpoint at all: the homepage status card for alloy reported "down"
|
||||
# continuously and there was nothing for a deploy to wait on. The command
|
||||
# below therefore moves the UI to the container's wildcard address and the
|
||||
# port is published on host loopback only, which puts alloy in the same class
|
||||
# as the other three: reachable from the host, by Caddy over the private
|
||||
# network, and by nothing off-host. Alloy has no authentication, which is
|
||||
# exactly why it is not published wider.
|
||||
#
|
||||
# SOURCES. The host journald socket and the host log/container-storage trees
|
||||
# are bind-mounted read-only; Alloy tails them and pushes to Loki. The config
|
||||
# lives at /opt/alloy/config/config.alloy, installed from
|
||||
# shared/alloy/config.alloy on every deploy.
|
||||
#
|
||||
# ORDERING. Wants/After loki.service: without a reachable Loki, Alloy's write
|
||||
# client fails and its log pipeline backs up.
|
||||
|
||||
[Unit]
|
||||
Description=Alloy log shipper (Dankclaw)
|
||||
After=network-online.target loki.service
|
||||
Wants=network-online.target loki.service
|
||||
|
||||
[Container]
|
||||
ContainerName=alloy
|
||||
Image=localhost/franksdank/alloy:latest
|
||||
Pull=never
|
||||
Network=dankclaw_default
|
||||
Exec=run --server.http.listen-addr=0.0.0.0:12345 /etc/alloy/config.alloy
|
||||
PublishPort=127.0.0.1:12345:12345
|
||||
Volume=/opt/alloy/config:/etc/alloy
|
||||
Volume=/run/systemd/journal/socket:/run/systemd/journal/socket:ro
|
||||
Volume=/var/log:/var/log:ro
|
||||
Volume=/var/lib/containers:/var/lib/containers:ro
|
||||
HealthCmd=/usr/local/bin/busybox wget --spider -q http://localhost:12345/-/healthy
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=30s
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,79 +0,0 @@
|
||||
# Quadlet unit: grafana (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/grafana.container
|
||||
# Unit name: grafana.service (container name: grafana)
|
||||
#
|
||||
# Dashboards. This replaces the compose service of the same name in
|
||||
# dankclaw/compose.yaml ("Observability" block).
|
||||
#
|
||||
# PUBLISHING. Bound to 127.0.0.1 for the same reason as prometheus: the browser
|
||||
# path is https://grafana.franksdank.cloud (Caddy → Anubis → Authentik outpost →
|
||||
# grafana:3000 over dankclaw_default) and the host-side homepage status timer
|
||||
# probes 127.0.0.1:3002. Publishing on 0.0.0.0 as the compose service did left
|
||||
# an off-host path into Grafana that skips the guard chain entirely.
|
||||
#
|
||||
# AUTH. Grafana is an OAuth2 relying party against Authentik (OAuth2Provider
|
||||
# "grafana", confidential client). The guard chain fronts unauthenticated
|
||||
# traffic; the login itself happens through the Authentik button inside
|
||||
# Grafana. The client secret is NOT written here: it is expanded by systemd
|
||||
# from the root .env (EnvironmentFile in [Service], never in [Container], so
|
||||
# the container receives exactly this one secret and not the whole file).
|
||||
# Without that .env entry the unit will not start — the same dependency every
|
||||
# other rendered-secret deploy has.
|
||||
#
|
||||
# STATE. /opt/grafana/data holds grafana.db. /opt/grafana/provisioning is
|
||||
# mounted read-only and was EMPTY as of 2026-09-11, and grafana.db reported
|
||||
# zero datasources and zero dashboards: this Grafana is an empty install whose
|
||||
# only configured account is the local admin. Nothing here queries Prometheus
|
||||
# or Loki yet.
|
||||
|
||||
[Unit]
|
||||
Description=Grafana dashboards (Dankclaw)
|
||||
After=network-online.target prometheus.service
|
||||
Wants=network-online.target prometheus.service
|
||||
|
||||
[Container]
|
||||
ContainerName=grafana
|
||||
Image=docker.io/grafana/grafana:latest
|
||||
Network=dankclaw_default
|
||||
User=0:0
|
||||
PublishPort=127.0.0.1:3002:3000
|
||||
Environment=GF_SERVER_ROOT_URL=https://grafana.franksdank.cloud
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_ENABLED=true
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_NAME=Authentik
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_CLIENT_ID=grafana-21ca190cde75d60f
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${GRAFANA_OAUTH_CLIENT_SECRET}
|
||||
# Global authorize path: this Authentik build resolves the provider by the
|
||||
# client_id query parameter — /application/o/<slug>/authorize/ does not exist.
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://auth.franksdank.cloud/application/o/authorize/
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_TOKEN_URL=https://auth.franksdank.cloud/application/o/token/
|
||||
# ?api=1 = introspection-style userinfo: returns custom claims (groups) too.
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_API_URL=https://auth.franksdank.cloud/application/o/userinfo/?api=1
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=false
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_AUTO_UPSERT=true
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_ORG=true
|
||||
Environment=GF_AUTH_GENERIC_OAUTH_ORG_ID=1
|
||||
Volume=/opt/grafana/data:/var/lib/grafana
|
||||
Volume=/opt/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
HealthCmd=wget --spider -q http://localhost:3000/api/health
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=60s
|
||||
|
||||
[Service]
|
||||
# systemd-side only: supplies ${GRAFANA_OAUTH_CLIENT_SECRET} for the expansion
|
||||
# above. The container never receives this file.
|
||||
#
|
||||
# It is the per-service env file the deployer seeds from dankclaw/.env — the
|
||||
# fleet convention (caddy, pihole, copyparty, romm and seerr all read one).
|
||||
# The repo-root .env that the compose service happened to pick up is a gluetun
|
||||
# env file: it carries HOME=/root and the WireGuard keys, and importing it into
|
||||
# this service makes podman look for its configuration under /root, which it
|
||||
# cannot read, so the unit fails to start. Seed the value into dankclaw/.env.
|
||||
EnvironmentFile=/home/dank/Containers/grafana/env
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,57 +0,0 @@
|
||||
# Quadlet unit: loki (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/loki.container
|
||||
# Unit name: loki.service (container name: loki)
|
||||
#
|
||||
# Log store. This replaces the compose service of the same name in
|
||||
# dankclaw/compose.yaml ("Observability" block).
|
||||
#
|
||||
# IMAGE. franksdank/loki is built locally by deploy-remote.sh
|
||||
# (build_compose_image) from dankclaw/loki/Containerfile: upstream
|
||||
# grafana/loki plus a static busybox, because loki is distroless and the
|
||||
# healthcheck has nothing to run inside it. Pull=never keeps a boot from
|
||||
# trying to fetch that local tag from a registry. The localhost/ prefix is
|
||||
# required — this host's quadlet path cannot resolve a registry-prefixed ref
|
||||
# from local storage at create time.
|
||||
#
|
||||
# PUBLISHING. Bound to 127.0.0.1: Alloy pushes to loki:3100 over
|
||||
# dankclaw_default, Caddy proxies the guarded vhost to loki:3100 the same way,
|
||||
# and the homepage status timer probes 127.0.0.1:3100. Loki has
|
||||
# auth_enabled: false, so the compose service's 0.0.0.0 binding was an
|
||||
# unauthenticated log-ingest and read API on every host interface.
|
||||
#
|
||||
# CONFIG. /opt/loki/config/loki-config.yaml is installed from
|
||||
# shared/loki/loki-config.yaml on every deploy (single-binary, filesystem
|
||||
# storage, 30-day retention). Chunks and rules live in /opt/loki/data.
|
||||
#
|
||||
# Rootless: container root maps to the invoking user (dank), who owns
|
||||
# /opt/loki/data; the image's default user (10001) maps to a subuid that
|
||||
# cannot write it.
|
||||
|
||||
[Unit]
|
||||
Description=Loki log store (Dankclaw)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Container]
|
||||
ContainerName=loki
|
||||
Image=localhost/franksdank/loki:latest
|
||||
Pull=never
|
||||
Network=dankclaw_default
|
||||
User=0:0
|
||||
Exec=-config.file=/etc/loki/loki-config.yaml
|
||||
PublishPort=127.0.0.1:3100:3100
|
||||
Volume=/opt/loki/config:/etc/loki
|
||||
Volume=/opt/loki/data:/loki
|
||||
# No HealthCmd here on purpose. A unit's HealthCmd reaches podman as
|
||||
# CMD-SHELL, which means `/bin/sh -c <command>` — and this image is
|
||||
# distroless, with no shell at all. The healthcheck therefore ships in the
|
||||
# image (dankclaw/loki/Containerfile) in exec form, where it also carries its
|
||||
# interval, timeout, start period and retry count.
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,54 +0,0 @@
|
||||
# Quadlet unit: node-exporter (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/node-exporter.container
|
||||
# Unit name: node-exporter.service (container name: node-exporter)
|
||||
#
|
||||
# Host metrics source for Prometheus. This replaces the compose service of the
|
||||
# same name in dankclaw/compose.yaml ("Observability" block).
|
||||
#
|
||||
# HOST NAMESPACE. It must run in the host network namespace to report the
|
||||
# host's own interfaces/connections, and --pid=host for the processes and
|
||||
# cgroups collectors. "localhost" inside the container is therefore the host's
|
||||
# loopback, which is what the homepage status timer probes.
|
||||
#
|
||||
# LISTEN ADDRESSES. Until 2026-09-11 this bound :9100 on every interface. It
|
||||
# now binds exactly two: loopback for the homepage status probe, and the LAN
|
||||
# address because Prometheus — which runs in a container on dankclaw_default —
|
||||
# scrapes http://192.168.0.143:9100 (shared/prometheus/prometheus.yml,
|
||||
# job "node-dankclaw"). node_exporter itself has no authentication, so keeping
|
||||
# the wildcard bind off the table matters: this is the second half of the
|
||||
# pairing that removes the unauthenticated observability surface, and both
|
||||
# consumers are accounted for. Dankserver runs its own copy of this container
|
||||
# (scraped by the "node-dankserver" job) — since 2026-09-16 as a Quadlet unit in
|
||||
# the same shape (dankserver/observability/quadlet/), because the compose-only
|
||||
# version it replaced had been dead for a day and a half before anyone noticed.
|
||||
#
|
||||
# MOUNTS. The container reads the host's /proc, /sys and / as /host/* read-only
|
||||
# and is pointed at them with --path.*; it writes nothing anywhere.
|
||||
|
||||
[Unit]
|
||||
Description=Prometheus node exporter (Dankclaw host metrics)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Container]
|
||||
ContainerName=node-exporter
|
||||
Image=docker.io/prom/node-exporter:latest
|
||||
Network=host
|
||||
PodmanArgs=--pid=host
|
||||
Exec=--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/rootfs --web.listen-address=127.0.0.1:9100 --web.listen-address=192.168.0.143:9100
|
||||
Volume=/proc:/host/proc:ro
|
||||
Volume=/sys:/host/sys:ro
|
||||
Volume=/:/rootfs:ro
|
||||
HealthCmd=wget --spider -q http://localhost:9100/metrics
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=30s
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,64 +0,0 @@
|
||||
# Quadlet unit: prometheus (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/prometheus.container
|
||||
# Unit name: prometheus.service (container name: prometheus)
|
||||
#
|
||||
# The homelab metrics store. This replaces the compose service of the same
|
||||
# name in dankclaw/compose.yaml ("Observability" block): these five containers
|
||||
# were started by hand with podman-compose and never came back after a reboot,
|
||||
# which the 2026-09-11 reboot for the amd_iommu=off change made obvious.
|
||||
#
|
||||
# PUBLISHING. The published port is bound to 127.0.0.1 deliberately. Prometheus
|
||||
# is reached by Caddy over the dankclaw_default network (the Caddyfile's
|
||||
# post-auth handler proxies to prometheus:9090) and by the host-side homepage
|
||||
# status timer over 127.0.0.1. The compose service published on 0.0.0.0, which
|
||||
# put an UNAUTHENTICATED metrics API — with the lifecycle endpoint enabled —
|
||||
# on every host interface: a bypass around the Anubis + Authentik guard on
|
||||
# prometheus.franksdank.cloud. No off-host consumer needs it.
|
||||
#
|
||||
# CONFIG. /opt/prometheus/config/prometheus.yml is installed from
|
||||
# shared/prometheus/prometheus.yml by deploy-remote.sh on every deploy, so the
|
||||
# repo copy is the source of truth. STATE (30-day retention) lives in
|
||||
# /opt/prometheus/data.
|
||||
#
|
||||
# Rootless: container root maps to the invoking user (dank) in this user
|
||||
# namespace and dank owns /opt/prometheus/data. The image's default user
|
||||
# (nobody) maps to a subuid that cannot write it.
|
||||
#
|
||||
# RULES. Five LLM alert rules and one recording rule load from
|
||||
# /etc/prometheus/rules/ (installed from shared/prometheus/rules/ by the
|
||||
# deployer). Delivery went live 2026-09-17: the alerting block in
|
||||
# prometheus.yml points at alertmanager:9093, which webhooks every firing
|
||||
# group to the host-side Signal relay (192.168.0.143:18093, reached from
|
||||
# the container as host.containers.internal), which sends through signal-cli account 318311.
|
||||
# The Grafana dashboard's ALERTS panel remains the second surface.
|
||||
#
|
||||
# The config volume is also why rules belong in git: editing them in the UI is
|
||||
# lost on the next deploy, since the deployer overwrites the whole directory.
|
||||
|
||||
[Unit]
|
||||
Description=Prometheus metrics store (Dankclaw)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Container]
|
||||
ContainerName=prometheus
|
||||
Image=docker.io/prom/prometheus:latest
|
||||
Network=dankclaw_default
|
||||
User=0:0
|
||||
Exec=--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --storage.tsdb.retention.time=30d --web.enable-lifecycle
|
||||
PublishPort=127.0.0.1:9090:9090
|
||||
Volume=/opt/prometheus/config:/etc/prometheus
|
||||
Volume=/opt/prometheus/data:/prometheus
|
||||
HealthCmd=wget --spider -q http://localhost:9090/-/healthy
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=30s
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,96 +0,0 @@
|
||||
# Quadlet unit: open-webui (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/open-webui.container
|
||||
# Unit name: open-webui.service (container name: open-webui)
|
||||
#
|
||||
# Chat UI for the LLM fleet. It runs on the CPU image (dankclaw is CPU-only, so
|
||||
# the :cuda image would waste a large pull for a GPU it cannot use) and points
|
||||
# its OpenAI connection at the live llama.cpp backend on this host via the
|
||||
# host LAN IP (192.168.0.143:7770, model frank-default-large). It publishes 3005
|
||||
# (3000 is Gitea's HTTP port). Data persists under
|
||||
# /home/dank/Containers/open-webui/data. See serverAppConfigs-docs/Dankclaw/OpenWebUI.md.
|
||||
#
|
||||
# Network history (2026-08-22): this unit used Network=slirp4netns. When the
|
||||
# slirp4netns helper process died, the container kept running but lost its tap
|
||||
# interface: a zombie that accepted TCP and reset every request, and chat
|
||||
# served empty responses for ~2 days with no error anywhere. It now runs on a
|
||||
# pinned bridge network (open-webui.network), which has no per-container
|
||||
# helper to die. The LLM is reached via the host LAN IP (192.168.0.143):
|
||||
# the bridge gateway lives in the rootless netns, not the host netns where
|
||||
# llama-server runs. The healthcheck probes /health (app liveness) and
|
||||
# requires a non-loopback interface (network liveness).
|
||||
|
||||
[Unit]
|
||||
Description=open-webui chat UI container
|
||||
After=network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=open-webui
|
||||
Network=open-webui.network
|
||||
Image=ghcr.io/open-webui/open-webui:main
|
||||
PublishPort=3005:8080
|
||||
Environment=OPENAI_API_BASE_URL=http://192.168.0.143:7770
|
||||
Environment=ANONYMIZED_TELEMETRY=False
|
||||
Environment=DO_NOT_TRACK=True
|
||||
|
||||
# SSO via authentik (OIDC authorization-code flow, 2026-09-24). The OAuth
|
||||
# client (application slug "open-webui") lives in authentik; the client
|
||||
# SECRET stays off-repo in the git-ignored host env file (mode 600). The
|
||||
# container reaches the IdP over the public URL: Caddy publishes 0.0.0.0:443
|
||||
# and Pi-hole resolves auth.franksdank.cloud to the host LAN IP, so the
|
||||
# server-side code exchange works from this bridge. Roles come from the
|
||||
# "Open WebUI roles" scope mapping (entitlements "Open WebUI Users" /
|
||||
# "Open WebUI Admins"); OAUTH_MERGE_ACCOUNTS_BY_EMAIL folds SSO logins into
|
||||
# the existing local account (frank8465@protonmail.com). The OAuth vars are
|
||||
# read directly from the environment at runtime (the per-key config DB holds
|
||||
# only Admin-Panel overrides, so nothing is seeded on the live data dir).
|
||||
# NOTE: the OAUTH_SCOPES value MUST stay quoted — systemd splits unquoted
|
||||
# Environment= values on whitespace (unquoted, only "openid" survived).
|
||||
EnvironmentFile=/home/dank/Containers/open-webui/env
|
||||
Environment=OAUTH_CLIENT_ID=KmtzKc0bjWjaIFeM0puYudI1CLXR2p0jnPCSf4N0
|
||||
Environment=OAUTH_PROVIDER_NAME=authentik
|
||||
Environment=OPENID_PROVIDER_URL=https://auth.franksdank.cloud/application/o/open-webui/.well-known/openid-configuration
|
||||
Environment=OPENID_REDIRECT_URI=https://ai.franksdank.cloud/oauth/oidc/callback
|
||||
Environment=WEBUI_URL=https://ai.franksdank.cloud
|
||||
Environment=ENABLE_OAUTH_SIGNUP=true
|
||||
Environment=OAUTH_MERGE_ACCOUNTS_BY_EMAIL=true
|
||||
Environment="OAUTH_SCOPES=openid email profile roles"
|
||||
Environment=ENABLE_OAUTH_ROLE_MANAGEMENT=true
|
||||
# SSO-only (2026-09-24): the OIDC round-trip is verified, so the local
|
||||
# username/password path is retired. ENABLE_LOGIN_FORM hides the form;
|
||||
# ENABLE_PASSWORD_AUTH refuses password logins entirely, so the only way in
|
||||
# is the authentik OIDC flow (which folds into the local account by email).
|
||||
# The per-key config DB override (ui.enable_login_form) is flipped to false
|
||||
# in the same change — the DB wins over env, so both must agree.
|
||||
Environment=ENABLE_LOGIN_FORM=false
|
||||
Environment=ENABLE_PASSWORD_AUTH=false
|
||||
|
||||
# Web search -> SearXNG, reached over the host-published port (NOT the
|
||||
# search.franksdank.cloud vhost): the Anubis->Authentik guard chain is a
|
||||
# browser mechanism and a plain HTTP client cannot solve it. The internal
|
||||
# port bypasses Caddy entirely. These env vars SEED the per-key config DB on
|
||||
# first run (fill-missing only); once the keys exist in /app/backend/data the
|
||||
# DB is authoritative, so this block documents the intent and makes a fresh
|
||||
# data dir come up wired. SearXNG's limiter exempts this source (the bridge
|
||||
# egress 10.89.x is in pass_ip), so the 4-req/IP/hour JSON cap never applies.
|
||||
Environment=ENABLE_WEB_SEARCH=True
|
||||
Environment=WEB_SEARCH_ENGINE=searxng
|
||||
Environment=SEARXNG_QUERY_URL=http://192.168.0.143:8081/search
|
||||
Environment=SEARXNG_LANGUAGE=all
|
||||
Environment=WEB_SEARCH_RESULT_COUNT=6
|
||||
Volume=/home/dank/Containers/open-webui/data:/app/backend/data
|
||||
|
||||
# App liveness (/health) AND network liveness (a non-loopback interface must
|
||||
# exist; a dead tap leaves only "lo" behind).
|
||||
HealthCmd=CMD-SHELL python -c 'import os,urllib.request,sys; urllib.request.urlopen("http://127.0.0.1:8080/health",timeout=5).read(); sys.exit(0 if any(i!="lo" for i in os.listdir("/sys/class/net")) else 1)'
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=300s
|
||||
|
||||
[Service]
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,31 +0,0 @@
|
||||
# Quadlet network: private bridge for open-webui (rootless).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/open-webui.network
|
||||
# Unit name: open-webui-network.service (podman network name: systemd-open-webui)
|
||||
#
|
||||
# Why a bridge instead of slirp4netns (2026-08-22 incident):
|
||||
# slirp4netns runs a user-space helper process per container. When that
|
||||
# helper died, the container kept running but lost its tap interface,
|
||||
# becoming a zombie that accepted TCP and reset every request. chat
|
||||
# served empty responses for ~2 days and nothing logged an error.
|
||||
# A kernel bridge has no per-container helper, so the failure mode
|
||||
# cannot occur.
|
||||
#
|
||||
# The subnet is pinned so the container's own IP (10.89.3.2) is
|
||||
# deterministic. Note the LLM backend is NOT reached via this gateway:
|
||||
# the gateway lives in the rootless netns, while llama-server runs in
|
||||
# the host netns. The host's LAN IP (192.168.0.143) reaches it through
|
||||
# the rootless NAT. host.containers.internal is a slirp4netns-only name
|
||||
# and does not resolve on a bridge.
|
||||
#
|
||||
# Subnet registry on dankclaw (rootless):
|
||||
# 10.88.0.0/16 default "podman" bridge
|
||||
# 10.89.0.0/24 gitea (systemd-gitea)
|
||||
# 10.89.1.0/24 compose stack (dankclaw_default)
|
||||
# 10.89.2.0/24 searxng (systemd-searxng)
|
||||
# 10.89.3.0/24 open-webui (systemd-open-webui) <- this file
|
||||
|
||||
[Network]
|
||||
Driver=bridge
|
||||
Subnet=10.89.3.0/24
|
||||
Gateway=10.89.3.1
|
||||
@@ -1,47 +0,0 @@
|
||||
# A records for the flat franksdank.name scheme (dnsmasq addn-hosts format:
|
||||
# <IP> <name> [<name> ...]). Served by the Pi-hole quadlet (FTL is a
|
||||
# dnsmasq fork): the quadlet mounts this file into /etc/pihole/hosts/ read-only.
|
||||
#
|
||||
# Caddy is consolidated on dankclaw (192.168.0.143): it fronts every service
|
||||
# on both hosts, so every name resolves to that one IP. Keep in sync with the
|
||||
# Caddyfile (tests/pihole.bats cross-checks this).
|
||||
#
|
||||
# 192.168.0.143 = dankclaw (runs the Caddy container + the AI/dev stack)
|
||||
# 192.168.0.235 = dankserver (media stack; reached by Caddy via published
|
||||
# LAN ports — no DNS records point here)
|
||||
|
||||
# --- Homepage (naked domain + www) + legacy names ---
|
||||
192.168.0.143 franksdank.cloud
|
||||
192.168.0.143 www.franksdank.cloud
|
||||
192.168.0.143 dankclaw.franksdank.cloud
|
||||
192.168.0.143 dankserver.franksdank.cloud
|
||||
|
||||
# --- Authentik (SSO / OIDC provider) ---
|
||||
192.168.0.143 auth.franksdank.cloud
|
||||
|
||||
# --- Media services (dankserver, proxied) ---
|
||||
192.168.0.143 plex.franksdank.cloud
|
||||
192.168.0.143 plex.franksdank.casa
|
||||
192.168.0.143 books.franksdank.cloud
|
||||
192.168.0.143 romm.franksdank.cloud
|
||||
192.168.0.143 files.franksdank.cloud
|
||||
192.168.0.143 jelly.franksdank.cloud
|
||||
192.168.0.143 radarr.franksdank.cloud
|
||||
192.168.0.143 lidarr.franksdank.cloud
|
||||
192.168.0.143 readarr.franksdank.cloud
|
||||
192.168.0.143 prowlarr.franksdank.cloud
|
||||
192.168.0.143 qbittorrent.franksdank.cloud
|
||||
|
||||
# --- AI / search / dev services (dankclaw, local) ---
|
||||
192.168.0.143 search.franksdank.cloud
|
||||
192.168.0.143 comfyui.franksdank.cloud
|
||||
192.168.0.143 ai.franksdank.cloud
|
||||
192.168.0.143 hermes.franksdank.cloud
|
||||
192.168.0.143 hermesadmin.franksdank.cloud
|
||||
192.168.0.143 gitea.franksdank.cloud
|
||||
192.168.0.143 grafana.franksdank.cloud
|
||||
192.168.0.143 prometheus.franksdank.cloud
|
||||
192.168.0.143 loki.franksdank.cloud
|
||||
192.168.0.143 pihole.franksdank.cloud
|
||||
192.168.0.143 seerr.franksdank.cloud
|
||||
192.168.0.143 marreta.franksdank.cloud
|
||||
@@ -1,18 +0,0 @@
|
||||
# Pi-hole container environment (dankclaw).
|
||||
# Copy to /home/dank/Containers/pihole/env and fill real values.
|
||||
# The real env file is never committed (it holds the web UI password).
|
||||
|
||||
# The host IP the container answers for (also shown in the web UI).
|
||||
ServerIP=192.168.0.143
|
||||
|
||||
# Timezone for the web UI and logs.
|
||||
TZ=America/Los_Angeles
|
||||
|
||||
# Pinned upstream resolvers. The slirp netns has no useful resolv.conf, so
|
||||
# pin them (mirrors the retired dnsmasq.conf server= lines).
|
||||
DNS1=1.1.1.1
|
||||
DNS2=1.0.0.1
|
||||
|
||||
# Web UI password (the shared family password; the same credential as
|
||||
# CADDY_BASICAUTH_HASH in the host .env).
|
||||
WEBPASSWORD=changeme
|
||||
@@ -1,42 +0,0 @@
|
||||
# Quadlet unit: Pi-hole LAN DNS (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/pihole.container
|
||||
# Unit name: pihole.service (container name: pihole)
|
||||
#
|
||||
# The LAN's authoritative DNS for the flat franksdank.cloud / franksdank.casa
|
||||
# name scheme. It retired the dnsmasq quadlet (2026-08-24): Pi-hole FTL is a
|
||||
# dnsmasq fork, so the addn-hosts record file carries over unchanged and is
|
||||
# mounted into /etc/pihole/hosts/ (the FTL hostsdir) read-only. It serves one A
|
||||
# record per Caddy vhost and forwards everything else to the pinned upstream
|
||||
# resolvers (DNS1/DNS2 in the env file). The SURFboard G36 router hands this
|
||||
# server out to DHCP clients (DNS statically configured -> 192.168.0.143),
|
||||
# so no client needs per-device config.
|
||||
#
|
||||
# Runs on slirp4netns with port 53 (DNS) and 8083 (web UI) published to the
|
||||
# LAN (the same pattern as open-webui). Config + state mount from
|
||||
# /home/dank/Containers/pihole. The web UI password is WEBPASSWORD in the env
|
||||
# file. See serverAppConfigs-docs/Infrastructure/LAN-DNS.md.
|
||||
|
||||
[Unit]
|
||||
Description=Pi-hole LAN DNS container
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Container]
|
||||
ContainerName=pihole
|
||||
Image=docker.io/pihole/pihole:latest
|
||||
Network=slirp4netns
|
||||
PublishPort=0.0.0.0:53:53/udp
|
||||
PublishPort=0.0.0.0:53:53/tcp
|
||||
PublishPort=0.0.0.0:8083:80/tcp
|
||||
EnvironmentFile=/home/dank/Containers/pihole/env
|
||||
Volume=/home/dank/Containers/pihole/etc-pihole:/etc/pihole
|
||||
Volume=/home/dank/Containers/pihole/etc-dockers:/etc/dockers
|
||||
Volume=/home/dank/Containers/pihole/addn-hosts:/etc/pihole/hosts/addn-hosts:ro
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,8 +0,0 @@
|
||||
# Rootless DNS must bind port 53: lower the unprivileged-port threshold so a
|
||||
# rootless container may publish 0.0.0.0:53 (80/443 stay allowed; 53 < 80).
|
||||
# Single source of truth for net.ipv4.ip_unprivileged_port_start on dankclaw;
|
||||
# deploy_pihole installs it to /etc/sysctl.d/ and applies it with
|
||||
# `sysctl --system` (the same code path boot uses via systemd-sysctl).
|
||||
# A later-sorted drop-in that reset this to 80 took LAN DNS down after the
|
||||
# 2026-08-26 reboot — deploy_pihole neutralizes such conflicts.
|
||||
net.ipv4.ip_unprivileged_port_start=53
|
||||
@@ -1,40 +0,0 @@
|
||||
# Quadlet unit: Gluetun VPN tunnel (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/gluetun.container
|
||||
# Unit name: gluetun.service (container name: gluetun)
|
||||
#
|
||||
# Creates the WireGuard tunnel to ProtonVPN (US-WA). SearXNG reuses this
|
||||
# container's network namespace (see searxng.container), so its outbound search
|
||||
# traffic exits here. The WireGuard credentials live in /opt/searxng/.env
|
||||
# (mode 0600); this file holds no secrets. See serverAppConfigs-docs/Dankclaw/Gluetun.md.
|
||||
|
||||
[Unit]
|
||||
Description=Gluetun VPN tunnel container
|
||||
Wants=searxng-network.service
|
||||
After=searxng-network.service network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=gluetun
|
||||
Network=searxng.network
|
||||
Image=docker.io/qmcgaw/gluetun:latest
|
||||
AddCapability=NET_ADMIN
|
||||
AddDevice=/dev/net/tun:/dev/net/tun
|
||||
Environment=VPN_SERVICE_PROVIDER=${VPN_SERVICE_PROVIDER}
|
||||
Environment=VPN_TYPE=${VPN_TYPE}
|
||||
Environment=WIREGUARD_PRIVATE_KEY=${WIREGUARD_PRIVATE_KEY}
|
||||
Environment=WIREGUARD_ADDRESSES=${WIREGUARD_ADDRESSES}
|
||||
Environment=WIREGUARD_PUBLIC_KEY=${WIREGUARD_PUBLIC_KEY}
|
||||
Environment=WIREGUARD_ENDPOINT_IP=${WIREGUARD_ENDPOINT_IP}
|
||||
Environment=WIREGUARD_ENDPOINT_PORT=${WIREGUARD_ENDPOINT_PORT}
|
||||
Environment=WIREGUARD_ALLOWED_IPS=${WIREGUARD_ALLOWED_IPS}
|
||||
Environment=WIREGUARD_PERSISTENT_KEEPALIVE_INTERVAL=${WIREGUARD_PERSISTENT_KEEPALIVE_INTERVAL}
|
||||
PublishPort=8081:8080
|
||||
PublishPort=8082:8085
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=/opt/searxng/.env
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,28 +0,0 @@
|
||||
# Quadlet unit: Valkey cache for SearXNG (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/searxng-valkey.container
|
||||
# Unit name: searxng-valkey.service (container name: searxng-valkey)
|
||||
#
|
||||
# The limiter's rate-limit state store. Joins the searxng network and is
|
||||
# reachable by the name "valkey" (settings.yml points there). Data persists in
|
||||
# /opt/searxng/valkey. See serverAppConfigs-docs/Dankclaw/Valkey.md.
|
||||
|
||||
[Unit]
|
||||
Description=SearXNG Valkey cache container
|
||||
Wants=searxng-network.service
|
||||
After=searxng-network.service network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=searxng-valkey
|
||||
Network=searxng.network
|
||||
NetworkAlias=valkey
|
||||
Image=docker.io/valkey/valkey:9-alpine
|
||||
Volume=/opt/searxng/valkey:/data
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=/opt/searxng/.env
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,30 +0,0 @@
|
||||
# Quadlet unit: SearXNG search engine (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/searxng.container
|
||||
# Unit name: searxng.service (container name: searxng)
|
||||
#
|
||||
# Reuses gluetun's network namespace (Network=gluetun.container) so its
|
||||
# outbound traffic exits the WireGuard tunnel. It publishes no port of its own;
|
||||
# the port lives on gluetun. It reaches valkey by name on the searxng network.
|
||||
# Config mounts from /opt/searxng/config. See serverAppConfigs-docs/Dankclaw/SearXNG.md.
|
||||
|
||||
[Unit]
|
||||
Description=SearXNG search engine container
|
||||
Wants=gluetun.service searxng-valkey.service
|
||||
After=gluetun.service searxng-valkey.service network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=searxng
|
||||
Network=gluetun.container
|
||||
Image=ghcr.io/searxng/searxng:latest
|
||||
Environment=SEARXNG_SECRET=${SEARXNG_SECRET}
|
||||
Environment=SEARXNG_BASE_URL=${SEARXNG_BASE_URL}
|
||||
Volume=/opt/searxng/config:/etc/searxng
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=/opt/searxng/.env
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,16 +0,0 @@
|
||||
# Quadlet network: private bridge for the SearXNG stack (rootless).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/searxng.network
|
||||
# Unit name: searxng-network.service (podman network name: systemd-searxng)
|
||||
#
|
||||
# DNS is enabled by default for quadlet networks. This matters: the automatic
|
||||
# "podman" default network has dns_enabled=false, so container names would not
|
||||
# resolve. searxng (which reuses gluetun's netns) reaches valkey by the name
|
||||
# "valkey" on this network.
|
||||
#
|
||||
# The subnet is pinned so the value stays stable across reboots. It must match
|
||||
# an entry in shared/searxng/limiter.toml trusted_proxies. 10.89.2.0/24 is free
|
||||
# (gitea uses 10.89.0.0/24; the old compose net used 10.89.1.0/24).
|
||||
|
||||
[Network]
|
||||
Subnet=10.89.2.0/24
|
||||
@@ -1,43 +0,0 @@
|
||||
# Quadlet unit: Seerr media request manager (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/seerr.container
|
||||
# Unit name: seerr.service (container name: seerr)
|
||||
#
|
||||
# Seerr is the unified Overseerr/Jellyseerr successor. It fronts the
|
||||
# *arr stack now living on FrankBook-Pro (192.168.0.112, resolved via the
|
||||
# Pi-hole addn-hosts record frankbook-pro.lan): requests are placed here,
|
||||
# routed through the arrs over the LAN, and downloads run inside the Mac's
|
||||
# gluetun tunnel. SQLite lives under /home/dank/Containers/seerr/data —
|
||||
# no separate database unit (decision D8).
|
||||
#
|
||||
# Exposure contract: the ONLY published port is loopback (127.0.0.1:5055)
|
||||
# for the homepage probe and host-local checks. User traffic must traverse
|
||||
# the guard chain: Caddy TLS edge → Anubis :8923 → Authentik outpost →
|
||||
# Caddy :8017 post-auth listener → seerr:5055 on dankclaw_default. Never
|
||||
# publish 5055 on a LAN or public address.
|
||||
#
|
||||
# Image pin: ghcr.io/seerr-team/seerr:v3.4.1 (manifest-verified on GHCR
|
||||
# 2026-09-01; released 2026-07-30). Upgrade = bump the tag, redeploy.
|
||||
#
|
||||
# No secrets in this file: OIDC (Authentik) and arr API keys live in
|
||||
# Seerr's own settings store inside the data volume, configured at runtime.
|
||||
|
||||
[Unit]
|
||||
Description=Seerr media request manager container
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Container]
|
||||
ContainerName=seerr
|
||||
Image=ghcr.io/seerr-team/seerr:v3.4.1
|
||||
Network=dankclaw_default
|
||||
PublishPort=127.0.0.1:5055:5055
|
||||
Volume=/home/dank/Containers/seerr/data:/app/config
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
TimeoutStartSec=900
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,21 +0,0 @@
|
||||
# /etc/udev/rules.d/99-comfyui-gpu-access.rules
|
||||
#
|
||||
# Why world-writable compute nodes: the comfyui container is spawned by a
|
||||
# systemd user unit (comfyui.service, Type=oneshot). User-unit processes are
|
||||
# spawned by the user manager with NO supplementary groups, so any spawn path
|
||||
# through systemd loses `render`/`video` membership regardless of how the
|
||||
# container is launched (keep-groups and named group_add both depend on the
|
||||
# spawning process already holding the group — verified 2026-08-31: keep-groups
|
||||
# preserved an empty group list under the user manager and /dev/kfd open still
|
||||
# failed with EACCES). A mode that grants access to "everyone in this uid
|
||||
# domain" is the only spawn-context-independent option short of running the
|
||||
# stack rootful.
|
||||
#
|
||||
# Blast radius: any local process (every container here runs as the same uid
|
||||
# domain) can already ptrace the GPU userspace through an existing holder's
|
||||
# fd inheritance; on a private single-administrator home server this trades a
|
||||
# capability gate that could never be held by the intended user anyway.
|
||||
# Note the missing SUBSYSTEM= guard on the kfd line: misc-class devices have no
|
||||
# subsystem for udev matching, so a SUBSYSTEM=="misc" guard never matches.
|
||||
KERNEL=="kfd", MODE="0666"
|
||||
KERNEL=="renderD*", SUBSYSTEM=="drm", MODE="0666"
|
||||
@@ -1,156 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""HTTP front door for the frank-medium single-GPU replicas.
|
||||
|
||||
Why two replicas instead of TP=2: one vLLM process spanning both R9700s needs
|
||||
RCCL collectives, and RCCL's kernels require PCIe atomics on every
|
||||
participating GPU. The second card sits behind chipset bridges advertising
|
||||
`AtomicOpsCap: Routing-`, which no software can enable (see the runbook), so
|
||||
no collective can launch on it. Two single-GPU replicas use both cards; this
|
||||
service keeps the fleet contract of one endpoint on :7770, streaming included.
|
||||
|
||||
Routing decisions live in routing.py — spread single-turn requests, pin
|
||||
continuations to the replica whose prefix cache holds their context. This file
|
||||
only moves bytes: parse enough of the request to route it, forward it verbatim,
|
||||
stream the answer back, and report replica health on /health.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from contextlib import suppress
|
||||
from aiohttp import ClientSession, ClientTimeout, web
|
||||
|
||||
import routing
|
||||
|
||||
PORT = int(os.environ.get("PORT", "7770"))
|
||||
HEALTH_TTL = float(os.environ.get("HEALTH_TTL", "10"))
|
||||
# Hop-by-hop headers must not be forwarded (RFC 9110 §7.6.1) — forwarding
|
||||
# transfer-encoding or content-length of a rewritten body breaks clients.
|
||||
HOP_BY_HOP = {
|
||||
"host", "content-length", "transfer-encoding", "connection",
|
||||
"keep-alive", "proxy-authenticate", "proxy-authorization", "te",
|
||||
"trailers", "upgrade",
|
||||
}
|
||||
|
||||
UPSTREAMS = routing.UPSTREAMS
|
||||
# Replica health, refreshed by the watcher. Module state rather than aiohttp app
|
||||
# state: the application is frozen once the site is up.
|
||||
_UP: list[bool] = [True] * len(UPSTREAMS)
|
||||
|
||||
|
||||
async def health(session: ClientSession) -> web.Response:
|
||||
results = await asyncio.gather(
|
||||
*(session.get(u + "/health", timeout=ClientTimeout(total=5)) for u in UPSTREAMS),
|
||||
return_exceptions=True,
|
||||
)
|
||||
lines, up = [], 0
|
||||
for url, res in zip(UPSTREAMS, results):
|
||||
if isinstance(res, Exception) or res.status != 200:
|
||||
lines.append(f"{url} DOWN")
|
||||
else:
|
||||
up += 1
|
||||
lines.append(f"{url} UP")
|
||||
return web.Response(status=200 if up else 503, text="\n".join(lines) + "\n")
|
||||
|
||||
|
||||
async def watch_health() -> None:
|
||||
"""Refresh the replica up/down table so routing never targets a dead one.
|
||||
|
||||
One HTTP request per replica per interval, in exchange for never handing a
|
||||
client a 502 after a restart.
|
||||
"""
|
||||
global _UP
|
||||
session = ClientSession()
|
||||
try:
|
||||
while True:
|
||||
res = await asyncio.gather(
|
||||
*(session.get(u + "/health", timeout=ClientTimeout(total=5)) for u in UPSTREAMS),
|
||||
return_exceptions=True,
|
||||
)
|
||||
_UP = [not isinstance(r, Exception) and r.status == 200 for r in res]
|
||||
await asyncio.sleep(HEALTH_TTL)
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def proxy(request: web.Request) -> web.StreamResponse:
|
||||
session: ClientSession = request.app["session"]
|
||||
if request.path == "/health":
|
||||
return await health(session)
|
||||
|
||||
body = await request.read()
|
||||
headers = {k: v for k, v in request.headers.items() if k.lower() not in HOP_BY_HOP}
|
||||
|
||||
# Fail over before anything is written to the client. A replica that
|
||||
# restarts is invisible to the health watcher for up to HEALTH_TTL, and a
|
||||
# deploy restarts both replicas — without this retry a 10 s window of
|
||||
# requests would 502 on a service that has a live card the whole time.
|
||||
# vLLM's OpenAI endpoints are stateless per request, so re-sending a
|
||||
# request that never reached an engine is safe; a request that failed
|
||||
# mid-stream is not retried (that path is below).
|
||||
tried: set[int] = set()
|
||||
upstream = None
|
||||
last_error = "no healthy frank-medium replica"
|
||||
while len(tried) < len(UPSTREAMS):
|
||||
idx = routing.route(body, request.remote, _UP)
|
||||
if idx < 0 or idx in tried:
|
||||
break
|
||||
tried.add(idx)
|
||||
target = UPSTREAMS[idx] + request.path_qs
|
||||
try:
|
||||
upstream = await session.request(
|
||||
request.method, target, headers=headers, data=body or None,
|
||||
timeout=ClientTimeout(total=None, sock_connect=10, sock_read=3600),
|
||||
)
|
||||
break
|
||||
except Exception as exc:
|
||||
# Demote at once: the watcher will confirm, but every request until
|
||||
# then should skip this replica rather than pay the same timeout.
|
||||
# Logged because a silent failover hides a card that is down.
|
||||
_UP[idx] = False
|
||||
last_error = f"{target} unreachable: {exc}"
|
||||
print(f"proxy: failover — {target} unreachable ({exc}); retrying "
|
||||
f"another replica", flush=True)
|
||||
if upstream is None:
|
||||
# 503 when no replica was even reachable through the routing table,
|
||||
# 502 when replicas were tried and all of them failed.
|
||||
status = 502 if tried else 503
|
||||
return web.Response(status=status, text=f"proxy: {last_error}\n")
|
||||
|
||||
out = web.StreamResponse(status=upstream.status, reason=upstream.reason)
|
||||
for k, v in upstream.headers.items():
|
||||
if k.lower() not in HOP_BY_HOP:
|
||||
out.headers[k] = v
|
||||
await out.prepare(request)
|
||||
try:
|
||||
async for chunk in upstream.content.iter_any():
|
||||
await out.write(chunk)
|
||||
except Exception:
|
||||
# The client went away mid-stream (an aborted generation). Nothing to
|
||||
# report: the engine sees its own cancellation.
|
||||
pass
|
||||
finally:
|
||||
upstream.release()
|
||||
with suppress(Exception):
|
||||
await out.write_eof()
|
||||
return out
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
app = web.Application(client_max_size=1024 ** 3)
|
||||
app["session"] = ClientSession()
|
||||
app.router.add_route("*", "/{tail:.*}", proxy)
|
||||
runner = web.AppRunner(app)
|
||||
await runner.setup()
|
||||
await web.TCPSite(runner, "0.0.0.0", PORT).start()
|
||||
print(f"proxy: :{PORT} -> {', '.join(UPSTREAMS)} routing={routing.ROUTING} "
|
||||
f"key={routing.AFFINITY_BYTES}B known_max={routing.KNOWN_MAX}", flush=True)
|
||||
watcher = asyncio.create_task(watch_health())
|
||||
with suppress(asyncio.CancelledError):
|
||||
await asyncio.gather(watcher)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with suppress(KeyboardInterrupt):
|
||||
asyncio.run(main())
|
||||
@@ -1,98 +0,0 @@
|
||||
# Quadlet unit: frank-medium replica A (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankserver: ~/.config/containers/systemd/frank-medium-a.container
|
||||
# Unit name: frank-medium-a.service (container name: frank-medium-a)
|
||||
#
|
||||
# WHY TWO REPLICAS INSTEAD OF TP=2. The intended shape for frank-medium is one
|
||||
# tensor-parallel process across both R9700s (see frank-medium.container, kept
|
||||
# as the target). That shape cannot run on this host: RCCL's collective kernels
|
||||
# carry a hidden hostcall-buffer parameter, and the ROCr runtime refuses to
|
||||
# dispatch them on a device without PCIe atomics ("Pcie atomics not enabled,
|
||||
# hostcall not supported" -> AQL dispatch failed -> hipErrorIllegalState). The
|
||||
# second R9700 sits behind the chipset bridges 05:00.0/06:09.0, which advertise
|
||||
# `AtomicOpsCap: Routing-` — a read-only capability, so no software can enable
|
||||
# atomics on that path; the amdgpu driver's pci_enable_atomic_ops_to_root()
|
||||
# therefore fails for that GPU and KFD marks the CPU<->GPU2 link NO_ATOMICS.
|
||||
# The first R9700 sits on CPU root port 00:01.1 (Routing+) and does have
|
||||
# atomics. Result: every collective that touches GPU 2 dies at launch.
|
||||
# Two independent single-GPU replicas use both cards instead; the proxy unit
|
||||
# keeps the single :7770 endpoint. Full evidence: the frank-medium runbook.
|
||||
#
|
||||
# SHAPE. vLLM binds 0.0.0.0:7771 on the host network directly; no PublishPort.
|
||||
# MAXLEN is 114688, not the TP=2 deployment's 262144: one card carries the
|
||||
# whole model (~20 GiB of weights + drafter), leaving ~5.2 GiB of KV, and
|
||||
# 131072 does not fit (measured: needs 5.58 GiB). KV_MEM=0 lets vLLM profile
|
||||
# this shape instead of using the 2x7551-32624 pin, which is only valid at
|
||||
# TP=2/262144.
|
||||
#
|
||||
# DEVICES. /dev/kfd + /dev/dri; HIP_VISIBLE_DEVICES=0 (the first R9700). The
|
||||
# 2 GiB Raphael iGPU enumerates as index 2 and is excluded by the pin.
|
||||
# Only HIP_VISIBLE_DEVICES is set, never ROCR_VISIBLE_DEVICES: ROCR filters
|
||||
# first and HIP indexes into the filtered list, so a single-valued pair
|
||||
# (ROCR=1,HIP=1) leaves an empty device set.
|
||||
#
|
||||
# DO NOT name an environment variable TEMP, HOST or GPU_MAX_HEAP_SIZE: the
|
||||
# HIP runtime reads bare TEMP for its JIT cache path (2026-08-13 incident).
|
||||
[Unit]
|
||||
Description=frank-medium replica A — vllm-radiance MXFP4 Qwen3.8-27B on R9700 #1
|
||||
After=network-online.target
|
||||
|
||||
[Container]
|
||||
ContainerName=frank-medium-a
|
||||
Image=localhost/radiance-vllm:0.9.3
|
||||
AddDevice=/dev/kfd
|
||||
AddDevice=/dev/dri
|
||||
GroupAdd=keep-groups
|
||||
# The upstream launcher's exact podman posture (rootless).
|
||||
PodmanArgs=--privileged --ipc=host --security-opt seccomp=unconfined --cap-add SYS_PTRACE --network=host
|
||||
Environment=HIP_VISIBLE_DEVICES=0
|
||||
Environment=SNAP=/models/Qwen3.8-27B-Uncensored-MXFP4-awq
|
||||
Environment=DRAFTER=/models/Qwen3.8-27B-DFlash2-FP8
|
||||
Environment=CHAT_TEMPLATE=/patches/qwen-fixed-v22.3.jinja
|
||||
Environment=SERVED_NAMES=frank-medium
|
||||
Environment=PORT=7771
|
||||
Environment=TP=1
|
||||
Environment=MAXSEQS=4
|
||||
Environment=CHUNK=8192
|
||||
Environment=MAXLEN=114688
|
||||
Environment=GPU_UTIL=0.98
|
||||
Environment=KV_MEM=0
|
||||
Environment=SPEC_METHOD=dflash
|
||||
Environment=SPEC=7
|
||||
# Checkpoints (rw: nothing downloads at runtime, but vLLM mmaps the shards).
|
||||
Volume=/home/dank/models:/models
|
||||
# Tokenizer files ride the HF cache; HF_HUB_OFFLINE=1 is set in the entrypoint.
|
||||
Volume=/home/dank/.cache/huggingface:/root/.cache/huggingface
|
||||
# Compile cache. Shared with the other replica and with the TP=2 target: the
|
||||
# entries are content-addressed, so shapes simply add subdirectories.
|
||||
Volume=/home/dank/.radiance-cache-w4a8-093-gdnm-nqft-fp8s-gnq:/cache
|
||||
# The pinned radiance clone: patches applied at every container start.
|
||||
Volume=/home/dank/GitPackages/radiance-vllm-mxfp4:/patches:z
|
||||
# The patched libr4d (pin b9e42ab + repo extras, key b9e42ab-rx6). The image's
|
||||
# shipped r4d.so predates the GDN overflow fix and NaNs this model.
|
||||
Volume=/home/dank/.cache/radiance-libr4d/b9e42ab-rx6:/r4d:z
|
||||
# Entrypoint + healthcheck from this checkout, mounted read-only.
|
||||
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/radiance-entrypoint.sh:/usr/local/bin/frank-medium-entrypoint.sh:ro
|
||||
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/frank-medium-healthcheck.py:/usr/local/bin/frank-medium-healthcheck.py:ro
|
||||
# The entrypoint must be the container command; the image's own entrypoint
|
||||
# (which the patched script execs at the end) must not run unpatched.
|
||||
PodmanArgs=--entrypoint /usr/local/bin/frank-medium-entrypoint.sh
|
||||
# 200 only once the engine serves; during first-start compilation the
|
||||
# container is legitimately unhealthy. HealthCmd is split on spaces into an
|
||||
# argv array by the podman 5.x v1 generator, hence the file-based probe.
|
||||
HealthCmd=python3 /usr/local/bin/frank-medium-healthcheck.py
|
||||
HealthInterval=60s
|
||||
HealthTimeout=30s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=1800s
|
||||
Pull=never
|
||||
|
||||
[Service]
|
||||
# First start compiles Triton/inductor kernels (several minutes) before the
|
||||
# engine loads ~20 GiB of weights.
|
||||
TimeoutStartSec=1800
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,98 +0,0 @@
|
||||
# Quadlet unit: frank-medium replica B (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankserver: ~/.config/containers/systemd/frank-medium-b.container
|
||||
# Unit name: frank-medium-b.service (container name: frank-medium-a)
|
||||
#
|
||||
# WHY TWO REPLICAS INSTEAD OF TP=2. The intended shape for frank-medium is one
|
||||
# tensor-parallel process across both R9700s (see frank-medium.container, kept
|
||||
# as the target). That shape cannot run on this host: RCCL's collective kernels
|
||||
# carry a hidden hostcall-buffer parameter, and the ROCr runtime refuses to
|
||||
# dispatch them on a device without PCIe atomics ("Pcie atomics not enabled,
|
||||
# hostcall not supported" -> AQL dispatch failed -> hipErrorIllegalState). The
|
||||
# second R9700 sits behind the chipset bridges 05:00.0/06:09.0, which advertise
|
||||
# `AtomicOpsCap: Routing-` — a read-only capability, so no software can enable
|
||||
# atomics on that path; the amdgpu driver's pci_enable_atomic_ops_to_root()
|
||||
# therefore fails for that GPU and KFD marks the CPU<->GPU2 link NO_ATOMICS.
|
||||
# The first R9700 sits on CPU root port 00:01.1 (Routing+) and does have
|
||||
# atomics. Result: every collective that touches GPU 2 dies at launch.
|
||||
# Two independent single-GPU replicas use both cards instead; the proxy unit
|
||||
# keeps the single :7770 endpoint. Full evidence: the frank-medium runbook.
|
||||
#
|
||||
# SHAPE. vLLM binds 0.0.0.0:7772 on the host network directly; no PublishPort.
|
||||
# MAXLEN is 114688, not the TP=2 deployment's 262144: one card carries the
|
||||
# whole model (~20 GiB of weights + drafter), leaving ~5.2 GiB of KV, and
|
||||
# 131072 does not fit (measured: needs 5.58 GiB). KV_MEM=0 lets vLLM profile
|
||||
# this shape instead of using the 2x7551-32624 pin, which is only valid at
|
||||
# TP=2/262144.
|
||||
#
|
||||
# DEVICES. /dev/kfd + /dev/dri; HIP_VISIBLE_DEVICES=1 (the second R9700). The
|
||||
# 2 GiB Raphael iGPU enumerates as index 2 and is excluded by the pin.
|
||||
# Only HIP_VISIBLE_DEVICES is set, never ROCR_VISIBLE_DEVICES: ROCR filters
|
||||
# first and HIP indexes into the filtered list, so a single-valued pair
|
||||
# (ROCR=1,HIP=1) leaves an empty device set.
|
||||
#
|
||||
# DO NOT name an environment variable TEMP, HOST or GPU_MAX_HEAP_SIZE: the
|
||||
# HIP runtime reads bare TEMP for its JIT cache path (2026-08-13 incident).
|
||||
[Unit]
|
||||
Description=frank-medium replica B — vllm-radiance MXFP4 Qwen3.8-27B on R9700 #2
|
||||
After=network-online.target
|
||||
|
||||
[Container]
|
||||
ContainerName=frank-medium-b
|
||||
Image=localhost/radiance-vllm:0.9.3
|
||||
AddDevice=/dev/kfd
|
||||
AddDevice=/dev/dri
|
||||
GroupAdd=keep-groups
|
||||
# The upstream launcher's exact podman posture (rootless).
|
||||
PodmanArgs=--privileged --ipc=host --security-opt seccomp=unconfined --cap-add SYS_PTRACE --network=host
|
||||
Environment=HIP_VISIBLE_DEVICES=1
|
||||
Environment=SNAP=/models/Qwen3.8-27B-Uncensored-MXFP4-awq
|
||||
Environment=DRAFTER=/models/Qwen3.8-27B-DFlash2-FP8
|
||||
Environment=CHAT_TEMPLATE=/patches/qwen-fixed-v22.3.jinja
|
||||
Environment=SERVED_NAMES=frank-medium
|
||||
Environment=PORT=7772
|
||||
Environment=TP=1
|
||||
Environment=MAXSEQS=4
|
||||
Environment=CHUNK=8192
|
||||
Environment=MAXLEN=114688
|
||||
Environment=GPU_UTIL=0.98
|
||||
Environment=KV_MEM=0
|
||||
Environment=SPEC_METHOD=dflash
|
||||
Environment=SPEC=7
|
||||
# Checkpoints (rw: nothing downloads at runtime, but vLLM mmaps the shards).
|
||||
Volume=/home/dank/models:/models
|
||||
# Tokenizer files ride the HF cache; HF_HUB_OFFLINE=1 is set in the entrypoint.
|
||||
Volume=/home/dank/.cache/huggingface:/root/.cache/huggingface
|
||||
# Compile cache. Shared with the other replica and with the TP=2 target: the
|
||||
# entries are content-addressed, so shapes simply add subdirectories.
|
||||
Volume=/home/dank/.radiance-cache-w4a8-093-gdnm-nqft-fp8s-gnq:/cache
|
||||
# The pinned radiance clone: patches applied at every container start.
|
||||
Volume=/home/dank/GitPackages/radiance-vllm-mxfp4:/patches:z
|
||||
# The patched libr4d (pin b9e42ab + repo extras, key b9e42ab-rx6). The image's
|
||||
# shipped r4d.so predates the GDN overflow fix and NaNs this model.
|
||||
Volume=/home/dank/.cache/radiance-libr4d/b9e42ab-rx6:/r4d:z
|
||||
# Entrypoint + healthcheck from this checkout, mounted read-only.
|
||||
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/radiance-entrypoint.sh:/usr/local/bin/frank-medium-entrypoint.sh:ro
|
||||
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/frank-medium-healthcheck.py:/usr/local/bin/frank-medium-healthcheck.py:ro
|
||||
# The entrypoint must be the container command; the image's own entrypoint
|
||||
# (which the patched script execs at the end) must not run unpatched.
|
||||
PodmanArgs=--entrypoint /usr/local/bin/frank-medium-entrypoint.sh
|
||||
# 200 only once the engine serves; during first-start compilation the
|
||||
# container is legitimately unhealthy. HealthCmd is split on spaces into an
|
||||
# argv array by the podman 5.x v1 generator, hence the file-based probe.
|
||||
HealthCmd=python3 /usr/local/bin/frank-medium-healthcheck.py
|
||||
HealthInterval=60s
|
||||
HealthTimeout=30s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=1800s
|
||||
Pull=never
|
||||
|
||||
[Service]
|
||||
# First start compiles Triton/inductor kernels (several minutes) before the
|
||||
# engine loads ~20 GiB of weights.
|
||||
TimeoutStartSec=1800
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,61 +0,0 @@
|
||||
# Quadlet unit: frank-medium proxy (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankserver: ~/.config/containers/systemd/frank-medium-proxy.container
|
||||
# Unit name: frank-medium-proxy.service (container name: frank-medium-proxy)
|
||||
#
|
||||
# WHAT IT IS. A prefix-affinity reverse proxy in front of the two single-GPU
|
||||
# frank-medium replicas (:7771 on R9700 #1, :7772 on R9700 #2). It preserves
|
||||
# the fleet contract: one endpoint on host port 7770 answering as the served
|
||||
# model name frank-medium, OpenAI /v1, streaming included.
|
||||
#
|
||||
# WHY IT EXISTS. TP=2 — one vLLM process spanning both cards — is impossible
|
||||
# on this host (see frank-medium-a.container and the runbook: RCCL's kernels
|
||||
# need PCIe atomics, and the second card's chipset path advertises
|
||||
# AtomicOpsCap: Routing-). Two replicas are how both cards get used.
|
||||
#
|
||||
# ROUTING. Requests are keyed on a hash of their leading 4 KiB of prompt text,
|
||||
# so a conversation family — an agent loop re-sending the same system prompt,
|
||||
# tool list and file context every turn — keeps landing on the same replica,
|
||||
# where --enable-prefix-caching has that prefix warm. Plain round-robin threw
|
||||
# half of those cache hits away. A replica that is down is skipped, and
|
||||
# ROUTING=roundrobin restores the old behaviour for A/B measurement. /health
|
||||
# answers 200 while at least one replica is up and names the ones that are
|
||||
# down. The proxy runs on host networking so it can bind 7770 and reach the
|
||||
# replicas on loopback.
|
||||
#
|
||||
# The script is a few dozen lines of aiohttp, mounted read-only out of
|
||||
# /home/dank/serverAppConfigs (deploy.sh pulls that checkout before dispatch).
|
||||
# It reuses the same image as the replicas: aiohttp is already there and no
|
||||
# extra image has to be pulled on this host.
|
||||
[Unit]
|
||||
Description=frank-medium proxy — prefix-affinity in front of the two single-GPU replicas
|
||||
After=network-online.target
|
||||
Wants=frank-medium-a.service frank-medium-b.service
|
||||
After=frank-medium-a.service frank-medium-b.service
|
||||
|
||||
[Container]
|
||||
ContainerName=frank-medium-proxy
|
||||
Image=localhost/radiance-vllm:0.9.3
|
||||
PodmanArgs=--network=host
|
||||
Environment=PORT=7770
|
||||
Environment=UPSTREAMS=http://127.0.0.1:7771,http://127.0.0.1:7772
|
||||
# The proxy is two files: routing.py holds the replica-selection policy, the
|
||||
# server holds the HTTP plumbing. Both are mounted because the server imports
|
||||
# the policy by name from its own directory (/usr/local/bin is sys.path[0]).
|
||||
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/proxy/routing.py:/usr/local/bin/routing.py:ro
|
||||
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/proxy/affinity-proxy.py:/usr/local/bin/affinity-proxy.py:ro
|
||||
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/frank-medium-healthcheck.py:/usr/local/bin/frank-medium-healthcheck.py:ro
|
||||
PodmanArgs=--entrypoint /usr/local/bin/affinity-proxy.py
|
||||
HealthCmd=python3 /usr/local/bin/frank-medium-healthcheck.py
|
||||
HealthInterval=60s
|
||||
HealthTimeout=30s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=60s
|
||||
Pull=never
|
||||
|
||||
[Service]
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,129 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Routing policy for the frank-medium proxy: which replica serves a request.
|
||||
|
||||
Kept apart from the HTTP plumbing in affinity-proxy.py so the decisions can be
|
||||
tested without aiohttp, a network or a GPU — the first version of these rules
|
||||
shipped with a bug that only appeared on a continuation request, and a test
|
||||
that loads this module catches that class of mistake directly.
|
||||
|
||||
Two cards, two behaviours, and the workload decides which is right:
|
||||
|
||||
* A **single-turn** request is one member of a batch (a benchmark run, a burst
|
||||
of parallel subagents). Spreading those over both replicas lets both cards
|
||||
prefill at once; measured on this host, a 16-request burst of 15k-token
|
||||
prompts finishes in 18.6 s spread versus 30.1 s pinned.
|
||||
* A **continuation** re-sends a prefix the replica has already prefilled. Pinned
|
||||
to that replica the turn costs a cache lookup; spread, half the turns pay a
|
||||
cold prefill again (measured: 1096 ms versus 2676 ms mean warm-turn TTFT on
|
||||
90k-token contexts).
|
||||
|
||||
ROUTING picks the strategy; the default `adaptive` applies both rules above.
|
||||
`affinity` and `roundrobin` exist so the choice can be re-measured, and are not
|
||||
production settings.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
|
||||
UPSTREAMS = [u.strip() for u in os.environ.get(
|
||||
"UPSTREAMS", "http://127.0.0.1:7771,http://127.0.0.1:7772").split(",") if u.strip()]
|
||||
ROUTING = os.environ.get("ROUTING", "adaptive").strip().lower()
|
||||
# Leading bytes of prompt text that identify a conversation's shared prefix.
|
||||
# 4 KiB covers a system prompt plus the start of the first user turn, which is
|
||||
# where agent loops keep the text they resend every turn.
|
||||
AFFINITY_BYTES = int(os.environ.get("AFFINITY_BYTES", "4096"))
|
||||
# Prefixes remembered for pinning. Bounded so a long-lived proxy cannot grow
|
||||
# without limit; 4096 conversations is far beyond what this fleet runs at once.
|
||||
KNOWN_MAX = int(os.environ.get("KNOWN_MAX", "4096"))
|
||||
|
||||
_counter = itertools.count()
|
||||
# digest -> replica index, insertion-ordered for O(1) LRU eviction. A prefix is
|
||||
# remembered as soon as any request carries it; only a continuation is pinned
|
||||
# there, because only a continuation can profit from the cache entry.
|
||||
_known: "collections.OrderedDict[bytes, int]" = collections.OrderedDict()
|
||||
|
||||
|
||||
def affinity_key(body: bytes, remote: str | None) -> tuple[bytes, bool]:
|
||||
"""Key a request by its shared prefix, and report whether it continues one.
|
||||
|
||||
Chat and completions bodies both carry their prompt text in JSON, so the
|
||||
leading bytes of that text are the prefix; a body we cannot parse (or an
|
||||
endpoint with no prompt, e.g. /v1/models) falls back to the client address,
|
||||
which still beats nothing for a single-client loop.
|
||||
"""
|
||||
fallback = (remote or "anonymous").encode()
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
except Exception:
|
||||
return fallback, False
|
||||
|
||||
text = ""
|
||||
continuation = False
|
||||
if isinstance(payload, dict):
|
||||
if isinstance(payload.get("prompt"), str):
|
||||
text = payload["prompt"]
|
||||
elif isinstance(payload.get("messages"), list):
|
||||
continuation = len(payload["messages"]) > 1
|
||||
parts = []
|
||||
for msg in payload["messages"]:
|
||||
if isinstance(msg, dict):
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str):
|
||||
parts.append(content)
|
||||
elif isinstance(content, list): # multimodal content blocks
|
||||
parts.extend(
|
||||
blk.get("text", "") for blk in content
|
||||
if isinstance(blk, dict) and isinstance(blk.get("text"), str)
|
||||
)
|
||||
text = "\n".join(parts)
|
||||
if not text:
|
||||
return fallback, continuation
|
||||
return text[:AFFINITY_BYTES].encode("utf-8", "replace"), continuation
|
||||
|
||||
|
||||
def pick(key: bytes, continuation: bool, up: list[bool]) -> int:
|
||||
"""Replica for a request, or -1 when none is available.
|
||||
|
||||
Down replicas are never selected — a rollout must not send traffic to an
|
||||
engine that is still compiling its graphs — and a pin whose replica is down
|
||||
is re-established on a live one rather than kept pointing at a corpse.
|
||||
"""
|
||||
digest = hashlib.sha1(key).digest()
|
||||
|
||||
if ROUTING == "roundrobin":
|
||||
start = next(_counter) % len(UPSTREAMS)
|
||||
elif ROUTING == "adaptive":
|
||||
entry = _known.get(digest)
|
||||
if continuation and entry is not None and up[entry]:
|
||||
_known.move_to_end(digest)
|
||||
return entry
|
||||
# Anything that is not a continuation spreads by counter — including a
|
||||
# batch of requests that happen to share one prefix. Hashing the prefix
|
||||
# would send every member of such a batch to whichever card the hash
|
||||
# names (measured: 29.8 s for a 16-request burst, against 18.6 s when
|
||||
# spread). The counter is only advanced by these requests, so batch
|
||||
# members alternate across the cards.
|
||||
start = next(_counter) % len(UPSTREAMS)
|
||||
else: # affinity
|
||||
start = int.from_bytes(digest[:8], "big") % len(UPSTREAMS)
|
||||
|
||||
for offset in range(len(UPSTREAMS)):
|
||||
idx = (start + offset) % len(UPSTREAMS)
|
||||
if up[idx]:
|
||||
if ROUTING == "adaptive":
|
||||
_known[digest] = idx
|
||||
_known.move_to_end(digest)
|
||||
if len(_known) > KNOWN_MAX:
|
||||
_known.popitem(last=False)
|
||||
return idx
|
||||
return -1
|
||||
|
||||
|
||||
def route(body: bytes, remote: str | None, up: list[bool]) -> int:
|
||||
"""Convenience wrapper: decode the request body and choose a replica."""
|
||||
return pick(*affinity_key(body, remote), up)
|
||||
@@ -1,22 +0,0 @@
|
||||
# Dankserver stack secrets and configuration.
|
||||
# Copy to .env and fill real values. Never commit real secrets.
|
||||
|
||||
# RomM admin credentials (first boot)
|
||||
ROMM_USERNAME=changeme
|
||||
ROMM_PASSWORD=changeme
|
||||
ROMM_AUTH_SECRET_KEY=changeme
|
||||
|
||||
# RomM database (MariaDB)
|
||||
ROMM_DB_USER=romm
|
||||
ROMM_DB_PASSWD=changeme
|
||||
MARIADB_ROOT_PASSWORD=changeme
|
||||
|
||||
# RomM Redis
|
||||
# (no password needed for local Redis)
|
||||
|
||||
# Copyparty
|
||||
COPAY_PASS=changeme
|
||||
|
||||
# Jellyfin
|
||||
JELLYFIN_PUBLISH_SERVER_URL=http://192.168.0.235:8096
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
# Quadlet unit: Calibre Content Server (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankserver:
|
||||
# ~/.config/containers/systemd/calibre-server.container
|
||||
# Unit name: calibre-server.service (podman 5.x drops the container- prefix)
|
||||
#
|
||||
# The image is the CURRENT linuxserver/calibre (9.15.0), which is a desktop
|
||||
# image (Selkies web-streamed GUI on 8080/8181) with the headless
|
||||
# calibre-server available inside. The compose-era snippet in
|
||||
# serverAppConfigs-docs/Dankserver/Calibre.md predates the desktop revamp;
|
||||
# the dedicated headless images (ta264/docker-calibre, agentgreen) are
|
||||
# dormant (calibre 6.10 / 2017), so this unit runs calibre-server as the
|
||||
# image's s6-overlay main service via Exec=; the desktop also runs but its
|
||||
# ports are internal-only (see the port map below).
|
||||
#
|
||||
# Port map: Network=pasta is port-FORWARD mode on this host (the container
|
||||
# keeps its own netns; only PublishPort'd ports reach the host — verified
|
||||
# 2026-09-21: 8080/8090/8182 inside the container are not visible on the
|
||||
# host, only the 8084 forward). So the ONLY exposed surface is:
|
||||
# 8090 -> host 8084 calibre-server (the content server; 8080 inside the
|
||||
# container is the image's own nginx, 8083 on the host
|
||||
# is calibre-web)
|
||||
# The desktop GUI (nginx 8080/8181 + WS 8082, inside the container) runs but
|
||||
# is internal-only: no PublishPort. Exposing it is one PublishPort line away
|
||||
# if the browser GUI is ever wanted (the basic-auth credential is already in
|
||||
# the env file).
|
||||
# Auth: calibre-server 9.x keeps its users in a SQLite userdb (the pw column
|
||||
# is PLAINTEXT, so the file is mode 600). The single admin account is
|
||||
# bootstrapped once by the deploy handler (deploy_calibre_server) via
|
||||
# `calibre-server --manage-users -- add`, password piped on stdin with
|
||||
# printf (a trailing newline from echo lands in the stored password and
|
||||
# breaks every login). The desktop's basic auth (CUSTOM_USER/CUSTOM_PASSWORD)
|
||||
# reuses the same credential pair. Both live in the machine-local env file;
|
||||
# this unit holds no secrets.
|
||||
#
|
||||
# --auth-mode=basic is required: on a non-SSL port the server defaults to
|
||||
# Digest auth, which rejects plain Basic clients with 400 "Unsupported
|
||||
# authentication method".
|
||||
|
||||
[Unit]
|
||||
Description=Calibre content server container
|
||||
After=network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=calibre-server
|
||||
Image=lscr.io/linuxserver/calibre:9.15.0
|
||||
Network=pasta
|
||||
EnvironmentFile=/home/dank/Containers/calibre-server/env
|
||||
Environment=PUID=1000
|
||||
Environment=PGID=1000
|
||||
Environment=TZ=America/Vancouver
|
||||
Volume=/home/dank/Containers/calibre-web/books:/books
|
||||
Volume=/home/dank/Containers/calibre-server/userdb.sqlite:/userdb.sqlite
|
||||
Exec=calibre-server --port 8090 --enable-auth --auth-mode=basic --userdb /userdb.sqlite /books
|
||||
PublishPort=8084:8090
|
||||
|
||||
[Service]
|
||||
TimeoutStartSec=300
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,43 +0,0 @@
|
||||
# Quadlet unit: Calibre-Web library server (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankserver:
|
||||
# ~/.config/containers/systemd/calibre-web.container
|
||||
# Unit name: calibre-web.service (podman 5.x drops the container- prefix)
|
||||
#
|
||||
# The data roots already lived under /home/dank/Containers/calibre-web in the
|
||||
# compose era; this unit keeps those exact paths. The image pin is the exact
|
||||
# tag of the image the live container runs (image id 986277563e76), so the
|
||||
# cutover recreation changes zero bits of application code. No secrets: this
|
||||
# unit takes no EnvironmentFile.
|
||||
|
||||
[Unit]
|
||||
Description=Calibre-Web library server container
|
||||
After=network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=calibre-web
|
||||
Image=lscr.io/linuxserver/calibre-web:0.6.27-ls402
|
||||
Network=pasta
|
||||
Environment=DOCKER_MODS=linuxserver/mods:universal-calibre
|
||||
Environment=LANG=en_US.UTF-8
|
||||
Environment=LANGUAGE=en_US.UTF-8
|
||||
Environment=LSIO_FIRST_PARTY=true
|
||||
Environment=OAUTHLIB_RELAX_TOKEN_SCOPE=1
|
||||
Environment=PGID=1000
|
||||
Environment=PUID=1000
|
||||
Environment=QTWEBENGINE_CHROMIUM_FLAGS=--no-sandbox
|
||||
Environment=S6_CMD_WAIT_FOR_SERVICES_MAXTIME=0
|
||||
Environment=S6_STAGE2_HOOK=/docker-mods
|
||||
Environment=TERM=xterm
|
||||
Environment=TZ=America/Vancouver
|
||||
Environment=VIRTUAL_ENV=/lsiopy
|
||||
PublishPort=8083:8083
|
||||
Volume=/home/dank/Containers/calibre-web/config:/config
|
||||
Volume=/home/dank/Containers/calibre-web/books:/books
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,154 +0,0 @@
|
||||
# Dankserver stack — media & library services.
|
||||
# Deploy: cd /home/dank/serverAppConfigs && podman-compose -f dankserver/compose.yaml up -d
|
||||
# See ../docs/Dankserver/ for service documentation.
|
||||
|
||||
networks:
|
||||
appnet:
|
||||
driver: bridge
|
||||
|
||||
services:
|
||||
# ═══════════════════════════════════════════════
|
||||
# RomM — retro game manager (requires MariaDB + Redis)
|
||||
# ═══════════════════════════════════════════════
|
||||
mariadb:
|
||||
image: docker.io/mariadb:11
|
||||
container_name: romm-mariadb
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
MARIADB_ROOT_PASSWORD: "${MARIADB_ROOT_PASSWORD}"
|
||||
MARIADB_DATABASE: romm
|
||||
MARIADB_USER: "${ROMM_DB_USER}"
|
||||
MARIADB_PASSWORD: "${ROMM_DB_PASSWD}"
|
||||
volumes:
|
||||
- /opt/romm/mariadb:/var/lib/mysql
|
||||
healthcheck:
|
||||
test: ["CMD", "mariadb-admin", "ping", "-h", "localhost"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
redis:
|
||||
image: docker.io/redis:7-alpine
|
||||
container_name: romm-redis
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /opt/romm/redis:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
romm:
|
||||
image: ghcr.io/rommapp/romm:latest
|
||||
container_name: romm
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
ROMM_USERNAME: "${ROMM_USERNAME}"
|
||||
ROMM_PASSWORD: "${ROMM_PASSWORD}"
|
||||
ROMM_AUTH_SECRET_KEY: "${ROMM_AUTH_SECRET_KEY}"
|
||||
DB_HOST: mariadb
|
||||
DB_NAME: romm
|
||||
DB_USER: "${ROMM_DB_USER}"
|
||||
DB_PASSWD: "${ROMM_DB_PASSWD}"
|
||||
REDIS_HOST: redis
|
||||
HASHEOUS_API_ENABLED: "true"
|
||||
ports:
|
||||
- "8085:8080"
|
||||
volumes:
|
||||
- /opt/romm/library:/romm/library
|
||||
- /opt/romm/resources:/romm/resources
|
||||
- /opt/romm/config:/romm/config
|
||||
depends_on:
|
||||
mariadb:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
# ═══════════════════════════════════════════════
|
||||
# Copyparty — file sharing and streaming
|
||||
# ═══════════════════════════════════════════════
|
||||
copyparty:
|
||||
image: ghcr.io/9001/copyparty-ac:latest
|
||||
container_name: copyparty
|
||||
restart: unless-stopped
|
||||
# read-only since 2026-08-31: copyparty perms are per-field "letters[,user]"
|
||||
# with no user = every unauthenticated request. Anonymous gets r (browse +
|
||||
# download; the guarded files.franksdank.cloud chain reaches copyparty
|
||||
# unauthenticated after authentik). dank gets ra (admin panel). No w/m/d
|
||||
# anywhere: uploads, moves, deletes rejected server-side. Interface: `--help-accounts` in-image.
|
||||
command: ["-a", "dank:${COPAY_PASS}", "-v", "/w::r:ra,dank"]
|
||||
ports:
|
||||
- "3923:3923"
|
||||
volumes:
|
||||
- /mnt/entropy:/w:ro
|
||||
- /opt/copyparty/cfg:/cfg
|
||||
healthcheck:
|
||||
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3923/')"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
# ═══════════════════════════════════════════════
|
||||
# Jellyfin — media server (movies, TV, music)
|
||||
# ═══════════════════════════════════════════════
|
||||
jellyfin:
|
||||
image: jellyfin/jellyfin:latest
|
||||
container_name: jellyfin
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8096:8096/tcp"
|
||||
- "8920:8920/tcp"
|
||||
- "7359:7359/udp"
|
||||
- "1900:1900/udp"
|
||||
environment:
|
||||
- JELLYFIN_PublishedServerUrl=http://192.168.0.235:8096
|
||||
- DOTNET_SYSTEM_IO_DISABLEFILELOCKING=true
|
||||
volumes:
|
||||
- /opt/jellyfin/config:/config
|
||||
- /opt/jellyfin/cache:/cache
|
||||
- /mnt/entropy:/data:ro
|
||||
devices:
|
||||
- /dev/dri:/dev/dri
|
||||
group_add:
|
||||
- "992"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8096/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
# ═══════════════════════════════════════════════
|
||||
# Calibre-Web — e-book web frontend
|
||||
# ═══════════════════════════════════════════════
|
||||
calibre-web:
|
||||
image: lscr.io/linuxserver/calibre-web:0.6.27-ls402
|
||||
container_name: calibre-web
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
PUID: "1000"
|
||||
PGID: "1000"
|
||||
TZ: Etc/UTC
|
||||
OAUTHLIB_RELAX_TOKEN_SCOPE: "1"
|
||||
ports:
|
||||
- "8083:8083"
|
||||
volumes:
|
||||
- /opt/calibre-web/config:/config
|
||||
- /opt/calibre-web/books:/books
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8083/"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
|
||||
# node-exporter left this file on 2026-09-16 for
|
||||
# dankserver/observability/quadlet/node-exporter.container: a compose-only
|
||||
# container does not survive a reboot here — it died `Exited (143)` and stayed
|
||||
# dead, leaving Prometheus' node-dankserver job scraping nothing — and the
|
||||
# compose form bound the wildcard address. One owner per container name.
|
||||
@@ -1,38 +0,0 @@
|
||||
# Quadlet unit: Copyparty file server (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankserver: ~/.config/containers/systemd/copyparty.container
|
||||
# Unit name: copyparty.service (podman 5.x drops the container- prefix)
|
||||
#
|
||||
# The image's default entrypoint is /bin/ash /z/cpp.sh -c /z/initcfg; the
|
||||
# Command= line below reproduces the compose-era argv exactly: the admin
|
||||
# account is passed as -a <user>:<password>. ${COPAY_ACCOUNT} and
|
||||
# ${COPAY_PASSWORD} are expanded by systemd at ExecStart time from the
|
||||
# host-local env file /home/dank/Containers/copyparty/env (mode 0600).
|
||||
# This file holds no secrets. The deployer refuses to start this unit before
|
||||
# that env file exists, so a missing env file fails the unit instead of
|
||||
# creating an admin account with a garbage password.
|
||||
#
|
||||
# /w is the entropy volume, mounted read-only: nothing writes into /mnt/entropy
|
||||
# anymore; every consumer of the share is a read-only client.
|
||||
|
||||
[Unit]
|
||||
Description=Copyparty file server container
|
||||
After=network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=copyparty
|
||||
Image=ghcr.io/9001/copyparty-ac:latest
|
||||
Network=dankserver_appnet
|
||||
EnvironmentFile=/home/dank/Containers/copyparty/env
|
||||
Exec=-a ${COPAY_ACCOUNT}:${COPAY_PASSWORD} -v /w::r:ra,dank
|
||||
Volume=/mnt/entropy:/w:ro
|
||||
Volume=/home/dank/Containers/copyparty/cfg:/cfg
|
||||
PublishPort=3923:3923
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=/home/dank/Containers/copyparty/env
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,57 +0,0 @@
|
||||
# Quadlet unit: Jellyfin media server (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankserver: ~/.config/containers/systemd/jellyfin.container
|
||||
# Unit name: jellyfin.service (podman 5.x drops the container- prefix)
|
||||
#
|
||||
# Data roots migrated from /opt/jellyfin to /home/dank/Containers/jellyfin at
|
||||
# cutover. All of /mnt/entropy is mounted read-only at /data: nothing automated
|
||||
# anywhere writes to the entropy volume. Every
|
||||
# creation-time environment variable of the compose-era container is kept as
|
||||
# an Environment= line (zero drift); none of them are secrets, so this unit
|
||||
# needs no EnvironmentFile.
|
||||
|
||||
[Unit]
|
||||
Description=Jellyfin media server container
|
||||
After=network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=jellyfin
|
||||
Image=ghcr.io/jellyfin/jellyfin:latest
|
||||
Network=dankserver_appnet
|
||||
Environment=DEBIAN_FRONTEND=noninteractive
|
||||
Environment=DOTNET_SYSTEM_IO_DISABLEFILELOCKING=true
|
||||
Environment=JELLYFIN_CACHE_DIR=/cache
|
||||
Environment=JELLYFIN_CONFIG_DIR=/config/config
|
||||
Environment=JELLYFIN_DATA_DIR=/config
|
||||
Environment=JELLYFIN_FFMPEG=/usr/lib/jellyfin-ffmpeg/ffmpeg
|
||||
Environment=JELLYFIN_LOG_DIR=/config/log
|
||||
Environment=JELLYFIN_PublishedServerUrl=http://192.168.0.235:8096
|
||||
Environment=JELLYFIN_WEB_DIR=/jellyfin/jellyfin-web
|
||||
Environment=LANG=en_US.UTF-8
|
||||
Environment=LANGUAGE=en_US:en
|
||||
Environment=LC_ALL=en_US.UTF-8
|
||||
Environment=LD_PRELOAD=/usr/lib/jellyfin/libjemalloc.so.2
|
||||
Environment=MALLOC_TRIM_THRESHOLD_=131072
|
||||
Environment=NVIDIA_DRIVER_CAPABILITIES=compute,video,utility
|
||||
Environment=NVIDIA_VISIBLE_DEVICES=all
|
||||
Environment=XDG_CACHE_HOME=/cache
|
||||
PublishPort=8096:8096
|
||||
PublishPort=8920:8920
|
||||
PublishPort=7359:7359/udp
|
||||
PublishPort=1900:1900/udp
|
||||
Volume=/home/dank/Containers/jellyfin/config:/config
|
||||
Volume=/home/dank/Containers/jellyfin/cache:/cache
|
||||
Volume=/mnt/entropy:/data:ro
|
||||
AddDevice=/dev/dri
|
||||
GroupAdd=keep-groups
|
||||
HealthCmd=curl -f http://localhost:8096/health
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=3
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,58 +0,0 @@
|
||||
# Quadlet unit: node-exporter (rootless, boot-persistent) — DANKSERVER.
|
||||
#
|
||||
# Install path on dankserver: ~/.config/containers/systemd/node-exporter.container
|
||||
# Unit name: node-exporter.service (container name: node-exporter)
|
||||
#
|
||||
# WHY THIS FILE EXISTS. Until 2026-09-16 the Dankserver node-exporter ran only
|
||||
# from `dankserver/compose.yaml` (the "Observability" block, `restart:
|
||||
# unless-stopped`). Rootless Podman does not restart bare containers after the
|
||||
# explicit stop that shutdown performs, so it died at a reboot and stayed dead:
|
||||
# `podman ps -a` showed it `Exited (143)` and Prometheus' `node-dankserver` job
|
||||
# had been scraping nothing since. The compose entry also bound the default
|
||||
# wildcard address, which is the unauthenticated-metrics-API-on-every-interface
|
||||
# posture that was already removed on Dankclaw. This unit replaces both: one
|
||||
# owner per container name (the compose service is deleted), explicit listen
|
||||
# addresses, and a generator-provided boot path.
|
||||
#
|
||||
# HOST NAMESPACE. Networking and PID in the host's namespaces: the collectors
|
||||
# report the host's own interfaces, connections, processes and cgroups. The
|
||||
# host's /proc, /sys and / are mounted read-only under /host and /rootfs, and
|
||||
# node_exporter writes nothing.
|
||||
#
|
||||
# LISTEN ADDRESSES. Exactly two:
|
||||
# - 127.0.0.1:9100 for host-local probes
|
||||
# - 192.168.0.235:9100 for Prometheus, which scrapes this host from Dankclaw
|
||||
# (shared/prometheus/prometheus.yml, job "node-dankserver"). It cannot use
|
||||
# loopback across hosts, and node_exporter has no authentication, so the
|
||||
# wildcard bind stays off the table. This is the same deliberate LAN
|
||||
# listener posture the observability runbook documents.
|
||||
#
|
||||
# MIRROR. Keep this file in step with dankclaw/observability/quadlet/
|
||||
# node-exporter.container; only the Description and the LAN address differ.
|
||||
|
||||
[Unit]
|
||||
Description=Prometheus node exporter (Dankserver host metrics)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Container]
|
||||
ContainerName=node-exporter
|
||||
Image=docker.io/prom/node-exporter:latest
|
||||
Network=host
|
||||
PodmanArgs=--pid=host
|
||||
Exec=--path.procfs=/host/proc --path.sysfs=/host/sys --path.rootfs=/rootfs --web.listen-address=127.0.0.1:9100 --web.listen-address=192.168.0.235:9100
|
||||
Volume=/proc:/host/proc:ro
|
||||
Volume=/sys:/host/sys:ro
|
||||
Volume=/:/rootfs:ro
|
||||
HealthCmd=wget --spider -q http://localhost:9100/metrics
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=30s
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,22 +0,0 @@
|
||||
[Unit]
|
||||
Description=Plex Media Server (quadlet, rootless fleet identity)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Container]
|
||||
Image=lscr.io/linuxserver/plex:1.43.4.10903-e5521bd8c-ls325
|
||||
ContainerName=plex
|
||||
Network=dankserver_appnet:alias=plex
|
||||
Environment=PUID=0
|
||||
Environment=PGID=0
|
||||
Environment=TZ=America/Los_Angeles
|
||||
Volume=/var/lib/plexmediaserver:/config
|
||||
Volume=/mnt/entropy:/mnt/entropy:ro
|
||||
PublishPort=32400:32400
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,34 +0,0 @@
|
||||
# Quadlet unit: MariaDB backing store for RomM (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankserver: ~/.config/containers/systemd/romm-mariadb.container
|
||||
# Unit name: romm-mariadb.service (podman 5.x drops the container- prefix)
|
||||
#
|
||||
# The alias `mariadb` on dankserver_appnet is what makes DB_HOST=mariadb in
|
||||
# the RomM unit resolve. Secrets (MARIADB_ROOT_PASSWORD, MARIADB_PASSWORD)
|
||||
# live in the shared host-local env file /home/dank/Containers/romm/env
|
||||
# (mode 0600); this file holds no secrets.
|
||||
|
||||
[Unit]
|
||||
Description=RomM MariaDB database container
|
||||
After=network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=romm-mariadb
|
||||
Image=docker.io/library/mariadb:11
|
||||
Network=dankserver_appnet:alias=mariadb
|
||||
EnvironmentFile=/home/dank/Containers/romm/env
|
||||
Environment=MARIADB_DATABASE=romm
|
||||
Environment=MARIADB_USER=romm
|
||||
Volume=/home/dank/Containers/romm/mariadb:/var/lib/mysql
|
||||
HealthCmd=mariadb-admin ping -h localhost
|
||||
HealthInterval=10s
|
||||
HealthTimeout=5s
|
||||
HealthRetries=3
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=/home/dank/Containers/romm/env
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,28 +0,0 @@
|
||||
# Quadlet unit: Redis cache for RomM (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankserver: ~/.config/containers/systemd/romm-redis.container
|
||||
# Unit name: romm-redis.service (podman 5.x drops the container- prefix)
|
||||
#
|
||||
# The alias `redis` on dankserver_appnet is what makes REDIS_HOST=redis in
|
||||
# the RomM unit resolve. No secrets: this unit takes no EnvironmentFile.
|
||||
|
||||
[Unit]
|
||||
Description=RomM Redis cache container
|
||||
After=network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=romm-redis
|
||||
Image=docker.io/library/redis:7-alpine
|
||||
Network=dankserver_appnet:alias=redis
|
||||
Volume=/home/dank/Containers/romm/redis:/data
|
||||
HealthCmd=redis-cli ping
|
||||
HealthInterval=10s
|
||||
HealthTimeout=5s
|
||||
HealthRetries=3
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,42 +0,0 @@
|
||||
# Quadlet unit: RomM library manager (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankserver: ~/.config/containers/systemd/romm.container
|
||||
# Unit name: romm.service (podman 5.x drops the container- prefix)
|
||||
#
|
||||
# The whole data root lives at /home/dank/Containers/romm and is mounted at
|
||||
# /romm, so config/, library/, resources/ and assets/ sit inside it exactly as
|
||||
# the compose-era bind mounts + anonymous volume did. Secrets (DB_PASSWD,
|
||||
# ROMM_AUTH_SECRET_KEY, ROMM_PASSWORD, SCREENSCRAPER_DEV_*) live in the
|
||||
# host-local env file /home/dank/Containers/romm/env (mode 0600); this file
|
||||
# holds no secrets. The network dankserver_appnet must exist before first
|
||||
# start (it is the compose-era network; on a fresh host create it first).
|
||||
|
||||
[Unit]
|
||||
Description=RomM library manager container
|
||||
Wants=romm-mariadb.service
|
||||
After=romm-mariadb.service romm-redis.service network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=romm
|
||||
Image=ghcr.io/rommapp/romm:latest
|
||||
Network=dankserver_appnet
|
||||
EnvironmentFile=/home/dank/Containers/romm/env
|
||||
Environment=DB_HOST=mariadb
|
||||
Environment=DB_NAME=romm
|
||||
Environment=DB_USER=romm
|
||||
Environment=REDIS_HOST=redis
|
||||
Environment=HASHEOUS_API_ENABLED=true
|
||||
PublishPort=8085:8080
|
||||
Volume=/home/dank/Containers/romm:/romm
|
||||
HealthCmd=curl -f http://localhost:8080/health
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=3
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=/home/dank/Containers/romm/env
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,25 +0,0 @@
|
||||
[Unit]
|
||||
Description=Tautulli (quadlet, rootless fleet identity)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Container]
|
||||
Image=lscr.io/linuxserver/tautulli:v2.18.1-ls245
|
||||
ContainerName=tautulli
|
||||
Network=dankserver_appnet:alias=tautulli
|
||||
Environment=PUID=0
|
||||
Environment=PGID=0
|
||||
Environment=TZ=America/Los_Angeles
|
||||
Volume=/home/dank/Containers/tautulli:/config
|
||||
PublishPort=8181:8181
|
||||
HealthCmd=curl -fs http://localhost:8181/home
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=3
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# deploy.sh — Deploy inference scripts to remote servers
|
||||
#
|
||||
# Usage: ./deploy.sh [TARGET] [SCRIPT]
|
||||
# TARGET: dankserver, dankclaw, or all
|
||||
# SCRIPT: Script name to deploy (default: all scripts in scripts/)
|
||||
|
||||
TARGET="${1:-all}"
|
||||
SCRIPT="${2:-}"
|
||||
|
||||
# Remote configuration
|
||||
declare -A REMOTE_HOSTS
|
||||
REMOTE_HOSTS=(
|
||||
["dankserver"]="dank@dankserver"
|
||||
["dankclaw"]="dank@dankclaw"
|
||||
)
|
||||
|
||||
declare -A REMOTE_DIRS
|
||||
REMOTE_DIRS=(
|
||||
["dankserver"]="/home/dank/vllm"
|
||||
["dankclaw"]="/home/dank/vllm"
|
||||
)
|
||||
|
||||
declare -A REMOTE_SERVICES
|
||||
REMOTE_SERVICES=(
|
||||
["dankserver"]="vllm-awq-qwen36.service"
|
||||
["dankclaw"]="vllm-thinkingcap.service"
|
||||
)
|
||||
|
||||
deploy_to() {
|
||||
local target="$1"
|
||||
local remote="${REMOTE_HOSTS[$target]}"
|
||||
local remote_dir="${REMOTE_DIRS[$target]}"
|
||||
local service="${REMOTE_SERVICES[$target]}"
|
||||
|
||||
echo "=== Deploying to $target ==="
|
||||
|
||||
# Create remote directory if it doesn't exist
|
||||
sshpass -p 'Assface.3241' ssh "$remote" "mkdir -p $remote_dir"
|
||||
|
||||
# Deploy scripts
|
||||
if [ -n "$SCRIPT" ]; then
|
||||
if [ -f "scripts/$SCRIPT" ]; then
|
||||
echo " Deploying scripts/$SCRIPT..."
|
||||
sshpass -p 'Assface.3241' scp "scripts/$SCRIPT" "$remote:$remote_dir/$SCRIPT"
|
||||
else
|
||||
echo " ERROR: scripts/$SCRIPT not found"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
for script in scripts/*.sh; do
|
||||
if [ -f "$script" ]; then
|
||||
local name=$(basename "$script")
|
||||
echo " Deploying $name..."
|
||||
sshpass -p 'Assface.3241' scp "$script" "$remote:$remote_dir/$name"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Make scripts executable
|
||||
if [ -n "$SCRIPT" ]; then
|
||||
sshpass -p 'Assface.3241' ssh "$remote" "chmod +x $remote_dir/$SCRIPT"
|
||||
else
|
||||
sshpass -p 'Assface.3241' ssh "$remote" "chmod +x $remote_dir/*.sh"
|
||||
fi
|
||||
|
||||
# Restart service if it exists
|
||||
echo " Restarting $service..."
|
||||
sshpass -p 'Assface.3241' ssh "$remote" "echo 'Assface.3241' | sudo -S systemctl restart $service 2>/dev/null || echo 'Service $service not found or failed'"
|
||||
|
||||
echo " Deploy to $target complete"
|
||||
echo ""
|
||||
}
|
||||
|
||||
case "$TARGET" in
|
||||
dankserver|dankclaw)
|
||||
deploy_to "$TARGET"
|
||||
;;
|
||||
all)
|
||||
for target in "${!REMOTE_HOSTS[@]}"; do
|
||||
deploy_to "$target"
|
||||
done
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: Unknown target $TARGET"
|
||||
echo "Usage: $0 [dankserver|dankclaw|all] [script]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "=== Deployment complete ==="
|
||||
@@ -0,0 +1,81 @@
|
||||
# Dankclaw System Specs
|
||||
|
||||
## Server Overview
|
||||
|
||||
Dankclaw is an AMD APU server for vLLM inference. The host runs Ubuntu 26.04 with an AMD Ryzen AI MAX+ 395 Strix Halo APU. The APU uses unified memory — CPU and GPU share the same physical RAM pool.
|
||||
## Hardware Specifications
|
||||
|
||||
### CPU
|
||||
|
||||
| Property | Value |
|
||||
|----------|-------|
|
||||
| Model | AMD RYZEN AI MAX+ 395 w/ Radeon 8060S |
|
||||
| Architecture | x86_64 |
|
||||
| Cores | 16 (1 socket, 1 thread per core) |
|
||||
| Max MHz | 5187.50 |
|
||||
| Min MHz | 625.00 |
|
||||
| NUMA Nodes | 1 (node 0, 62909 MB) |
|
||||
|
||||
### GPU (Integrated)
|
||||
|
||||
| Property | Value |
|
||||
|----------|-------|
|
||||
| Model | AMD Radeon 8060S (Strix Halo) |
|
||||
| Device ID | 0x1586 |
|
||||
| Architecture | gfx1151 (Strix Halo) |
|
||||
| GPU-Visible Memory | 96 GiB (ROCm addressable range) |
|
||||
| Physical RAM | 30 GiB (shared CPU + GPU) |
|
||||
|
||||
### ROCm
|
||||
|
||||
| Property | Value |
|
||||
|----------|-------|
|
||||
| ROCk Module | Loaded |
|
||||
| Runtime Version | 1.18 |
|
||||
| ROCm Version | 7.2.4 (from AMD repos) |
|
||||
| Agent Name | AMD RYZEN AI MAX+ 395 w/ Radeon 8060S |
|
||||
| GPU Arch | gfx1151 (Strix Halo) |
|
||||
| GPU Global Memory | 96 GiB (overcommitted, physical is 30 GiB) |
|
||||
|
||||
### Memory
|
||||
|
||||
| Property | Value |
|
||||
|----------|-------|
|
||||
| System RAM | 30 GiB (shared CPU + GPU) |
|
||||
| GPU-Visible (ROCm) | 96 GiB (overcommitted addressable range) |
|
||||
| Swap | 8.0 GiB |
|
||||
| /dev/shm | ~15 GiB (half of RAM) |
|
||||
|
||||
### Storage
|
||||
|
||||
| Filesystem | Size | Used | Avail | Use% |
|
||||
|------------|------|------|-------|------|
|
||||
| / (nvme0n1p2) | 1.9T | 391G | 1.4T | 22% |
|
||||
|
||||
### OS and Software
|
||||
|
||||
| Property | Value |
|
||||
|----------|-------|
|
||||
| OS | Ubuntu 26.04 LTS (Resolute Raccoon) |
|
||||
| Kernel | 7.0.0-29-generic |
|
||||
| Python | 3.14.4 (system), 3.12.13 (vLLM venv) |
|
||||
| vLLM | 0.26.0+rocm723 |
|
||||
| uv | 0.12.2 |
|
||||
|
||||
## VLLM Relevance
|
||||
|
||||
- Unified memory eliminates separate VRAM limits. The GPU uses system RAM.
|
||||
- ROCm reports 96 GiB GPU memory (overcommitted). Physical RAM is 30 GiB. vLLM sees 96 GiB and will try to use it. Set `--gpu-memory-utilization` conservatively (0.80-0.85) to avoid OOM when the OS needs memory.
|
||||
- Only 8 GiB swap. Set `--swap-space` conservatively (4-8 GiB).
|
||||
- ROCm 7.2.4 from AMD repos. vLLM ROCm support requires ROCm 6.3+.
|
||||
- Python 3.14.4 is very new. vLLM ROCm wheels target Python 3.12. Use a Python 3.12 venv.
|
||||
- Radeon 8060S falls under Ryzen AI MAX series (gfx1151). vLLM lists this as supported.
|
||||
- 1.4 TB free disk space is ample for model storage.
|
||||
- `libmpi_cxx.so.40` is missing from the system. Install `openmpi-bin` or provide a workaround for torch MPI dependency.
|
||||
|
||||
## See Also
|
||||
|
||||
- [[VLLM Overview]]
|
||||
- [[Shared Memory Architecture Dankclaw]]
|
||||
- [[RAM CPU Offloading in VLLM]]
|
||||
- [[Running Qwen3.6 27B FP8 on Dankclaw]]
|
||||
@@ -0,0 +1,68 @@
|
||||
# Dankserver System Specs
|
||||
|
||||
## Server Overview
|
||||
|
||||
Dankserver is an NVIDIA GPU server for vLLM inference. The host runs Ubuntu 24.04 with an RTX 3090 and 62 GB of system RAM.
|
||||
|
||||
## Hardware Specifications
|
||||
|
||||
### CPU
|
||||
|
||||
| Property | Value |
|
||||
|----------|-------|
|
||||
| Architecture | x86_64 |
|
||||
| Vendor | AuthenticAMD |
|
||||
| Cores | 8 (1 socket) |
|
||||
| NUMA Nodes | 1 (node 0, 63503 MB) |
|
||||
|
||||
### GPU
|
||||
|
||||
| Property | Value |
|
||||
|----------|-------|
|
||||
| Model | NVIDIA GeForce RTX 3090 |
|
||||
| VRAM | 24576 MiB (24 GB) |
|
||||
| Driver | 595.84 |
|
||||
| CUDA Version | 13.2 |
|
||||
| CUDA Toolkit | 12.0 (nvcc) |
|
||||
|
||||
### Memory
|
||||
|
||||
| Property | Value |
|
||||
|----------|-------|
|
||||
| System RAM | 62 GiB |
|
||||
| Swap | 127 GiB |
|
||||
| /dev/shm | 32 GiB |
|
||||
|
||||
### Storage
|
||||
|
||||
| Filesystem | Size | Used | Avail | Use% |
|
||||
|------------|------|------|-------|------|
|
||||
| / (nvme0n1p2) | 915G | 849G | 20G | 98% |
|
||||
| /mnt/entropy (dm-0) | 91T | 86T | 977G | 99% |
|
||||
|
||||
### OS and Software
|
||||
|
||||
| Property | Value |
|
||||
|----------|-------|
|
||||
| OS | Ubuntu 24.04.4 LTS (Noble Numbat) |
|
||||
| Kernel | 7.0.0-28-generic |
|
||||
| Python | 3.12.3 |
|
||||
|
||||
### Active Services
|
||||
|
||||
A llama-server process occupies approximately 20 GB of the RTX 3090 VRAM.
|
||||
|
||||
## VLLM Relevance
|
||||
|
||||
- 24 GB VRAM limits model size. Use FP8 quantization for large models.
|
||||
- 62 GB system RAM provides ample swap space for KV cache offloading.
|
||||
- 127 GB swap supports large `--swap-space` values for CPU KV cache paging.
|
||||
- Root disk has only 20 GB free. Store models on /mnt/entropy.
|
||||
- CUDA 12.0 and Python 3.12.3 are compatible with vLLM.
|
||||
|
||||
## See Also
|
||||
|
||||
- [[VLLM Overview]]
|
||||
- [[Hybrid CPU GPU Processing Dankserver]]
|
||||
- [[RAM CPU Offloading in VLLM]]
|
||||
- [[Running Qwen3.6 35B A3B FP8 on Dankserver]]
|
||||
+7
-15
@@ -1,22 +1,14 @@
|
||||
---
|
||||
type: reference
|
||||
description: "Dankserver uses NVIDIA GPU with system RAM for hybrid processing."
|
||||
tags:
|
||||
- llm
|
||||
timestamp: 2026-08-28
|
||||
---
|
||||
|
||||
# Hybrid CPU GPU Processing Dankserver
|
||||
|
||||
## Overview
|
||||
|
||||
Dankserver uses NVIDIA GPU with system RAM for hybrid processing. vLLM splits model weights onto GPU and moves KV cache between GPU VRAM and system RAM. This allows larger models to run on GPUs with limited VRAM.
|
||||
Dankserver uses NVIDIA GPU with system RAM for hybrid processing. VLLM splits model weights onto GPU and moves KV cache between GPU VRAM and system RAM. This allows larger models to run on GPUs with limited VRAM.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Model Weights
|
||||
|
||||
Model weights stay on GPU. vLLM loads the entire model into GPU memory. You cannot offload model weights to CPU. The GPU must hold all model parameters.
|
||||
Model weights stay on GPU. VLLM loads the entire model into GPU memory. You cannot offload model weights to CPU. The GPU must hold all model parameters.
|
||||
|
||||
### KV Cache
|
||||
|
||||
@@ -49,9 +41,9 @@ Set `--gpu-memory-utilization` based on your GPU:
|
||||
|
||||
Lower values leave more GPU memory for KV cache. This reduces swap frequency.
|
||||
|
||||
### Swap Space / KV Offload
|
||||
### Swap Space
|
||||
|
||||
Recent vLLM releases removed the `--swap-space` engine argument. Use `--kv-offloading-size` to move KV cache to CPU RAM. In Podman, mount the host cache read-write for KV offload writes. The table below shows historical `--swap-space` guidance as a sizing reference.
|
||||
Set `--swap-space` based on system RAM:
|
||||
|
||||
| System RAM | Recommended Swap |
|
||||
|------------|------------------|
|
||||
@@ -78,7 +70,7 @@ vllm serve meta-llama/Llama-3.1-70B-Instruct \
|
||||
Use FP8 KV cache for Ampere and newer GPUs:
|
||||
|
||||
```
|
||||
vllm serve Qwen/Qwen3.6-35B-A3B-FP8 \
|
||||
vllm serve Qwen/Qwen3-30B-A3B-FP8 \
|
||||
--kv-cache-dtype fp8 \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
--swap-space 16
|
||||
@@ -185,7 +177,7 @@ htop
|
||||
### Single GPU, Medium Model
|
||||
|
||||
```
|
||||
vllm serve Qwen/Qwen3.6-35B-A3B \
|
||||
vllm serve Qwen/Qwen3-30B-A3B \
|
||||
--gpu-memory-utilization 0.85 \
|
||||
--swap-space 16 \
|
||||
--max-num-batched-tokens 4096 \
|
||||
@@ -195,7 +187,7 @@ vllm serve Qwen/Qwen3.6-35B-A3B \
|
||||
### Single GPU, FP8 Model
|
||||
|
||||
```
|
||||
vllm serve Qwen/Qwen3.6-35B-A3B-FP8 \
|
||||
vllm serve Qwen/Qwen3-30B-A3B-FP8 \
|
||||
--kv-cache-dtype fp8 \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
--swap-space 16 \
|
||||
+12
-22
@@ -1,19 +1,8 @@
|
||||
---
|
||||
type: reference
|
||||
description: "vLLM uses CPU RAM and swap space to extend GPU memory capacity."
|
||||
tags:
|
||||
- llm
|
||||
- vllm
|
||||
timestamp: 2026-08-28
|
||||
---
|
||||
|
||||
# RAM CPU Offloading in vLLM
|
||||
# RAM CPU Offloading in VLLM
|
||||
|
||||
## Overview
|
||||
|
||||
vLLM uses CPU RAM and swap space to extend GPU memory capacity. This allows larger models to run on GPUs with limited VRAM. vLLM does not offload model weights to CPU. Instead, vLLM moves KV cache data between GPU and CPU.
|
||||
|
||||
> **Current vLLM note:** Recent vLLM releases removed the `--swap-space` engine argument. CPU KV-cache offloading now uses `--kv-offloading-size`. See the parameters below for details.
|
||||
VLLM uses CPU RAM and swap space to extend GPU memory capacity. This allows larger models to run on GPUs with limited VRAM. VLLM does not offload model weights to CPU. Instead, VLLM moves KV cache data between GPU and CPU.
|
||||
|
||||
## How CPU Offloading Works
|
||||
|
||||
@@ -37,19 +26,19 @@ This parameter controls the fraction of GPU memory used for KV cache.
|
||||
|-------|-------------|
|
||||
| 0.7 | Uses 70% of GPU memory for KV cache |
|
||||
| 0.8 | Uses 80% of GPU memory for KV cache |
|
||||
| 0.92 | Uses 92% of GPU memory for KV cache (default) |
|
||||
| 0.9 | Uses 90% of GPU memory for KV cache (default) |
|
||||
| 0.95 | Uses 95% of GPU memory for KV cache |
|
||||
|
||||
Lower values leave more GPU memory for model weights. Higher values allow more concurrent requests but increase swap frequency.
|
||||
|
||||
### swap_space (removed) / kv-offloading-size (current)
|
||||
### swap_space
|
||||
|
||||
The `--swap-space` argument was removed from current vLLM engine args. Use `--kv-offloading-size` instead. The table below describes the historical `--swap-space` values for reference.
|
||||
This parameter sets the CPU swap space in GiB.
|
||||
|
||||
| Value | Description |
|
||||
|-------|-------------|
|
||||
| 0 | No CPU swap space |
|
||||
| 4 | 4 GiB swap space (historical default) |
|
||||
| 4 | 4 GiB swap space (default) |
|
||||
| 8 | 8 GiB swap space |
|
||||
| 16 | 16 GiB swap space |
|
||||
| 32 | 32 GiB swap space |
|
||||
@@ -89,7 +78,8 @@ This parameter sets the maximum number of sequences per batch.
|
||||
| Value | Description |
|
||||
|-------|-------------|
|
||||
| 128 | Default for online serving |
|
||||
| 256 | Large batch |
|
||||
| 256 | Default for offline inference |
|
||||
| 512 | Large batch |
|
||||
|
||||
### kv_cache_dtype
|
||||
|
||||
@@ -176,10 +166,10 @@ If you see frequent preemptions:
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| vLLM_CPU_KVCACHE_SPACE | KV cache space in GiB for CPU backend | 4 |
|
||||
| vLLM_CPU_OMP_THREADS_BIND | CPU thread binding for OpenMP | auto |
|
||||
| vLLM_CPU_NUM_OF_RESERVED_CPU | Reserved CPU cores per rank | 0-1 |
|
||||
| vLLM_CACHE_ROOT | torch.compile cache directory | ~/.cache/vllm |
|
||||
| VLLM_CPU_KVCACHE_SPACE | KV cache space in GiB for CPU backend | 4 |
|
||||
| VLLM_CPU_OMP_THREADS_BIND | CPU thread binding for OpenMP | auto |
|
||||
| VLLM_CPU_NUM_OF_RESERVED_CPU | Reserved CPU cores per rank | 0-1 |
|
||||
| VLLM_CACHE_ROOT | torch.compile cache directory | ~/.cache/vllm |
|
||||
|
||||
## See Also
|
||||
|
||||
+13
-14
@@ -1,11 +1,3 @@
|
||||
---
|
||||
type: reference
|
||||
description: "Qwen3.6-27B is a dense vision-language model from the Qwen3.6 series."
|
||||
tags:
|
||||
- llm
|
||||
timestamp: 2026-09-03
|
||||
---
|
||||
|
||||
# Running Qwen3.6 27B FP8 on Dankclaw
|
||||
|
||||
## Model Information
|
||||
@@ -34,7 +26,7 @@ Qwen3.6-27B is a dense vision-language model from the Qwen3.6 series. It uses 27
|
||||
|----------|-------|
|
||||
| CPU | AMD RYZEN AI MAX+ 395 w/ Radeon 8060S (Strix Halo) |
|
||||
| GPU | Integrated Radeon 8060S (unified memory) |
|
||||
| Physical RAM | 64 GiB (61 GiB visible; corrected 2026-08-17) |
|
||||
| Physical RAM | 30 GiB (shared CPU + GPU) |
|
||||
| GPU-Visible (ROCm) | 96 GiB (overcommitted) |
|
||||
| Swap | 8 GiB |
|
||||
| ROCm | 7.2.4 |
|
||||
@@ -51,12 +43,10 @@ Qwen3.6-27B is a dense vision-language model from the Qwen3.6 series. It uses 27
|
||||
| Overhead | ~2 GB | ~2 GB |
|
||||
| **Total (approximate)** | ~58 GB | ~16.5 GB |
|
||||
|
||||
The FP8 variant requires approximately 16-17 GB. ThinkingCap-FP8 requires ~29 GB. Both fit within the 61 GiB physical RAM, but ROCm reports 96 GiB to vLLM. Set `--gpu-memory-utilization` conservatively (0.80) to avoid OOM.
|
||||
The FP8 variant requires approximately 16-17 GB. ThinkingCap-FP8 requires ~29 GB. Both fit within the 30 GiB physical RAM, but ROCm reports 96 GiB to vLLM. Set `--gpu-memory-utilization` conservatively (0.80) to avoid OOM.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
> **Production note:** Dankclaw's primary service now runs vLLM-ROCm with the Ling-3.0-flash CIRU INT4 model (see `scripts/frank-large.service`). This page documents the alternative vLLM FP8 deployment of Qwen3.6-27B.
|
||||
|
||||
Install vLLM for ROCm. Version 0.24.0+ is required for Qwen3.6 with speculative decoding:
|
||||
|
||||
```
|
||||
@@ -65,7 +55,7 @@ source .venv/bin/activate
|
||||
uv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/
|
||||
```
|
||||
|
||||
Installed: vLLM 0.26.0 in Python 3.12.13 venv at `/home/dank/vllm-rocm-venv`.
|
||||
Installed: vLLM 0.26.0+rocm723 in Python 3.12.13 venv at `/home/dank/vllm-env`.
|
||||
|
||||
### MPI Dependency
|
||||
|
||||
@@ -84,6 +74,15 @@ export HSA_OVERRIDE_GFX_VERSION=11.5.1
|
||||
export HF_HOME=/home/dank/.cache/huggingface
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Set these for ROCm on Strix Halo:
|
||||
|
||||
```
|
||||
export HSA_OVERRIDE_GFX_VERSION=11.5.1
|
||||
export HF_HOME=/home/dank/.cache/huggingface
|
||||
```
|
||||
|
||||
## vLLM Serve Commands
|
||||
|
||||
### Text-Only (Recommended for Memory Savings)
|
||||
@@ -199,7 +198,7 @@ VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 vllm serve Qwen/Qwen3.6-27B-FP8 \
|
||||
## ROCm Notes
|
||||
|
||||
- Strix Halo (gfx1151) is listed as a supported AMD GPU for vLLM
|
||||
- FP8 uses **software emulation** on RDNA architecture (gfx1151 has no native FP8 tensor cores). This gives memory savings, not full compute speedup. Compare against a BF16 test first
|
||||
- FP8 may use software emulation on RDNA architecture. Verify with a BF16 test first
|
||||
- Attention backends: try `ROCM_AITER_UNIFIED_ATTN` if the default backend underperforms
|
||||
- Context length on ROCm may be limited for some models. Start with 131072 and increase gradually
|
||||
|
||||
+3
-18
@@ -1,15 +1,5 @@
|
||||
---
|
||||
type: reference
|
||||
description: "Qwen3.6-35B-A3B is a Mixture-of-Experts vision-language model from the Qwen3.6 series."
|
||||
tags:
|
||||
- llm
|
||||
timestamp: 2026-09-03
|
||||
---
|
||||
|
||||
# Running Qwen3.6 35B A3B FP8 on Dankserver
|
||||
|
||||
> **RETIRED.** This page documents the FP8 vLLM path for Qwen3.6-35B-A3B on Dankserver. The llama.cpp 35B service is retired (2026-08-16), its unit file was removed (2026-08-17), and the FP8 vLLM unit (`vllm-qwen36`) never shipped and is disabled. The content is kept as research history. The RTX 3090 now runs the PP worker: see [[Qwen3.8 PP Deployment]].
|
||||
|
||||
## Model Information
|
||||
|
||||
Qwen3.6-35B-A3B is a Mixture-of-Experts vision-language model from the Qwen3.6 series. It uses 35B total parameters with only 3B activated per token.
|
||||
@@ -59,12 +49,6 @@ Install vLLM. Version 0.19.0 or later is required for Qwen3.6:
|
||||
uv pip install vllm --torch-backend=auto
|
||||
```
|
||||
|
||||
## Hardware Quantization Notes (RTX 3090 / Ampere)
|
||||
|
||||
The RTX 3090 is an Ampere GPU (compute capability 8.6). It has **no native FP8 tensor cores**. FP8 W8A8 compute runs through a fallback/dequantization path, so the FP8 model gives memory savings but not full compute speedup. vLLM loads these models and runs them; verify quality against a BF16 run. `--kv-cache-dtype fp8` is still valid on CUDA 12 for an FP8 KV cache.
|
||||
|
||||
> Do not run on production without a BF16 comparison. The FP8 precision path may differ from native-FP8 hardware.
|
||||
|
||||
## vLLM Serve Commands
|
||||
|
||||
### Text-Only (Recommended for Memory Savings)
|
||||
@@ -107,7 +91,7 @@ vllm serve Qwen/Qwen3.6-35B-A3B-FP8 \
|
||||
|-----------|-------|--------|
|
||||
| --kv-cache-dtype fp8 | FP8 KV cache | Cuts KV cache memory by 50% |
|
||||
| --gpu-memory-utilization 0.75 | 75% of VRAM | Leaves room for all MoE weights |
|
||||
| --swap-space 32 (removed) | 32 GB CPU overflow | Modern vLLM removed swap-space; use --kv-offloading-size |
|
||||
| --swap-space 32 | 32 GB CPU swap | Dankserver has 62 GB RAM |
|
||||
| --max-model-len 131072 | 128K context | Full 262K context is too large for 24 GB |
|
||||
| --max-num-batched-tokens 2048 | Conservative batch | Prevents OOM with MoE |
|
||||
| --reasoning-parser qwen3 | Qwen3 parser | Required for thinking mode output |
|
||||
@@ -122,9 +106,10 @@ export HF_HOME=/mnt/entropy/huggingface
|
||||
|
||||
### Active Service Conflict
|
||||
|
||||
The retired `frank-small` service (llama.cpp) used about 22 GB of the 24 GB VRAM. The current PP worker holds ~23.8 GiB. The `vllm-awq-qwen36` (Podman) and `vllm-qwen36` units are disabled. Before starting any research service, verify that the GPU is free:
|
||||
A llama-server process occupies approximately 20 GB of VRAM. Stop it before starting vLLM:
|
||||
|
||||
```
|
||||
pkill llama-server
|
||||
nvidia-smi # verify GPU is free
|
||||
```
|
||||
|
||||
+11
-20
@@ -1,11 +1,3 @@
|
||||
---
|
||||
type: reference
|
||||
description: "Dankclaw uses AMD Strix Halo APU with unified memory architecture."
|
||||
tags:
|
||||
- llm
|
||||
timestamp: 2026-08-28
|
||||
---
|
||||
|
||||
# Shared Memory Architecture Dankclaw
|
||||
|
||||
## Overview
|
||||
@@ -21,12 +13,12 @@ In unified memory:
|
||||
1. CPU and GPU share one physical memory pool
|
||||
2. The OS and GPU driver manage memory allocation
|
||||
3. There is no separate VRAM or system RAM
|
||||
4. The total available memory appears as GPU memory to vLLM
|
||||
4. The total available memory appears as GPU memory to VLLM
|
||||
5. Memory pages move between CPU and GPU transparently
|
||||
|
||||
### Implications for vLLM
|
||||
### Implications for VLLM
|
||||
|
||||
- vLLM sees the full memory as GPU memory
|
||||
- VLLM sees the full memory as GPU memory
|
||||
- No explicit swap_space parameter needed for memory extension
|
||||
- The GPU driver handles memory paging automatically
|
||||
- Performance depends on memory bandwidth and latency
|
||||
@@ -92,12 +84,13 @@ Since memory is unified, set these parameters carefully:
|
||||
```
|
||||
vllm serve model \
|
||||
--gpu-memory-utilization 0.85 \
|
||||
--swap-space 8 \
|
||||
--max-num-batched-tokens 4096
|
||||
```
|
||||
|
||||
- `gpu_memory_utilization`: Set lower (0.8-0.85) to leave room for CPU operations
|
||||
- `swap_space`: Keep moderate (8-16 GB) for overflow
|
||||
- `max_model_len`: Match actual context needs
|
||||
- The old `--swap-space` argument was removed from current vLLM. Use `--kv-offloading-size` for CPU KV offload
|
||||
|
||||
### NUMA Binding
|
||||
|
||||
@@ -150,7 +143,7 @@ Unified memory has lower bandwidth than dedicated VRAM:
|
||||
|
||||
### ROCm Overcommit
|
||||
|
||||
ROCm reports ~114 GiB of GPU memory on Strix Halo APU (the GTT aperture, `gtt_total=122402963456` bytes = 114 GiB; `llama-server --list-devices` shows 116,732 MiB — the 2026-08-17 "96 GiB" figure was stale). Physical RAM is 64 GiB (61 GiB visible; the box was upgraded from 32 GB). The GPU can address more than physical RAM exists because the GTT aperture maps over the shared LPDDR. vLLM will see ~114 GiB and try to allocate accordingly. Set `--gpu-memory-utilization` to 0.80 or lower to prevent OOM.
|
||||
ROCm reports 96 GiB GPU memory on Strix Halo APU. Physical RAM is 30 GiB. The GPU can address more than physical RAM exists. vLLM will see 96 GiB and try to allocate accordingly. Set `--gpu-memory-utilization` to 0.80 or lower to prevent OOM.
|
||||
|
||||
Lower bandwidth means slower inference. Plan for lower throughput.
|
||||
|
||||
@@ -204,17 +197,15 @@ Some models on ROCm have context length limited to 4096 tokens. Check model docu
|
||||
FP8 quantization depends on GPU architecture:
|
||||
|
||||
- MI300 series supports native FP8
|
||||
- Radeon RX 7900 supports FP8 via software emulation
|
||||
- Strix Halo (gfx1151) uses **software emulation** for FP8 (no native FP8 tensor cores); expect memory savings, not full compute speedup
|
||||
|
||||
For FP4 on Dankclaw, the ROCmFPX llama.cpp fork (`Q2_0_ROCMFP4`) provides native ROCm FP4 kernels.
|
||||
- Radeon RX 7900 supports FP8 via software
|
||||
- Strix Halo may support FP8 via software emulation
|
||||
|
||||
## Example Commands
|
||||
|
||||
### Basic APU Deployment
|
||||
|
||||
```
|
||||
vllm serve Qwen/Qwen3.6-35B-A3B \
|
||||
vllm serve Qwen/Qwen3-30B-A3B \
|
||||
--attention-backend ROCM_AITER_UNIFIED_ATTN \
|
||||
--gpu-memory-utilization 0.85 \
|
||||
--swap-space 8 \
|
||||
@@ -225,7 +216,7 @@ vllm serve Qwen/Qwen3.6-35B-A3B \
|
||||
### With FP8 Model
|
||||
|
||||
```
|
||||
vllm serve Qwen/Qwen3.6-35B-A3B-FP8 \
|
||||
vllm serve Qwen/Qwen3-30B-A3B-FP8 \
|
||||
--attention-backend ROCM_AITER_UNIFIED_ATTN \
|
||||
--gpu-memory-utilization 0.85 \
|
||||
--kv-cache-dtype fp8 \
|
||||
@@ -235,7 +226,7 @@ vllm serve Qwen/Qwen3.6-35B-A3B-FP8 \
|
||||
### With Long Context
|
||||
|
||||
```
|
||||
vllm serve Qwen/Qwen3.6-35B-A3B \
|
||||
vllm serve Qwen/Qwen3-30B-A3B \
|
||||
--attention-backend ROCM_AITER_UNIFIED_ATTN \
|
||||
--gpu-memory-utilization 0.8 \
|
||||
--swap-space 16 \
|
||||
@@ -1,36 +1,25 @@
|
||||
---
|
||||
type: reference
|
||||
description: "This index contains all documentation for vLLM deployment on Dankserver and Dankclaw."
|
||||
tags:
|
||||
- llm
|
||||
- vllm
|
||||
timestamp: 2026-08-28
|
||||
---
|
||||
|
||||
# vLLM Index
|
||||
|
||||
> **RETIRED (2026-08-18).** The vLLM service on Dankclaw was replaced by the CIRU ROCmFPX llama.cpp runtime (v3). These pages are historical reference. See [[ciru-activefpx-promptforge]].
|
||||
# VLLM Index
|
||||
|
||||
## Documentation Overview
|
||||
|
||||
This index contains all documentation for vLLM deployment on Dankserver and Dankclaw.
|
||||
This index contains all documentation for VLLM deployment on Dankserver and Dankclaw.
|
||||
|
||||
## Hardware Guides
|
||||
|
||||
- [[Dankserver System Specs]] — NVIDIA GPU server specifications
|
||||
- [[Dankclaw System Specs]] — AMD GPU server specifications
|
||||
- [[Dankserver System Specs]] -- NVIDIA GPU server specifications
|
||||
- [[Dankclaw System Specs]] -- AMD GPU server specifications
|
||||
|
||||
## vLLM Architecture
|
||||
## VLLM Architecture
|
||||
|
||||
- [[VLLM Overview]] — Complete vLLM documentation including installation, CLI, API, and key arguments
|
||||
- [[RAM CPU Offloading in VLLM]] — CPU RAM offloading mechanisms, swap space, and memory configuration
|
||||
- [[Hybrid CPU GPU Processing Dankserver]] — NVIDIA GPU with system RAM hybrid processing
|
||||
- [[Shared Memory Architecture Dankclaw]] — AMD ROCm unified memory and APU configuration
|
||||
- [[VLLM Overview]] -- Complete VLLM documentation including installation, CLI, API, and key arguments
|
||||
- [[RAM CPU Offloading in VLLM]] -- CPU RAM offloading mechanisms, swap space, and memory configuration
|
||||
- [[Hybrid CPU GPU Processing Dankserver]] -- NVIDIA GPU with system RAM hybrid processing
|
||||
- [[Shared Memory Architecture Dankclaw]] -- AMD ROCm unified memory and APU configuration
|
||||
|
||||
## Model Deployment Guides
|
||||
|
||||
- [[Running Qwen3.6 35B A3B FP8 on Dankserver]] — Qwen3.6-35B-A3B FP8 deployment on NVIDIA GPU
|
||||
- [[Running Qwen3.6 27B FP8 on Dankclaw]] — Qwen3.6-27B FP8 deployment on AMD GPU
|
||||
- [[Running Qwen3.6 35B A3B FP8 on Dankserver]] -- Qwen3-30B-A3B FP8 deployment on NVIDIA GPU
|
||||
- [[Running Qwen3.6 27B FP8 on Dankclaw]] -- Qwen3-32B FP8 deployment on AMD GPU
|
||||
|
||||
## Quick Reference
|
||||
|
||||
@@ -78,20 +67,20 @@ vllm serve Qwen/Qwen2.5-1.5B-Instruct --gpu-memory-utilization 0.8 --swap-space
|
||||
| Parameter | Description | Typical Value |
|
||||
|-----------|-------------|---------------|
|
||||
| tensor_parallel_size | Number of GPUs for model sharding | 1-8 |
|
||||
| gpu_memory_utilization | Fraction of GPU memory for KV cache (default 0.92) | 0.7-0.95 |
|
||||
| swap_space | Removed in current vLLM; use `--kv-offloading-size` | n/a |
|
||||
| gpu_memory_utilization | Fraction of GPU memory for KV cache | 0.7-0.95 |
|
||||
| swap_space | CPU swap space in GiB | 4-32 |
|
||||
| max_model_len | Maximum context length | auto |
|
||||
| max_num_batched_tokens | Max tokens per batch | 2048-8192 |
|
||||
| max_num_seqs | Max sequences per batch | 128-512 |
|
||||
| max_num_seqs | Max sequences per batch | 128-256 |
|
||||
|
||||
### Quantization Methods
|
||||
|
||||
| Method | GPU Support | Description |
|
||||
|--------|-------------|-------------|
|
||||
| FP8 | Hopper, Ada (native); Ampere W8A16 via Marlin; RDNA via emulation | FP8 E4M3 inference |
|
||||
| FP8 | Hopper, Ada, Ampere | Native FP8 inference |
|
||||
| INT8 | Most GPUs | Weight-only INT8 |
|
||||
| GPTQ | Most GPUs (NVIDIA; limited on ROCm) | Post-training GPTQ |
|
||||
| AWQ | Turing+ (NVIDIA) | Activation-aware weight quantization |
|
||||
| GPTQ | Most GPUs | Post-training GPTQ |
|
||||
| AWQ | Turing+ | Activation-aware weight quantization |
|
||||
| GGUF | CPU, GPU | General GPU unification format |
|
||||
|
||||
## See Also
|
||||
@@ -1,30 +1,21 @@
|
||||
---
|
||||
type: reference
|
||||
description: "vLLM is a fast and easy library for large language model inference and serving."
|
||||
tags:
|
||||
- llm
|
||||
- vllm
|
||||
timestamp: 2026-08-28
|
||||
---
|
||||
# VLLM Overview
|
||||
|
||||
# vLLM Overview
|
||||
## What Is VLLM
|
||||
|
||||
## What Is vLLM
|
||||
VLLM is a fast and easy library for large language model inference and serving. You can use VLLM to run models on NVIDIA GPUs, AMD GPUs, Intel GPUs, and CPUs. VLLM supports over 200 model architectures from Hugging Face.
|
||||
|
||||
vLLM is a fast and easy library for large language model inference and serving. You can use vLLM to run models on NVIDIA GPUs, AMD GPUs, Intel GPUs, and CPUs. vLLM supports over 200 model architectures from Hugging Face.
|
||||
|
||||
vLLM is fast with these features:
|
||||
VLLM is fast with these features:
|
||||
|
||||
- PagedAttention for efficient key and value memory management
|
||||
- Continuous batching of incoming requests
|
||||
- Chunked prefill and prefix caching
|
||||
- FP8, INT8, INT4, GPTQ, AWQ, and GGUF quantization
|
||||
- Tensor, pipeline, data, expert, and context parallelism
|
||||
- Speculative decoding with n-gram, suffix, EAGLE/EAGLE3, DFlash, DSpark, and MTP
|
||||
- Speculative decoding with n-gram, suffix, and EAGLE
|
||||
- Automatic kernel generation with torch.compile
|
||||
- Disaggregated prefill, decode, and encode
|
||||
|
||||
vLLM is flexible with these features:
|
||||
VLLM is flexible with these features:
|
||||
|
||||
- Seamless integration with Hugging Face models
|
||||
- High-throughput serving with parallel sampling and beam search
|
||||
@@ -37,7 +28,7 @@ vLLM is flexible with these features:
|
||||
|
||||
## Installation
|
||||
|
||||
You can install vLLM with uv (recommended) or pip:
|
||||
You can install VLLM with uv (recommended) or pip:
|
||||
|
||||
```
|
||||
uv pip install vllm
|
||||
@@ -139,11 +130,11 @@ curl http://localhost:8000/v1/chat/completions \
|
||||
|----------|---------|-------------|
|
||||
| `tensor_parallel_size` | 1 | Number of GPUs for tensor parallelism |
|
||||
| `pipeline_parallel_size` | 1 | Number of GPUs for pipeline parallelism |
|
||||
| `gpu_memory_utilization` | 0.92 | Fraction of GPU memory for KV cache |
|
||||
| `swap_space` | removed | Removed in current vLLM; use `--kv-offloading-size` |
|
||||
| `gpu_memory_utilization` | 0.9 | Fraction of GPU memory for KV cache |
|
||||
| `swap_space` | 4 | CPU swap space in GiB |
|
||||
| `enforce_eager` | false | Use eager mode (no CUDA graphs) |
|
||||
| `max_num_batched_tokens` | 2048 | Max tokens per batch |
|
||||
| `max_num_seqs` | 128 | Max sequences per batch |
|
||||
| `max_num_seqs` | 256 | Max sequences per batch |
|
||||
| `block_size` | 16 | Token block size |
|
||||
| `num_gpu_blocks_override` | null | Override number of GPU blocks |
|
||||
| `max_model_len_per_seq` | null | Max length per sequence |
|
||||
@@ -166,8 +157,8 @@ curl http://localhost:8000/v1/chat/completions \
|
||||
|
||||
| Argument | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `gpu_memory_utilization` | 0.92 | Fraction of GPU memory for KV cache |
|
||||
| `swap_space` | removed | Removed in current vLLM; use `--kv-offloading-size` |
|
||||
| `gpu_memory_utilization` | 0.9 | Fraction of GPU memory for KV cache |
|
||||
| `swap_space` | 4 | CPU swap space in GiB |
|
||||
| `kv_cache_dtype` | auto | KV cache data type (auto, fp8, fp8_e5m2) |
|
||||
| `max_model_len` | auto | Maximum context length |
|
||||
| `num_gpu_blocks` | auto | Number of GPU blocks |
|
||||
@@ -183,15 +174,15 @@ curl http://localhost:8000/v1/chat/completions \
|
||||
|
||||
## Architecture
|
||||
|
||||
vLLM uses a multi-process architecture:
|
||||
VLLM uses a multi-process architecture:
|
||||
|
||||
- API server process handles HTTP requests, tokenization, and input processing
|
||||
- Engine core process runs the scheduler and coordinates GPU workers
|
||||
- One GPU worker process per GPU executes model forward passes
|
||||
|
||||
vLLM uses PagedAttention for memory management. PagedAttention treats KV cache as a set of memory pages. This allows efficient memory allocation and reduces fragmentation.
|
||||
VLLM uses PagedAttention for memory management. PagedAttention treats KV cache as a set of memory pages. This allows efficient memory allocation and reduces fragmentation.
|
||||
|
||||
vLLM supports multiple attention backends:
|
||||
VLLM supports multiple attention backends:
|
||||
|
||||
- FlashAttention for NVIDIA GPUs
|
||||
- FlashInfer for NVIDIA GPUs
|
||||
@@ -200,7 +191,7 @@ vLLM supports multiple attention backends:
|
||||
|
||||
## Supported Models
|
||||
|
||||
vLLM supports over 200 model architectures including:
|
||||
VLLM supports over 200 model architectures including:
|
||||
|
||||
- Decoder-only LLMs (Llama, Qwen, Gemma)
|
||||
- Mixture-of-Expert LLMs (Mixtral, DeepSeek-V3, Qwen-MoE)
|
||||
@@ -211,25 +202,23 @@ vLLM supports over 200 model architectures including:
|
||||
|
||||
## Quantization
|
||||
|
||||
vLLM supports these quantization methods:
|
||||
VLLM supports these quantization methods:
|
||||
|
||||
- FP8 for Hopper and Ada GPUs (native). Ampere runs FP8 W8A16 via Marlin; FP8 W8A8 computes via fallback
|
||||
- FP8 for Hopper and Ada GPUs
|
||||
- MXFP8/MXFP4 for Hopper GPUs
|
||||
- NVFP4 for Hopper and Blackwell GPUs
|
||||
- NVFP4 for Hopper GPUs
|
||||
- INT8 for most GPUs
|
||||
- INT4 for most GPUs
|
||||
- GPTQ for most GPUs
|
||||
- AWQ for Turing and newer GPUs (NVIDIA; native support varies on ROCm)
|
||||
- AWQ for Turing and newer GPUs
|
||||
- GGUF for CPU and GPU
|
||||
- Compressed-tensors for custom quantization
|
||||
- ModelOpt for NVIDIA Model Optimizer
|
||||
- TorchAO for PyTorch AO
|
||||
|
||||
AMD RDNA 3 / RDNA 3.5 (gfx11, Strix Halo) run FP8 via software emulation, not native tensor cores. AWQ and GPTQ support in vLLM on AMD ROCm is limited; use GGUF or AMD Quark.
|
||||
|
||||
## Parallelism
|
||||
|
||||
vLLM supports multiple parallelism strategies:
|
||||
VLLM supports multiple parallelism strategies:
|
||||
|
||||
- Tensor Parallelism (TP) shards model parameters across GPUs
|
||||
- Pipeline Parallelism (PP) distributes model layers across GPUs
|
||||
@@ -0,0 +1,83 @@
|
||||
# AWQ + DFlash on Ampere Hardware
|
||||
|
||||
## Date: 2026-08-08
|
||||
|
||||
## Objective
|
||||
Deploy Qwen3.6-35B-A3B with AWQ 4bit quantization and DFlash speculative decoding on NVIDIA Ampere-class GPU.
|
||||
|
||||
## What Worked
|
||||
|
||||
### Base AWQ Model
|
||||
- `cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit` loads successfully
|
||||
- AWQ Marlin kernel engaged automatically for weight conversion
|
||||
- `--cpu-offload-gb 40` with `--cpu-offload-params "experts"` helps fit model in memory
|
||||
- `--gpu-memory-utilization 0.95` is safe for this model
|
||||
|
||||
### DFlash Draft Model
|
||||
- `z-lab/Qwen3.6-35B-A3B-DFlash` detected as EAGLE model
|
||||
- torch.compile with cudagraphs works with `--max-num-batched-tokens 1536`
|
||||
- Draft model shares embedding/lm_head weights with target model
|
||||
|
||||
### Performance (no speculative decoding)
|
||||
| Streams | Prompt Speed | Completion Speed | Total Time |
|
||||
|---------|-------------|-----------------|------------|
|
||||
| 1 | 4.27 tok/s | 21.42 tok/s | 11,951ms |
|
||||
| 2 | 2.44 tok/s | 12.25 tok/s | 20,891ms |
|
||||
| 3 | 1.35 tok/s | 6.76 tok/s | 37,849ms |
|
||||
| 4 | 1.43 tok/s | 7.19 tok/s | 35,606ms |
|
||||
|
||||
Note: Speeds decrease with more streams due to KV cache contention.
|
||||
|
||||
## What Didn't Work
|
||||
|
||||
### Prefix Caching + Mamba Cache Conflict
|
||||
- **Error**: `AssertionError` in `unify_kv_cache_spec_page_size`
|
||||
- **Cause**: Mamba cache "align" mode conflicts with prefix caching
|
||||
- **Fix**: Remove `--enable-prefix-caching` from command line
|
||||
|
||||
### OOM with 262144 Context
|
||||
- **Error**: CUDA out of memory during AWQ→Marlin weight conversion
|
||||
- **Cause**: 24GB GPU cannot handle 262144 context + AWQ conversion
|
||||
- **Fix**: Use `--max-model-len 32768` (32k context)
|
||||
|
||||
### Speculative Decoding KV Cache Bug
|
||||
- **Error**: `AssertionError` in nightly vLLM when using DFlash
|
||||
- **Cause**: Known bug in vLLM nightly with Mamba cache + speculative decoding
|
||||
- **Status**: Temporarily disabled speculative decoding to test base model
|
||||
|
||||
## Systemd Service
|
||||
- Service: `vllm-awq-qwen36.service`
|
||||
- ExecStart: `/home/dank/vllm/vllm-awq-qwen36-dflash-ampere.sh`
|
||||
- Port: 34401 (to avoid conflicts with other services)
|
||||
- Status: Disabled (pending bug fixes)
|
||||
|
||||
## Configuration Summary
|
||||
|
||||
### Working Configuration
|
||||
```bash
|
||||
--max-model-len 32768
|
||||
--cpu-offload-gb 40
|
||||
--cpu-offload-params "experts"
|
||||
--gpu-memory-utilization 0.95
|
||||
--max-num-batched-tokens 1536
|
||||
--performance-mode throughput
|
||||
--enable-chunked-prefill
|
||||
# --enable-prefix-caching # DISABLED - conflicts with Mamba
|
||||
```
|
||||
|
||||
### Thread Binding
|
||||
```bash
|
||||
-e OMP_NUM_THREADS=8
|
||||
-e VLLM_CPU_OMP_THREADS_BIND=8
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
1. Monitor vLLM nightly for Mamba cache + speculative decoding fix
|
||||
2. Test with larger context windows when memory allows
|
||||
3. Benchmark with speculative decoding enabled (when bug is fixed)
|
||||
4. Consider using stable vLLM release instead of nightly
|
||||
|
||||
## References
|
||||
- Model: https://huggingface.co/cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit
|
||||
- Draft: https://huggingface.co/z-lab/Qwen3.6-35B-A3B-DFlash
|
||||
- vLLM Docs: https://docs.vllm.ai/en/stable/
|
||||
@@ -0,0 +1,26 @@
|
||||
# vLLM Scripts — Knowledge Base
|
||||
|
||||
Documentation for running vLLM inference on two local machines.
|
||||
|
||||
## Servers
|
||||
|
||||
| Server | GPU | VRAM / Memory | OS |
|
||||
| --------------------------------------- | ----------------------------- | --------------------- | ------------ |
|
||||
| [[Dankserver System Specs\|Dankserver]] | NVIDIA RTX 3090 | 24 GB VRAM, 62 GB RAM | Ubuntu 24.04 |
|
||||
| [[Dankclaw System Specs\|Dankclaw]] | AMD Radeon 8060S (Strix Halo) | 61 GB unified | Ubuntu 26.04 |
|
||||
|
||||
## vLLM Reference
|
||||
|
||||
- [[VLLM Overview]] — installation, CLI, API, architecture, quantization, parallelism
|
||||
- [[VLLM Index]] — quick reference with commands, endpoints, and parameter table
|
||||
|
||||
## Memory Management
|
||||
|
||||
- [[RAM CPU Offloading in VLLM]] — KV cache swap between GPU VRAM and system RAM
|
||||
- [[Hybrid CPU GPU Processing Dankserver]] — NVIDIA GPU compute with system RAM offload
|
||||
- [[Shared Memory Architecture Dankclaw]] — AMD unified memory and ROCm configuration
|
||||
|
||||
## Model Deployment
|
||||
|
||||
- [[Running Qwen3.6 35B A3B FP8 on Dankserver]] — MoE model (35B total, 3B active) on RTX 3090
|
||||
- [[Running Qwen3.6 27B FP8 on Dankclaw]] — Dense model (27B) on Strix Halo APU
|
||||
@@ -1,12 +1,3 @@
|
||||
---
|
||||
type: reference
|
||||
description: "vLLM v0.26.0 (latest stable, released July 25, 2026) supports DFlash speculative decoding with the `--spec-method dflash` and `--spec-model` flags."
|
||||
tags:
|
||||
- llm
|
||||
- vllm
|
||||
timestamp: 2026-08-28
|
||||
---
|
||||
|
||||
# Research: vLLM Parameters for Specified Configuration
|
||||
|
||||
## Summary
|
||||
@@ -1,49 +0,0 @@
|
||||
# Quadlet unit: Gluetun VPN tunnel in FrankBook-Pro's podman machine.
|
||||
#
|
||||
# Install path is INSIDE the machine VM:
|
||||
# /home/core/.config/containers/systemd/gluetun.container
|
||||
# (installed by the frankbook-pro handler in scripts/deploy-remote.sh —
|
||||
# install_quadlet targets the Linux hosts' systemctl --user, not the VM).
|
||||
#
|
||||
# WireGuard tunnel to ProtonVPN. qBittorrent and all five *arrs share this
|
||||
# container's network namespace (Network=gluetun.container) so every fleet
|
||||
# container's egress — torrent traffic, indexer APIs, metadata lookups —
|
||||
# exits ONLY through the tunnel (AllowedIPs=0.0.0.0/0). Every fleet port
|
||||
# publishes HERE; a netns-sharer publishes no ports of its own (same
|
||||
# invariant as the dankclaw searxng/gluetun pair). Credentials live in
|
||||
# /home/core/Containers/gluetun/env inside the VM (mode 0600, never
|
||||
# committed); the private key is Mac-tunnel-specific and MUST NOT be copied
|
||||
# from dankclaw — one WireGuard peer key cannot serve two machines at once.
|
||||
[Unit]
|
||||
Description=Gluetun VPN tunnel (FrankBook-Pro podman machine)
|
||||
|
||||
[Container]
|
||||
ContainerName=gluetun
|
||||
Image=docker.io/qmcgaw/gluetun:latest
|
||||
AddCapability=NET_ADMIN
|
||||
AddDevice=/dev/net/tun:/dev/net/tun
|
||||
Environment=VPN_SERVICE_PROVIDER=${VPN_SERVICE_PROVIDER}
|
||||
Environment=VPN_TYPE=${VPN_TYPE}
|
||||
Environment=WIREGUARD_PRIVATE_KEY=${WIREGUARD_PRIVATE_KEY}
|
||||
Environment=WIREGUARD_ADDRESSES=${WIREGUARD_ADDRESSES}
|
||||
Environment=WIREGUARD_PUBLIC_KEY=${WIREGUARD_PUBLIC_KEY}
|
||||
Environment=WIREGUARD_ENDPOINT_IP=${WIREGUARD_ENDPOINT_IP}
|
||||
Environment=WIREGUARD_ENDPOINT_PORT=${WIREGUARD_ENDPOINT_PORT}
|
||||
Environment=WIREGUARD_ALLOWED_IPS=${WIREGUARD_ALLOWED_IPS}
|
||||
Environment=WIREGUARD_PERSISTENT_KEEPALIVE_INTERVAL=${WIREGUARD_PERSISTENT_KEEPALIVE_INTERVAL}
|
||||
PublishPort=8080:8080
|
||||
PublishPort=6881:6881
|
||||
PublishPort=6881:6881/udp
|
||||
PublishPort=9696:9696
|
||||
PublishPort=8989:8989
|
||||
PublishPort=7878:7878
|
||||
PublishPort=8686:8686
|
||||
PublishPort=8787:8787
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=/home/core/Containers/gluetun/env
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,31 +0,0 @@
|
||||
# Quadlet unit: Lidarr in FrankBook-Pro's podman machine.
|
||||
#
|
||||
# Install path is INSIDE the machine VM:
|
||||
# /home/core/.config/containers/systemd/lidarr.container
|
||||
#
|
||||
#
|
||||
# Joins gluetun's network namespace: all egress exits through the WireGuard
|
||||
# tunnel; 8686 publishes on the gluetun unit.
|
||||
# Grab-only: no media binds; config migrated from dankserver's retired
|
||||
# bare container.
|
||||
[Unit]
|
||||
Description=Lidarr (FrankBook-Pro)
|
||||
Wants=gluetun.service
|
||||
After=gluetun.service
|
||||
|
||||
[Container]
|
||||
ContainerName=lidarr
|
||||
Image=lscr.io/linuxserver/lidarr:latest
|
||||
Environment=PUID=0
|
||||
Environment=PGID=0
|
||||
Environment=TZ=America/Los_Angeles
|
||||
Environment=UMASK_SET=022
|
||||
Volume=/Volumes/COMPLETE/lidarr-data:/config
|
||||
Network=gluetun.container
|
||||
|
||||
[Service]
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,40 +0,0 @@
|
||||
# Quadlet unit: Prowlarr in FrankBook-Pro's podman machine.
|
||||
#
|
||||
# Install path is INSIDE the machine VM:
|
||||
# /home/core/.config/containers/systemd/prowlarr.container
|
||||
#
|
||||
#
|
||||
# Joins gluetun's network namespace: all egress (indexer APIs, metadata
|
||||
# lookups) exits through the WireGuard tunnel; 9696 publishes on the
|
||||
# gluetun unit. Same-netns targets (qB, the arrs) are loopback addresses.
|
||||
# Arr on the Mac is grab-only: indexer + app-sync management lives here, the
|
||||
# arrs never auto-import media (the user reviews completed torrents in
|
||||
# /Volumes/COMPLETE/"Finished Torrents" and moves files manually into the
|
||||
# library, which lives on dankserver's entropy disk).
|
||||
#
|
||||
# Config is migrated from dankserver's retired bare container
|
||||
# (tar-archived first, per the Phase 4 cutover note) — indexers and app
|
||||
# settings survive; stale host pointers (old qbit at 192.168.0.235:8080)
|
||||
# are re-pointed to loopback qBittorrent (127.0.0.1:8080, same netns) via
|
||||
# its API after first boot.
|
||||
[Unit]
|
||||
Description=Prowlarr (FrankBook-Pro)
|
||||
Wants=gluetun.service
|
||||
After=gluetun.service
|
||||
|
||||
[Container]
|
||||
ContainerName=prowlarr
|
||||
Image=lscr.io/linuxserver/prowlarr:latest
|
||||
Environment=PUID=0
|
||||
Environment=PGID=0
|
||||
Environment=TZ=America/Los_Angeles
|
||||
Environment=UMASK_SET=022
|
||||
Volume=/Volumes/COMPLETE/prowlarr-data:/config
|
||||
Network=gluetun.container
|
||||
|
||||
[Service]
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,46 +0,0 @@
|
||||
# Quadlet unit: qBittorrent in FrankBook-Pro's podman machine.
|
||||
#
|
||||
# Install path is INSIDE the machine VM:
|
||||
# /home/core/.config/containers/systemd/qbittorrent.container
|
||||
#
|
||||
# Joins gluetun's network namespace (Network=gluetun.container) so all
|
||||
# BitTorrent egress exits the WireGuard tunnel. It publishes NO port of its
|
||||
# own — WebUI 8080 and torrent port 6881 publish on the gluetun unit.
|
||||
#
|
||||
# Data layout (plan-locked, verbatim paths):
|
||||
# /config <- /Volumes/COMPLETE/qbittorrent-data (durable config)
|
||||
# /Volumes/COMPLETE + /Volumes/INCOMPLETE same-path virtiofs passthroughs —
|
||||
# the seeded qBittorrent.conf references those literal paths:
|
||||
# Session\DefaultSavePath=/Volumes/COMPLETE
|
||||
# Session\TempPath=/Volumes/INCOMPLETE/incomplete (in-progress disk)
|
||||
# category "Finished Torrents" -> /Volumes/COMPLETE/Finished Torrents
|
||||
# (the human review dir; nothing automated ever writes to entropy).
|
||||
#
|
||||
# WebUI password: generated fresh at provisioning by the deploy handler,
|
||||
# stored machine-local only (conf + /home/core/Containers/qbittorrent/env).
|
||||
[Unit]
|
||||
Description=qBittorrent behind gluetun (FrankBook-Pro)
|
||||
Wants=gluetun.service
|
||||
After=gluetun.service
|
||||
|
||||
[Container]
|
||||
ContainerName=qbittorrent
|
||||
Network=gluetun.container
|
||||
# Pinned to the last 4.x line: Readarr's retired qB client requires the
|
||||
# qB 4.x login response (HTTP 200, body "Ok."); qB 5.x answers 204 with an
|
||||
# empty body, which fails Readarr's auth check. All other clients accept 4.x.
|
||||
Image=lscr.io/linuxserver/qbittorrent:4.6.7
|
||||
Environment=PUID=0
|
||||
Environment=PGID=0
|
||||
Environment=TZ=America/Los_Angeles
|
||||
Environment=UMASK_SET=022
|
||||
Volume=/Volumes/COMPLETE/qbittorrent-data:/config
|
||||
Volume=/Volumes/COMPLETE:/Volumes/COMPLETE
|
||||
Volume=/Volumes/INCOMPLETE:/Volumes/INCOMPLETE
|
||||
|
||||
[Service]
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,31 +0,0 @@
|
||||
# Quadlet unit: Radarr in FrankBook-Pro's podman machine.
|
||||
#
|
||||
# Install path is INSIDE the machine VM:
|
||||
# /home/core/.config/containers/systemd/radarr.container
|
||||
#
|
||||
#
|
||||
# Joins gluetun's network namespace: all egress exits through the WireGuard
|
||||
# tunnel; 7878 publishes on the gluetun unit.
|
||||
# Grab-only: no media binds; config migrated from dankserver's retired
|
||||
# bare container.
|
||||
[Unit]
|
||||
Description=Radarr (FrankBook-Pro)
|
||||
Wants=gluetun.service
|
||||
After=gluetun.service
|
||||
|
||||
[Container]
|
||||
ContainerName=radarr
|
||||
Image=lscr.io/linuxserver/radarr:latest
|
||||
Environment=PUID=0
|
||||
Environment=PGID=0
|
||||
Environment=TZ=America/Los_Angeles
|
||||
Environment=UMASK_SET=022
|
||||
Volume=/Volumes/COMPLETE/radarr-data:/config
|
||||
Network=gluetun.container
|
||||
|
||||
[Service]
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,35 +0,0 @@
|
||||
# Quadlet unit: Readarr in FrankBook-Pro's podman machine.
|
||||
#
|
||||
# Install path is INSIDE the machine VM:
|
||||
# /home/core/.config/containers/systemd/readarr.container
|
||||
#
|
||||
#
|
||||
# Joins gluetun's network namespace: all egress exits through the WireGuard
|
||||
# tunnel; 8787 publishes on the gluetun unit.
|
||||
# Grab-only: no media binds; config migrated from dankserver's retired
|
||||
# bare container.
|
||||
#
|
||||
# Image pinned to amd64-1.0.980 (per project image-pin policy — same tag the
|
||||
# dankserver compose ran): `latest` historically shipped an amd64-only build
|
||||
# of a broken 1.x nightly for Readarr; the pin avoids surprises.
|
||||
[Unit]
|
||||
Description=Readarr (FrankBook-Pro)
|
||||
Wants=gluetun.service
|
||||
After=gluetun.service
|
||||
|
||||
[Container]
|
||||
ContainerName=readarr
|
||||
Image=lscr.io/linuxserver/readarr:amd64-1.0.980
|
||||
Environment=PUID=0
|
||||
Environment=PGID=0
|
||||
Environment=TZ=America/Los_Angeles
|
||||
Environment=UMASK_SET=022
|
||||
Volume=/Volumes/COMPLETE/readarr-data:/config
|
||||
Network=gluetun.container
|
||||
|
||||
[Service]
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,32 +0,0 @@
|
||||
# Quadlet unit: Sonarr in FrankBook-Pro's podman machine.
|
||||
#
|
||||
# Install path is INSIDE the machine VM:
|
||||
# /home/core/.config/containers/systemd/sonarr.container
|
||||
#
|
||||
#
|
||||
# Joins gluetun's network namespace: all egress exits through the WireGuard
|
||||
# tunnel; 8989 publishes on the gluetun unit.
|
||||
# Grab-only: no media binds (the TV library lives on dankserver's entropy
|
||||
# disk, which the Mac never mounts); config migrated from dankserver's
|
||||
# retired bare container.
|
||||
[Unit]
|
||||
Description=Sonarr (FrankBook-Pro)
|
||||
Wants=gluetun.service
|
||||
After=gluetun.service
|
||||
|
||||
[Container]
|
||||
ContainerName=sonarr
|
||||
Image=lscr.io/linuxserver/sonarr:latest
|
||||
Environment=PUID=0
|
||||
Environment=PGID=0
|
||||
Environment=TZ=America/Los_Angeles
|
||||
Environment=UMASK_SET=022
|
||||
Volume=/Volumes/COMPLETE/sonarr-data:/config
|
||||
Network=gluetun.container
|
||||
|
||||
[Service]
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,20 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# download-frank-primary-fp8.sh — fetch/verify the Qwen3.8-Flash-Next-FP8
|
||||
# payload (131 safetensors shards, 172.78 GiB total, plus aux files) for the
|
||||
# vLLM frank-primary deployment on dankserver.
|
||||
#
|
||||
# bash scripts/download-frank-primary-fp8.sh [dest_dir]
|
||||
#
|
||||
# The manifest (scripts/manifests/qwen3.8-flash-next-fp8.manifest) carries
|
||||
# per-file sizes + LFS sha256s from the Hub; hf-fetch.sh is idempotent — an
|
||||
# interrupted re-run fetches only what is missing or corrupt, and a complete
|
||||
# on-disk payload is re-verified (full sha256 pass) with zero downloads.
|
||||
# The repo is public; no HF_TOKEN required.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
dest="${1:-/home/dank/models/Qwen3.8-Flash-Next-FP8}"
|
||||
|
||||
exec bash "$REPO_ROOT/scripts/hf-fetch.sh" "$dest" \
|
||||
"$REPO_ROOT/scripts/manifests/qwen3.8-flash-next-fp8.manifest" \
|
||||
Qwen/Qwen3.8-Flash-Next-FP8
|
||||
@@ -1,144 +0,0 @@
|
||||
.gitattributes 1635
|
||||
LICENSE 3235
|
||||
README.md 65400
|
||||
chat_template.jinja 8952
|
||||
config.json 72514
|
||||
generation_config.json 202
|
||||
merges.txt 3353259
|
||||
model-00001-of-00131.safetensors 1040155912 774f0ceeadb40d165f2b3ff397d5f3840e6ca8fcb8f3d39d8acb4fea9e52c941
|
||||
model-00002-of-00131.safetensors 1678209208 6841fe21fa8a8a7a693c585efe65cd2732889095b696da88bda0cb287366910b
|
||||
model-00003-of-00131.safetensors 993901136 974a2a2ab551f8f1405a4955ab32a8721c68c73dd85b382491d9f0e6a34ee752
|
||||
model-00004-of-00131.safetensors 1678209208 591f488ab5cd5f0bd4fe28266099523761b9b9339137800734c34ffd84595538
|
||||
model-00005-of-00131.safetensors 1717267690 c50bf465a4a0129f1a1196c0e75d881048238f7107b356fc734007eac0d3b123
|
||||
model-00006-of-00131.safetensors 1600008328 57f103367fe36c9dc355c648d521d026caabb4ce5ba51038c78bd935f6593736
|
||||
model-00007-of-00131.safetensors 1600008328 cde2a12854770c706a74a5c7b85ffc140e3d4dba93dd4bee1cd7aa07cc348c90
|
||||
model-00008-of-00131.safetensors 1600008328 98ae323b5a61abdcf58a0dc9db7eb1a5a748cebc1ef1932804a1411500c09d84
|
||||
model-00009-of-00131.safetensors 1600008328 70f4b66bca5c4c0b441cf4284d4ab311670e182bfc5ec51c7294a4054baeb20f
|
||||
model-00010-of-00131.safetensors 1600008328 6d5dfa1dfa9a6c482f100f6e75c1f5e8ac02391c62e844e53f4a53996baf2206
|
||||
model-00011-of-00131.safetensors 1600008328 de085adb563e530e7ad157220cd1a219be189a1db0fcc3f2507886a2d78bd4e1
|
||||
model-00012-of-00131.safetensors 1600008328 97dac7c366ba19858d78c01364054136449ccb3cf60768587461cbf0bb582c90
|
||||
model-00013-of-00131.safetensors 1600008328 467be10b473d25e0581f8fb1bc986916c8b9301272f72e7a05762484ffc080a9
|
||||
model-00014-of-00131.safetensors 1600008328 38def6e9bbe6f242c2f92858c3e9a2b356d74d0180e323677158016dbee32ae7
|
||||
model-00015-of-00131.safetensors 1600008328 90c6c665f1f819bc6350714652c8fb77064fec57832ef51b878a0cdf9f602236
|
||||
model-00016-of-00131.safetensors 1600008328 88185c2530641b37dcad80d37b505c5257035e87ad29fa86640aae4814903e4f
|
||||
model-00017-of-00131.safetensors 1600008328 42e3b96359116d7471a74cb7de08f6a4e9e173830d562f1f74ff09d1d253ad38
|
||||
model-00018-of-00131.safetensors 1600008328 040adc7270efca6498c6f6a07dd7da14925d20217fe6de67223d6856fb4f2ad7
|
||||
model-00019-of-00131.safetensors 1600008328 b8a831992deb1a694439e37b7136ff2e181457a8683bff65f1f4b99772cabe58
|
||||
model-00020-of-00131.safetensors 1600008328 9426112fbd49ef4f5e9dd38dc2f93b046602e6d92cff732d33b96413ff1b40dc
|
||||
model-00021-of-00131.safetensors 1600008328 fce8426665f749ef38c4fad33b612f892b728c0f74fe60cab0e43ece785b4e77
|
||||
model-00022-of-00131.safetensors 1600008328 a20dd415f280774321538e6b9e78fba9f8e218a9090eae52ea7172b85b9c5a6d
|
||||
model-00023-of-00131.safetensors 1600008328 4100591839471a8721d9cedb251554dcef5ebc1dfd96ea7dce8912dd1e9921b2
|
||||
model-00024-of-00131.safetensors 1600008328 bbd6d7a4b54b6a6032e15b7c2fc9d6526377f89992d61da8837ad059f3210b45
|
||||
model-00025-of-00131.safetensors 1600008328 f9fa7b04de155dd2be74b5693d5b36e5463dabf956b9be36f67d4aae6b8dd304
|
||||
model-00026-of-00131.safetensors 1600008328 193c6b9da2c6b69c34d8a14b65234871db272761e712c2e56bac3f13c2d0cbf7
|
||||
model-00027-of-00131.safetensors 1600008328 d7efa372552c07ee8783e27c4b8ebd12fc75333f92553861bec8c7c0155a03fe
|
||||
model-00028-of-00131.safetensors 1600008328 5fc13d084d62c06b928dfb388a1b36f5c27cd25bd3cbb68a0b21e1c4cfc68e6e
|
||||
model-00029-of-00131.safetensors 1600008328 a11f5a6360b349159575871db5c553c40c37be68382b82551a2c2372f15f2811
|
||||
model-00030-of-00131.safetensors 1600008336 cac780a675f87fc699435a5207cbecb30a887271b5514b60b7e45d9bebe0e1ec
|
||||
model-00031-of-00131.safetensors 1600008336 d390a872eb3928b9d7c0a720ced792cc111613812a163d9fd6c97fd35d633897
|
||||
model-00032-of-00131.safetensors 1600008336 3b36fe4e7215a12cdc6c7499aafde51cfbce51266785184b99d3838a6b37b2c8
|
||||
model-00033-of-00131.safetensors 1600008336 3fb0504f406c0c71ee40e9e158492973c5c35337889c81b974892b407620c64c
|
||||
model-00034-of-00131.safetensors 1600008336 6d989a812a8d8d9330c8c4d21d7be98a0114705ca3bd7834d43bb268bad6be08
|
||||
model-00035-of-00131.safetensors 1600008336 8e56ba0a714198bd07a2d8fe9d91fbf5c72601c1b98edf09b289a2538445cb75
|
||||
model-00036-of-00131.safetensors 1600008336 c23fd6f6cfa9acfa961ba26e7c9c0dd7b05782c5bbf8f55e2526119b5c516fd4
|
||||
model-00037-of-00131.safetensors 942343240 b824a11bf460bd78c068e3d301e914a0bec2f243a6eea4679be7200d179248e7
|
||||
model-00038-of-00131.safetensors 1678211256 716eda81a763ffdefd1ac5306b4c80eef123c4dee24ed3a565a22438497581cb
|
||||
model-00039-of-00131.safetensors 877983696 b91e3bfbcd765ce704db85b78f69055da59456ae733df635e3973ff2f966a148
|
||||
model-00040-of-00131.safetensors 1678211256 3311ed7821b68bddb1542f06ca349feec72e68288113fe4899b08a3b8156343f
|
||||
model-00041-of-00131.safetensors 1103350296 1d79949b61730a0fa1f6cd1af149fa50fedc051566cdea48283de8c7a32d972c
|
||||
model-00042-of-00131.safetensors 1678211256 3f61684338e203cb2ccff6e451eda5705e10043d386b7ed55d4732d23eb5c77b
|
||||
model-00043-of-00131.safetensors 1000455912 bab92cfefbb811ee63c4e32595024117c7df3537f5359606eb1de23fe4f115b2
|
||||
model-00044-of-00131.safetensors 1678211256 1d05f138848edca4138c3d1c6be4a432a0817eb29fa65f12d8611ecc7685c493
|
||||
model-00045-of-00131.safetensors 993902176 4610384e28f7a91b8bf2a1527178c82cbe7d50ecd9a33b61ee878fa8e8bced85
|
||||
model-00046-of-00131.safetensors 1678211256 5ee9b0e45810300cbd4fd6f2acfe85080e7032a9ce5d77038b86f357b6700c23
|
||||
model-00047-of-00131.safetensors 878065752 efa6c0cc0e527486dc5955b00fa75665151d99997b722d8e56bd32da575e414c
|
||||
model-00048-of-00131.safetensors 1678211256 57b148a266c55a896f171b5c98b350092bf394c1fb836658c58c589a0a317ccf
|
||||
model-00049-of-00131.safetensors 1096817160 801dd3b475b43413ade810f4e8d3ace69d6ef5ee4bac5cb98d15e369343a5569
|
||||
model-00050-of-00131.safetensors 1678211256 b4d00aca34ba7a9f8427983fd0829408fb5e15b2bb7268199e4010bf0c460a04
|
||||
model-00051-of-00131.safetensors 993902176 f430125e9ac92c2de15bbb74bcd3c91bc5374eef389a1be1bc4a592f3a082f0c
|
||||
model-00052-of-00131.safetensors 1678211256 06c3d64136b097919b7e5a1715a70a60b8c2302fc4e80a33235607bb99d00f35
|
||||
model-00053-of-00131.safetensors 1000455920 147d38f786512f1a8f570a00100b46a3faab1bd649b3445ca0a50c00aa2e99a3
|
||||
model-00054-of-00131.safetensors 1678211256 31c08218039a246c333bfe0b068b8027fd92dddd542e581997af546eb104dd4d
|
||||
model-00055-of-00131.safetensors 877983696 971cd69c8d955bef6d557ebab3f170807a599517f7c69e9d8a6ba375214137f4
|
||||
model-00056-of-00131.safetensors 1678211256 a6a975ac346aa3bd1176cb816d8bbd31a4d7542aa3de9306ca5899bea44bd159
|
||||
model-00057-of-00131.safetensors 1103350288 d6acae1622abfa1e0253ae548bc1bdd4bc38cd9b096082acd455b6be592f4a54
|
||||
model-00058-of-00131.safetensors 1678209208 f02b6bd8f5cbd742e8bac3ba9537aa14a64c5a8f3fc75495227a14e1a5cec4ec
|
||||
model-00059-of-00131.safetensors 877982872 f3e2c94b8157a021b27e1efdb659333a8d1db330da5cd72ab9012fa0e5351506
|
||||
model-00060-of-00131.safetensors 1794100080 908cce7f5fe8dc34f4cca1fc8a676569ab6ce08bad536bd0091c4d75e7b65a05
|
||||
model-00061-of-00131.safetensors 1678211256 718532c88f4b7a4f402a50e4f1b6efc3cf71484c9e5a083ee30e96d5094a1bb3
|
||||
model-00062-of-00131.safetensors 1832994688 3c21b895a5a4f0dc03b669c730f8d8d9ce9a29cbc7628a25894a16ffda6f0a7a
|
||||
model-00063-of-00131.safetensors 1678211256 91d215e2135b2f5db5bc5a9541e5f6bf761205d67c6e7b8955ac73c815198f58
|
||||
model-00064-of-00131.safetensors 993902176 03a11cad2c283fb9752c6ac01c8f1d8b3127dcda963c5a8225b78f567239d872
|
||||
model-00065-of-00131.safetensors 1678211256 0f2f96331120779ce1ebd3f957882092d3c733aa9a6073ca87fc86c6eb4e9465
|
||||
model-00066-of-00131.safetensors 880605240 5dbe501e4e0cb9e3f37cc43da46673b4157fdffba09c065ca0b12d75092a79f2
|
||||
model-00067-of-00131.safetensors 1678211256 3ece325d2ca53eddf7b1782d5bbc4c376bc75c6d3da67813e110fd0af67cd09b
|
||||
model-00068-of-00131.safetensors 1096796552 0c5ef706afd6b3058af5bf8fdc1e0938cb6e29d0a526c771893c9dcb8e702463
|
||||
model-00069-of-00131.safetensors 1678211256 84999b24e3c55d2eae3ca187219a2e1a1872436ae9bafc91cc3e301180f36cfd
|
||||
model-00070-of-00131.safetensors 993907416 eac7940317ee401859caf38089b2b924274ab81a3e5fbd87c42d89923361aad8
|
||||
model-00071-of-00131.safetensors 1678211256 5ac60e474ccdab21257245630ed8240d75bc7115b3adb27ca65e39eb9ea8f20e
|
||||
model-00072-of-00131.safetensors 1000456024 b50a05c4c426fb00635d513a733a290da1007a89b8785f1ec02aad8e4d93d955
|
||||
model-00073-of-00131.safetensors 1678211256 9e0594d389debdfb26d22259725831ff34280edeef4144e683d70205050919d9
|
||||
model-00074-of-00131.safetensors 877983696 1b1a0294f8df2a431885783185cdb185106228c9567e67224fe2c3c6a2467f5a
|
||||
model-00075-of-00131.safetensors 1678211256 7dba189804ef7a8f9d926b27e1ddbe73767f1213cb7bb3cc582ab15fffec1792
|
||||
model-00076-of-00131.safetensors 1100073480 d95ebacb144b4220f2df60cce85a2c86e87cb8291b836d1bd04f9902fcbb9297
|
||||
model-00077-of-00131.safetensors 1678211256 26046977983e025defdf0f84cf5fc5f4ddfeabbd27e7abc43ff2b413265ab062
|
||||
model-00078-of-00131.safetensors 993984232 6fd46a95777392265b2afa66bf71d488856dafb7b76ff0337a0fe4ac857e961c
|
||||
model-00079-of-00131.safetensors 1678211256 77398002634ce852917c59cd642bec7dd3903381239da68912bf69743631e54e
|
||||
model-00080-of-00131.safetensors 877983688 f44aad95d36c443995b965aeef844c8e3a30ed172647dc57fcc0c5476c18d024
|
||||
model-00081-of-00131.safetensors 1678209208 b295a48088573654d1b4754e861ec329956504cb6687e9e2a8eb652b35eaf988
|
||||
model-00082-of-00131.safetensors 1096816112 83e0e171116c22eb8fdddfa3c387bebb9017eb0586410fd704ce70e7b3d23d68
|
||||
model-00083-of-00131.safetensors 1678211256 7e1021cde06c970e4b1d9eb3d02f66f29786c59a0da8a3f8879de1a4e3fbf6c0
|
||||
model-00084-of-00131.safetensors 877983696 8938fd3c7f88d61fd5b5306eb0ab253d3e42e7f64baf3c3e494d315c8ad5ab55
|
||||
model-00085-of-00131.safetensors 1678211256 cd5875a29d28d52578ae699fdd8891f605a0953cd64516bd88dc45dab50c383a
|
||||
model-00086-of-00131.safetensors 1103350296 885e570a662aaf266fd9af9562c65116998df5710b2a27d1bf6aa0c8708b3b74
|
||||
model-00087-of-00131.safetensors 1678211256 1472604826f0a0c2bbf7cd0f4a0c5b29f3012e4b3a4424d024d1ec17a3c9d0ea
|
||||
model-00088-of-00131.safetensors 1000455920 e7273f5435571e761429330ece81732e8fb3f8b5f0087bf80db693630cf702a1
|
||||
model-00089-of-00131.safetensors 1678211256 eb349562d23e330bbbd0a5f5ba90ac25ed0baa660dff230538a0fcaaf4c8b875
|
||||
model-00090-of-00131.safetensors 997179104 532915eebd4e231a240e185ba838a95a00ee14277e28fb04ad21508da5ff50aa
|
||||
model-00091-of-00131.safetensors 1678211256 68d8f46ffcd0ff46f4e73c848525cc6fd6af15eebbb9311ce9b4eb14638f5660
|
||||
model-00092-of-00131.safetensors 877983696 91abc087cd244ae62b8469576e209c8e25bc7726bc4fc020b687378059f5b5bd
|
||||
model-00093-of-00131.safetensors 1678211256 dacd0f2e5215fbf18fb7c6803f61963333e6243f404de83fcd5dfd38f43fe0d1
|
||||
model-00094-of-00131.safetensors 1096796928 cac515a778e89fb37e9f38fde6c8fdfa01ebf9d8944a2d6ff35c51ecf4bc875f
|
||||
model-00095-of-00131.safetensors 1678211256 a35975c2d21f18d58f21cb59e4eaf34a4bce11440e5112948e7fa6923658f8c4
|
||||
model-00096-of-00131.safetensors 993902176 03fd77f9d35024bba9e894c811300ef8aaf5b5ff99ebc2c89c9d9f87685adfe8
|
||||
model-00097-of-00131.safetensors 1678211256 f7b81ae09d97b8e2bb96a1916ede51da92209ec8481a31c56fb446acd691b8a8
|
||||
model-00098-of-00131.safetensors 993902176 70f208f47d99c25d5413cd32e0553cce22524cd3863c7797e2cfa37c2d526319
|
||||
model-00099-of-00131.safetensors 1678211256 de5c33dd717495657496067ea715442356fd9f7b64384d1a4c10c9f8fe6f9b7d
|
||||
model-00100-of-00131.safetensors 877984072 43752e81a5c2b376073d5d0d7b70da8d08c49bdbbfcfeb695ea947a2913e4bc8
|
||||
model-00101-of-00131.safetensors 1678211256 c2e978f7238036063914fe909214246730640be77eca215e3cf66d65724de9f1
|
||||
model-00102-of-00131.safetensors 1096797160 639efed3ea400a8dddeae25db120096898a514697dfc4cbbc0cd15f621c18421
|
||||
model-00103-of-00131.safetensors 1678209208 f8da1b61296e012dc268b4f0d84c127ef0967df488fe65b037f74c8f893decd1
|
||||
model-00104-of-00131.safetensors 993901144 adff0f2aee7b7b701e39247cfdfa135c0cab8dfa4f8c7662352c0233adb78829
|
||||
model-00105-of-00131.safetensors 1678211256 6643d71382f00f7ffe16b05879e6ff0f6e56986dee8ec079a84aae82912c6987
|
||||
model-00106-of-00131.safetensors 1025359600 680495ea0a23e73844e91670437b7b2a5ac5d46f9582923fb7fa91fe640821df
|
||||
model-00107-of-00131.safetensors 1678211256 886528d121b77433c1d66654294c13b4f61a004f1e6d1abf2d445fb0a99f7234
|
||||
model-00108-of-00131.safetensors 1062060032 a966c28a6267d43052fe87cb19e2e5730bad6154b12eb806b9ce49e007bfec96
|
||||
model-00109-of-00131.safetensors 1678211256 f6074252e492ee5b0933f78cc48f3b90694a8096befb7666cc379c3d47093756
|
||||
model-00110-of-00131.safetensors 877983696 e3e45e96bc13b52b748a0e2f397a624ad49958396468667d73ed4d725c8b5c1c
|
||||
model-00111-of-00131.safetensors 1678211256 d1d354e691eab0b77607faf7eb4adf00a75ee7f270c1cbcbad06aefc19aeac94
|
||||
model-00112-of-00131.safetensors 1096797168 e61a9a6ef0e538afca09d97a3ccbf3e7e2c647f65cc8eeebebb18621d7779b02
|
||||
model-00113-of-00131.safetensors 1678211256 a14b94db5326bfaab1d8029296b8938f6daf91ba2dc07a4950a38afafa84f776
|
||||
model-00114-of-00131.safetensors 993907400 ce4aeac52776038ec0c885915bb270dc65ee07d70d5f71827d5fba1045e2b43b
|
||||
model-00115-of-00131.safetensors 1678211256 8185d8822428d2125876a9aa05aea11c8165713bc99e04ace29921eef0e1b683
|
||||
model-00116-of-00131.safetensors 993902176 65954dccfa3be20611728311d97679e7f5d78ad519e52312524aa7cf666a48e5
|
||||
model-00117-of-00131.safetensors 1678211256 12c8027363b1a7bb782282293314fa57b07be27d17670861858c0aed1e28a3ac
|
||||
model-00118-of-00131.safetensors 878004272 2d06ec9c1726f42bfc9ce0bbb47129917d8ab373c88eed4e758fb6940c92ad4a
|
||||
model-00119-of-00131.safetensors 1678211256 36008b48c4480085bfd1a81439d70d1029cfaf06cfdd037cec19b491a40659ec
|
||||
model-00120-of-00131.safetensors 1109903856 49e4f90d92f60f6489bfe6d3e5250d8fe879c5995ae72ce67379cc7187fa4b0a
|
||||
model-00121-of-00131.safetensors 1678209208 ae8e07cceeef2f568df035a7bc0c555656acb566ba5936f61a8a019df23361ad
|
||||
model-00122-of-00131.safetensors 993901136 b3b21d33b925e31d475a509757cf7459aa4d3e73cb493706f40461387541e137
|
||||
model-00123-of-00131.safetensors 1678209208 f32fec30685afadd76f09bbb1dda680445ad765a97ff1febfc3122b7f73d7ec3
|
||||
model-00124-of-00131.safetensors 891089968 e857f36ddd0db135f858c2c87c9fcf39c758965c626b280e47cb0c1e6c1423f8
|
||||
model-00125-of-00131.safetensors 1678209208 b9de3774218d3cd7946cdbf913dedfc82591c18212d9c3fc9b9484e7dc68e9fe
|
||||
model-00126-of-00131.safetensors 1096795496 29b39319522a00b1ef4ac909c4fb306059999f0a48fd07ca73cea687647631ee
|
||||
model-00127-of-00131.safetensors 1678209208 bf48ccb3b4ba75c288d9db2b94817c6a3993e9f84a5b9410d9f24edcec12e177
|
||||
model-00128-of-00131.safetensors 993901136 7feaa51c906a1a735e5fb52fd5499b59c473d442d77b44cbd610c9b6a3a9e0ba
|
||||
model-00129-of-00131.safetensors 1678209208 a5374c8d86aea48f21a4be2ad6cda37fbfbb01eff333d1179939a2e55d8db816
|
||||
model-00130-of-00131.safetensors 2136177320 204b122d02425f61115db9ab896648d95f6dadd6a716c65daf6aec297392ed56
|
||||
model-00131-of-00131.safetensors 1271398496 ba6fc69ebb6a9da1938c2e041b8ef625f4550b641812da75845bc7dd7c361d80
|
||||
model.safetensors.index.json 17410140 0419e2c2dfbb925257d7409405433a793cf7ff7d96f3eba882a815ec6d9fe7a6
|
||||
preprocessor_config.json 390
|
||||
tokenizer.json 12809320 0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3
|
||||
tokenizer_config.json 17928
|
||||
video_preprocessor_config.json 385
|
||||
vocab.json 6722759
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user