diff --git a/.AGENTTRASH/frank-default-small.env.llama.cpp b/.AGENTTRASH/frank-default-small.env.llama.cpp deleted file mode 100644 index d11f10c..0000000 --- a/.AGENTTRASH/frank-default-small.env.llama.cpp +++ /dev/null @@ -1,71 +0,0 @@ -# Tunables for frank-default-small (Tiel-Coder-35B-A3B UD-Q6_K_XL, -# qwen35moe hybrid MoE, Dankserver, RTX 3090). -# -# Runtime: upstream llama.cpp (latest master, CUDA build with -# -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86) on RTX 3090 -# 24 GB; Ryzen 7 7700X; 62 GB RAM. Replaced the ik_llama.cpp Ornith -# CRACK Q8_0 deployment on 2026-09-01. -# -# Model: peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF, UD-Q6_K_XL quant -# (30 GB). Architecture: qwen35moe — 40 layers, full_attention_interval=4 -# (10 attention + 30 SSM/Mamba), 256 experts / 8 used, 2 KV heads × 256 -# dim, embedding 2048, rope dimension_count 64, rope freq_base 10 000 000. -# Text-only (no mmproj). Built-in chat template in the GGUF. -# -# Expert offload: --cpu-moe keeps all 256 expert weights in CPU RAM. -# Non-expert weights (~1.6 GB) fit entirely on the 24 GB card. -# -# KV cache: Q8_0 for both K and V. With CTX_SIZE=200700 (66 900 per slot -# × 3 parallel) and 10 attention layers, the KV cache is ~2 GB — the -# requested 2 GB VRAM buffer. -# -# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The -# ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP -# incident). This box is NVIDIA/CUDA, but the rule keeps the env portable. - -# Model file (UD-Q6_K_XL, 30 GB, text-only MoE hybrid). -MODEL_DIR=/home/dank/models/tiel-coder-35b -MODEL_FILE=Tiel-Coder-35B-A3B-UD-Q6_K_XL.gguf - -# Upstream llama.cpp llama-server (CUDA build, latest master). -# Built with -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86 -# using the staged CUDA 13.2 toolkit at /home/dank/cuda13-stage. -LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server - -ALIAS=frank-default-small -BIND_ADDR=0.0.0.0 -LISTEN_PORT=7770 - -# Total context budget across all parallel slots. 200700 / 3 = 66 900 -# per slot. The model's context_length is 262 144, so 66 900 is well -# within range. -CTX_SIZE=200700 -PARALLEL=3 - -# Expert offload: all MoE expert weights pinned to CPU RAM. -CPU_MOE=1 - -# All non-expert layers on GPU (auto-fit will determine the exact count; -# with --cpu-moe the non-expert footprint is ~1.6 GB, leaving ample room). -N_GPU_LAYERS=999 - -# KV cache: Q8_0 for both K and V. ~2 GB total for 200700 ctx × 3 slots -# × 10 attention layers × (2 KV heads × 256 dim × 1 byte) × 2 (K+V). -CACHE_TYPE_K=q8_0 -CACHE_TYPE_V=q8_0 - -VERBOSITY=2 - -# Prefill batch sizing. -BATCH_SIZE=4096 -UBATCH_SIZE=1024 - -# Sampling. -TEMPERATURE=1.0 -TOP_K=20 -TOP_P=0.95 -MIN_P=0.0 - -# Compute threads lowered so other processes keep a core. -THREADS=7 -THREADS_HTTP=8 \ No newline at end of file diff --git a/.AGENTTRASH/frank-default-small.service.llama.cpp b/.AGENTTRASH/frank-default-small.service.llama.cpp deleted file mode 100644 index 0805e2c..0000000 --- a/.AGENTTRASH/frank-default-small.service.llama.cpp +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=llama.cpp Tiel-Coder-35B-A3B Q6_K_XL qwen35moe hybrid MoE (dankserver, frank-default-small) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=dank -Group=dank -Environment="FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-default-small.env" -ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/.AGENTTRASH/frank-dragoon.env.llama.cpp b/.AGENTTRASH/frank-dragoon.env.llama.cpp deleted file mode 100644 index ff82436..0000000 --- a/.AGENTTRASH/frank-dragoon.env.llama.cpp +++ /dev/null @@ -1,126 +0,0 @@ -# Tunables for frank-dragoon (kingjones777/Qwen3.8-Flash-Next-ROCmFP4- -# STRIX-GGUF on the kingjones30/ROCmFPX llama.cpp runtime, Dankclaw, -# Strix Halo gfx1151, unified memory). -# -# Runtime: kingjones30/ROCmFPX (fork of charlie12345/ROCmFPX with the -# qwen4exp-on-rocmfpx-d3ca537 patch applied — the model card's pinned -# runtime: ROCmFP4 tensor types AND the qwen4exp architecture in one -# tree; stock llama.cpp and charlie's fork alone cannot load this GGUF). -# Built per the card recipe: -# cmake -B build -DGGML_HIP=ON -DGPU_TARGETS=gfx1151 -DGGML_NATIVE=ON -DCMAKE_BUILD_TYPE=Release -# at /home/dank/GitPackages/kingjones-ROCmFPX/build. -# Replaced the agentionai ROCmFP4-FAST imatrix deployment (charlie's -# build-qwen38 runtime) on 2026-08-30. -# -# Model: Qwen3.8-Flash-Next base (qwen4exp MoE: 512 experts / 10 used, -# 48 layers = 12 attention + 36 SSM, QSA indexer, hyper-connections, PLE -# at layer 1), Q4_0_ROCMFP4_STRIX (file_type 105), 113.47 GiB single -# file. STRIX is the top quality tier: PLE table at Q8_0 (50.66 GiB), -# token embeddings + lm head at Q6_K. The PLE table NEVER enters GPU -# memory — it streams from the file through the OS page cache. Two hard -# card constraints: -# 1. mmap MUST stay enabled (default) — --no-mmap forces the 50.66 GiB -# table into anonymous memory and the cgroup killer takes the -# process with nothing in the server log (only dmesg). -# 2. Do NOT pin the PLE tensor to CPU with -ot — measured gen fell -# ~23 -> 13.4 tok/s; the kernel streams it better. -# NOTE: the card's own build_attn_qsa still calls ggml_fill on the -# F16-cast kq mask (the path that aborted our charlie build with -# --flash-attn on before scripts/patches/qwen4exp-qsa-f32-mask.patch). -# If a clean kingjones-tree build aborts graph build with -# GGML_ASSERT(a-type == GGML_TYPE_F32), port that fix (F32 base mask + -# cast at the attention call) into this tree and rebuild. -# -# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP -# incident). llama.cpp reads --host/--port/--temp as CLI flags, never as -# env vars. - -# kingjones30/ROCmFPX runtime binary (card recipe build). -LLAMA_BIN=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server - -# Model package from -# https://huggingface.co/kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX-GGUF -# Layout: single unsharded GGUF, text-only (the repo ships no mmproj; -# the vision tower below is borrowed from the agentionai FAST package — -# same base model, f16 — kept only if it loads clean, drop -# MMPROJ_PATH if it does not). -MODEL_DIR=/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX -MODEL_FILE=Qwen3.8-Flash-Next-Q4_0-ROCmFP4-STRIX.gguf -MMPROJ_PATH=/home/dank/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST/mmproj/mmproj-Qwen3.8-Flash-Next-f16.gguf - -# Qwen-Sharp chat template (tracked in the repo, staged on the host). The -# Sharp template drives thinking on by default (medium effort); the server -# emits reasoning in DeepSeek format so clients receive reasoning_content -# separately instead of literal thinking tags in content. -CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja -CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true,"preserve_reasoning":true}' - -# Alias + network. -ALIAS=frank-dragoon -BIND_ADDR=0.0.0.0 -LISTEN_PORT=7770 - -# Serving: 401408-token context, TWO slots (parallel 2), batch 4096 / -# ubatch 512, 16+16 threads, f16 KV, 16 context checkpoints, 8 GiB -# prompt cache. --fit off: auto-fit segfaults on this model's -# hyper-connection/indexer config; -ngl 999 puts every layer on the -# SVM-backed GPU pool. ubatch 512 matches the card's re-measurement -# (-ub 1024/2048/4096 showed no effect; -t 32 vs 16 is noise). -# -# CTX_SIZE=401408 / PARALLEL=2: carried over from the FAST deployment -# where it loaded clean (~74 s, health 200). The card measures the full -# 262144 window at ~71.7 GiB GTT at load on the same 128 GB board; QSA -# caps KV growth so the extra window is cheap. KV footprint scales -# with the 12 attention layers only. Verify load on first STRIX boot. -CTX_SIZE=401408 -PARALLEL=2 - -BATCH_SIZE=4096 -UBATCH_SIZE=512 -NUM_THREADS=16 -THREAD_BATCH=16 -CACHE_TYPE_K=f16 -CACHE_TYPE_V=f16 -CTX_CHECKPOINTS=16 -CACHE_RAM=8192 -SLOT_SAVE_PATH=/home/dank/serverAppConfigs/scripts/slot-state -SPLIT_MODE=none -FIT=off -NGL=999 -FLASH_ATTN=on - -# Sampling (card instruct table, non-thinking). -TEMPERATURE=0.7 -TOP_K=20 -TOP_P=0.8 -MIN_P=0.0 -PRESENCE_PENALTY=1.5 - -# Reasoning on (Sharp template drives it; preserve_thinking + preserve_reasoning -# keep the reasoning trail in the conversation). DeepSeek format projects the -# reasoning into the API's reasoning_content field instead of literal thinking -# tags. -REASONING=on -REASONING_FORMAT=deepseek - -# Strix Halo unified memory + native gfx1151. -HSA_OVERRIDE_GFX_VERSION=11.5.1 - -VERBOSITY=4 - - -# Speculative decoding: disabled. The card's Usage block ships no spec -# flags. The Flash-Next MTP heads do not load on this loader (requires -# per-layer blk.N.hc_attn_norm; every head carries only blk.48 — -# verified 3x on build-qwen38). DRAFT_MODEL_* point at the retired -# agentionai FAST head and are unused while SPEC_TYPE=none. -DRAFT_MODEL_DIR=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST -DRAFT_MODEL_FILE=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf -SPEC_TYPE=none -SPEC_DRAFT_N_MAX=3 -SPEC_DRAFT_N_MIN=0 -SPEC_DRAFT_P_MIN=0.75 -SPEC_DRAFT_NGL=999 -SPEC_DRAFT_DEVICE=ROCm0 -SPEC_DRAFT_TYPE_K=f16 -SPEC_DRAFT_TYPE_V=f16 \ No newline at end of file diff --git a/.AGENTTRASH/frank-dragoon.service.llama.cpp b/.AGENTTRASH/frank-dragoon.service.llama.cpp deleted file mode 100644 index 6e166e1..0000000 --- a/.AGENTTRASH/frank-dragoon.service.llama.cpp +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=ROCmFPX (kingjones30 fork) llama.cpp Qwen3.8-Flash-Next ROCmFP4 STRIX 177B MoE (dankclaw, frank-dragoon) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=dank -Group=dank -Environment="FRANK_DRAGOON_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-dragoon.env" -ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-dragoon.sh -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/.AGENTTRASH/frank-npu.env.flm-npu b/.AGENTTRASH/frank-npu.env.flm-npu deleted file mode 100644 index adb9c08..0000000 --- a/.AGENTTRASH/frank-npu.env.flm-npu +++ /dev/null @@ -1,23 +0,0 @@ -# Tunables for frank-npu (FastFlowLM NPU endpoint, Dankclaw) -# Runtime: FLM 1.0.1 (/usr/bin/flm) on the XDNA2 NPU (50 TOPS). Boot tag is -# nuextract3:4b (NuExtract3-4B-NPU2, Qwen3.5-4B skeleton, structured-extraction -# model); serve routes every request by the exact FLM registry tag in the -# request's model field, so any registered tag (gemma4-it:e4b, qwythos:9b, ...) -# is reachable on this port regardless of the boot tag. The NPU is a -# single-lane device: generations serialize FIFO, and the first request after a -# model swap pays the ~6-11 s load cost. The NPU draws from the unified RAM -# pool shared with frank-dragoon (iGPU). -FLM_MODEL=nuextract3:4b -FLM_HOST=0.0.0.0 -FLM_PORT=7771 -# turbo = max NPU power mode (xrt-smi configure --pmode turbo at serve start). -# No other tenant uses the NPU on this host, so the top mode costs nothing to -# neighbours; drop to `performance` if iGPU power-budget contention ever shows. -FLM_PMODE=turbo -# FLM_ASR=1 additionally loads Whisper-V3-Turbo-NPU2 (+0.62 GB) and exposes -# /v1/audio/transcriptions. NuExtract3 answers text-only; audio stays off. -FLM_ASR=0 -# -1 = no --ctx-len flag: every pack loads with its own trained context -# (nuextract3:4b -> 32768, gemma4-it:e4b -> 131072). A global override would -# over-allocate KV for the small packs. -FLM_CTX_LEN=-1 diff --git a/.AGENTTRASH/frank-npu.service.flm-npu b/.AGENTTRASH/frank-npu.service.flm-npu deleted file mode 100644 index 42400c0..0000000 --- a/.AGENTTRASH/frank-npu.service.flm-npu +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=FastFlowLM NPU sight+sound inference (frank-npu) on Dankclaw -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=dank -Group=dank -Environment="FRANK_NPU_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-npu.env" -ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-npu.sh -LimitMEMLOCK=infinity -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/.AGENTTRASH/serve-frank-dragoon.sh.llama.cpp b/.AGENTTRASH/serve-frank-dragoon.sh.llama.cpp deleted file mode 100755 index 0fcaa24..0000000 --- a/.AGENTTRASH/serve-frank-dragoon.sh.llama.cpp +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env bash -# Serve kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX-GGUF as -# frank-dragoon via the kingjones30/ROCmFPX llama.cpp runtime on -# Dankclaw (Strix Halo gfx1151, unified memory). -# -# Runtime: kingjones30/ROCmFPX (fork of charlie12345/ROCmFPX with the -# model card's qwen4exp-on-rocmfpx patch) built at -# /home/dank/GitPackages/kingjones-ROCmFPX/build (HIP, gfx1151, -# ROCm 7.2.4, card recipe). The ROCmFP4 quant types plus the qwen4exp -# architecture must be in one tree; stock llama.cpp cannot load this -# GGUF. -# -# Model: Qwen3.8-Flash-Next base (qwen4exp MoE: 512 experts / 10 used, -# 48 layers = 12 attention + 36 SSM, QSA indexer, hyper-connections, PLE -# at layer 1) quantized Q4_0_ROCMFP4_STRIX, 113.47 GiB single file. The -# 50.66 GiB PLE n-gram table stays OUT of GPU memory and streams via -# mmap from the OS page cache — never add --no-mmap or -ot for it. -# -# Profile: card sampling table (temp 0.7, top_p 0.80, top_k 20, min_p 0.0, -# presence 1.5 — non-thinking instruct) mixed with the validated serving -# profile: ctx 401408 x 2 slots, b 4096 / ub 512, t 16 / tb 16, f16 KV, -# fa on (if the graph build aborts on the QSA F16 mask, see the note in -# the env file: port scripts/patches/qwen4exp-qsa-f32-mask.patch into -# this tree and rebuild), cont-batching, prompt cache + 8 GiB cache-ram -# + idle-slot cache, 16 ctx checkpoints, --fit off (auto-fit segfaults -# on qwen4exp hyper-connection config). Speculative decoding off: the -# Flash-Next MTP heads do not load on this loader; the card ships no -# spec flags. -set -euo pipefail - -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}" -if [[ -f "$ENV_FILE" ]]; then - set -a - # shellcheck disable=SC1090 - source "$ENV_FILE" - set +a -fi - -# --- defaults (used when the env file does not set them) ------------------- -: "${LLAMA_BIN:=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server}" -: "${MODEL_DIR:=/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX}" -: "${MODEL_FILE:=Qwen3.8-Flash-Next-Q4_0-ROCmFP4-STRIX.gguf}" -: "${MMPROJ_PATH:=/home/dank/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST/mmproj/mmproj-Qwen3.8-Flash-Next-f16.gguf}" -: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}" -: "${CHAT_TEMPLATE_KWARGS:={}}" -: "${ALIAS:=frank-dragoon}" -: "${BIND_ADDR:=0.0.0.0}" -: "${LISTEN_PORT:=7770}" -: "${CTX_SIZE:=401408}" -: "${PARALLEL:=2}" -: "${BATCH_SIZE:=4096}" -: "${UBATCH_SIZE:=512}" -: "${NUM_THREADS:=16}" -: "${THREAD_BATCH:=16}" -: "${CACHE_TYPE_K:=f16}" -: "${CACHE_TYPE_V:=f16}" -: "${CTX_CHECKPOINTS:=16}" -: "${CACHE_RAM:=8192}" -: "${SLOT_SAVE_PATH:=$SCRIPT_DIR/slot-state}" -: "${SPLIT_MODE:=none}" -: "${FIT:=off}" -: "${NGL:=999}" -: "${FLASH_ATTN:=on}" -: "${TEMPERATURE:=0.7}" -: "${TOP_K:=20}" -: "${TOP_P:=0.8}" -: "${MIN_P:=0.0}" -: "${PRESENCE_PENALTY:=1.5}" -: "${REASONING:=on}" -: "${REASONING_FORMAT:=deepseek}" -: "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}" -: "${SPEC_TYPE:=none}" -: "${DRAFT_MODEL_DIR:=/home/dank/models/mtp-Qwen3.8-Flash-Next-FAST}" -: "${DRAFT_MODEL_FILE:=Qwen3.8-Flash-Next-MTP-ROCmFP4-FAST.gguf}" -: "${SPEC_DRAFT_N_MAX:=3}" -: "${SPEC_DRAFT_N_MIN:=0}" -: "${SPEC_DRAFT_P_MIN:=0.75}" -: "${SPEC_DRAFT_NGL:=999}" -: "${SPEC_DRAFT_DEVICE:=ROCm0}" -: "${SPEC_DRAFT_TYPE_K:=f16}" -: "${SPEC_DRAFT_TYPE_V:=f16}" - -MODEL="$MODEL_DIR/$MODEL_FILE" -for f in "$LLAMA_BIN" "$MODEL" "$CHAT_TEMPLATE_FILE"; do - if [[ ! -r "$f" ]]; then - echo "missing or unreadable required artifact: $f" >&2 - exit 1 - fi -done -if [[ ! -x "$LLAMA_BIN" ]]; then - echo "missing executable runtime: $LLAMA_BIN" >&2 - exit 1 -fi - -# --- speculative decoding (MTP draft head) --------------------------------- -# The MTP draft head is a separate GGUF sidecar loaded via --spec-draft-model -# (alias --model-draft). ngram-only specs do not need a draft file. -# Set SPEC_TYPE=none in the env to disable speculative decoding entirely. -SPEC_ARGS=() -if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then - SPEC_ARGS=( - --spec-type "$SPEC_TYPE" - --spec-draft-n-max "$SPEC_DRAFT_N_MAX" - --spec-draft-n-min "$SPEC_DRAFT_N_MIN" - --spec-draft-p-min "$SPEC_DRAFT_P_MIN" - ) - if [[ "$SPEC_TYPE" == *draft-mtp* ]]; then - DRAFT_MODEL="$DRAFT_MODEL_DIR/$DRAFT_MODEL_FILE" - if [[ ! -r "$DRAFT_MODEL" ]]; then - echo "missing or unreadable draft model: $DRAFT_MODEL" >&2 - exit 1 - fi - SPEC_ARGS+=( - --spec-draft-model "$DRAFT_MODEL" - --spec-draft-ngl "$SPEC_DRAFT_NGL" - --spec-draft-device "$SPEC_DRAFT_DEVICE" - --spec-draft-type-k "$SPEC_DRAFT_TYPE_K" - --spec-draft-type-v "$SPEC_DRAFT_TYPE_V" - ) - fi - if [[ -n "${SPEC_DRAFT_P_SPLIT:-}" ]]; then - SPEC_ARGS+=(--spec-draft-p-split "$SPEC_DRAFT_P_SPLIT") - fi -fi - -# The STRIX repo ships no mmproj; this is the agentionai FAST package's -# f16 vision tower for the same base model. Passed only when readable. -MMPROJ_ARGS=() -if [[ -n "${MMPROJ_PATH:-}" && -r "${MMPROJ_PATH:-}" ]]; then - MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH") -fi - -SERVER_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -mkdir -p "$SLOT_SAVE_PATH" -ROCM_PATH="${ROCM_PATH:-/opt/rocm}" -LIBRARY_PATH="$SERVER_DIR:$ROCM_PATH/lib:$ROCM_PATH/lib/rocm_sysdeps/lib:$ROCM_PATH/lib/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - -exec /usr/bin/env \ - ROCM_PATH="$ROCM_PATH" \ - HIP_VISIBLE_DEVICES=0 \ - LD_LIBRARY_PATH="$LIBRARY_PATH" \ - HSA_OVERRIDE_GFX_VERSION="$HSA_OVERRIDE_GFX_VERSION" \ - "$LLAMA_BIN" \ - --model "$MODEL" \ - --alias "$ALIAS" \ - --chat-template-file "$CHAT_TEMPLATE_FILE" \ - --chat-template-kwargs "$CHAT_TEMPLATE_KWARGS" \ - --host "$BIND_ADDR" \ - --port "$LISTEN_PORT" \ - --jinja \ - --device ROCm0 \ - --gpu-layers "$NGL" \ - --split-mode "$SPLIT_MODE" \ - --parallel "$PARALLEL" \ - --ctx-size "$CTX_SIZE" \ - --batch-size "$BATCH_SIZE" \ - --ubatch-size "$UBATCH_SIZE" \ - --fit "$FIT" \ - --flash-attn "$FLASH_ATTN" \ - --cache-type-k "$CACHE_TYPE_K" \ - --cache-type-v "$CACHE_TYPE_V" \ - --threads "$NUM_THREADS" \ - --threads-batch "$THREAD_BATCH" \ - --ctx-checkpoints "$CTX_CHECKPOINTS" \ - --cache-ram "$CACHE_RAM" \ - --slot-save-path "$SLOT_SAVE_PATH" \ - --cache-prompt \ - --cache-idle-slots \ - --metrics \ - --temp "$TEMPERATURE" \ - --top-p "$TOP_P" \ - --top-k "$TOP_K" \ - --min-p "$MIN_P" \ - --presence-penalty "$PRESENCE_PENALTY" \ - --reasoning "$REASONING" \ - --reasoning-format "$REASONING_FORMAT" \ - --reasoning-budget -1 \ - "${MMPROJ_ARGS[@]}" \ - "${SPEC_ARGS[@]}" \ No newline at end of file diff --git a/.AGENTTRASH/serve-frank-npu.sh.flm-npu b/.AGENTTRASH/serve-frank-npu.sh.flm-npu deleted file mode 100755 index 61a60d1..0000000 --- a/.AGENTTRASH/serve-frank-npu.sh.flm-npu +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -# Serve the FastFlowLM (FLM) sight+sound endpoint — frank-npu. -# -# Runs `flm serve ` on the XDNA2 NPU (50 TOPS, Dankclaw). The NPU is a -# single-lane device: generations serialize in FIFO order, so this is a -# background/triage endpoint, not an interactive multi-user chat. The NPU -# draws from the unified RAM pool shared with frank-dragoon (the iGPU -# llama.cpp service); see serverAppConfigs-docs/LLM/frank-npu.md -# for the memory budget and the model-selection tradeoff. -# -# NOTE: FLM requires unlimited memlock. The unit sets LimitMEMLOCK=infinity. -# NOTE: an unknown model tag does NOT 404 — /v1/chat/completions answers HTTP 200 with a -# flat {"error":"Failed to load model!"} body and no choices; /v1/completions returns a -# nested OpenAI-shaped server_error (code 500) also under HTTP 200. Client model IDs must be the exact FLM registry -# tag (e.g. gemma4-it:e4b), never a service alias. Custom tags only resolve -# when FLM_CONFIG_PATH points at the user registry that carries them and their -# kernels are symlinked into /home/dank/.config/flm/xclbins/. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ENV_FILE="${FRANK_NPU_ENV_FILE:-$SCRIPT_DIR/frank-npu.env}" -if [[ -f "$ENV_FILE" ]]; then - set -a - # shellcheck disable=SC1090 - source "$ENV_FILE" - set +a -fi - -# --- defaults (used when the env file does not set them) ------------------- -: "${FLM_BIN:=/usr/bin/flm}" -: "${FLM_MODEL:=gemma4-it:e4b}" -: "${FLM_HOST:=0.0.0.0}" -: "${FLM_PORT:=7771}" -: "${FLM_PMODE:=turbo}" -: "${FLM_ASR:=0}" -: "${FLM_CTX_LEN:=131072}" - -# The user registry is a superset of the system registry (it carries the -# custom tags too), and its model_path resolves relative to the registry file, -# so exports keep a tag swap in the env file from needing script edits. -export FLM_CONFIG_PATH="${FLM_CONFIG_PATH:-/home/dank/.config/flm/model_list.json}" -export FLM_XCLBIN_PATH="${FLM_XCLBIN_PATH:-/home/dank/.config/flm/xclbins}" - -if [[ ! -x "$FLM_BIN" ]]; then - echo "missing flm binary: $FLM_BIN" >&2 - exit 1 -fi - -# --- optional flags ---------------------------------------------------------- -OPT_ARGS=() -if [[ "${FLM_ASR}" != "0" ]]; then - OPT_ARGS+=(--asr "${FLM_ASR}") -fi -if [[ "${FLM_CTX_LEN}" != "-1" ]]; then - OPT_ARGS+=(--ctx-len "${FLM_CTX_LEN}") -fi - -exec "$FLM_BIN" serve "$FLM_MODEL" \ - --host "$FLM_HOST" \ - -p "$FLM_PORT" \ - --pmode "$FLM_PMODE" \ - "${OPT_ARGS[@]}" diff --git a/.AGENTTRASH/serve-frank-small.sh.llama.cpp b/.AGENTTRASH/serve-frank-small.sh.llama.cpp deleted file mode 100755 index 5b3b6e8..0000000 --- a/.AGENTTRASH/serve-frank-small.sh.llama.cpp +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env bash -# Serve peculiar-ragdoll/Tiel-Coder-35B-A3B (UD-Q6_K_XL) as -# frank-default-small via stock llama.cpp on Dankserver (RTX 3090). -# -# Runtime: upstream llama.cpp (latest master, CUDA build with -# -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86). -# The model is a 35B-total / 8-active MoE hybrid (qwen35moe arch) with -# SSM/Mamba layers (full_attention_interval=4: 10 attention + 30 SSM -# layers out of 40). 256 experts, 8 used per token. -# -# Expert offload: --cpu-moe keeps all 256 expert weights in CPU RAM. -# Non-expert weights (attention, SSM, shared expert, embeddings, output) -# are ~1.6 GB and fit entirely on the 24 GB RTX 3090. -# -# KV cache: Q8_0 for both K and V. With CTX_SIZE=200700 (66 900 per -# slot × 3 parallel) and 10 attention layers (2 KV heads × 256 dim), -# the KV cache is ~2 GB — the requested 2 GB VRAM buffer. -# -# Model is loaded with mmap (default). The 30 GB Q6_K_XL file is -# memory-mapped; only accessed pages enter RAM. -# -# Tunables come from frank-default-small.env unless FRANK_SMALL_ENV_FILE -# overrides that path. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-default-small.env}" -if [[ -f "$ENV_FILE" ]]; then - set -a - # shellcheck disable=SC1090 - source "$ENV_FILE" - set +a -fi - -# --- defaults (used when the env file does not set them) ----------------- -: "${MODEL_DIR:=/home/dank/models/tiel-coder-35b}" -: "${MODEL_FILE:=Tiel-Coder-35B-A3B-UD-Q6_K_XL.gguf}" -: "${LLAMA_SERVER:=/home/dank/llama.cpp/build/bin/llama-server}" -: "${ALIAS:=frank-default-small}" -: "${BIND_ADDR:=0.0.0.0}" -: "${LISTEN_PORT:=7770}" -: "${CTX_SIZE:=200700}" -: "${PARALLEL:=3}" -: "${CACHE_TYPE_K:=q8_0}" -: "${CACHE_TYPE_V:=q8_0}" -: "${N_GPU_LAYERS:=999}" -: "${BATCH_SIZE:=4096}" -: "${UBATCH_SIZE:=1024}" -: "${VERBOSITY:=2}" -: "${CPU_MOE:=1}" -: "${TEMPERATURE:=1.0}" -: "${TOP_K:=20}" -: "${TOP_P:=0.95}" -: "${MIN_P:=0.0}" -: "${THREADS:=7}" -: "${THREADS_HTTP:=8}" - -MODEL="$MODEL_DIR/$MODEL_FILE" - -if [[ ! -f "$MODEL" ]]; then - echo "missing required file: $MODEL" >&2 - exit 1 -fi -if [[ ! -x "$LLAMA_SERVER" ]]; then - echo "missing llama-server binary: $LLAMA_SERVER" >&2 - exit 1 -fi - -exec "$LLAMA_SERVER" \ - --model "$MODEL" \ - $([ "$CPU_MOE" = "1" ] && printf -- "--cpu-moe") \ - --cache-type-k "$CACHE_TYPE_K" \ - --cache-type-v "$CACHE_TYPE_V" \ - --ctx-size "$CTX_SIZE" \ - --parallel "$PARALLEL" \ - --gpu-layers "$N_GPU_LAYERS" \ - --batch-size "$BATCH_SIZE" \ - --ubatch-size "$UBATCH_SIZE" \ - --temp "$TEMPERATURE" \ - --top-k "$TOP_K" \ - --top-p "$TOP_P" \ - --min-p "$MIN_P" \ - --threads "$THREADS" \ - --threads-http "$THREADS_HTTP" \ - --alias "$ALIAS" \ - --host "$BIND_ADDR" \ - --port "$LISTEN_PORT" \ No newline at end of file diff --git a/dankserver/.AGENTTRASH/20260916-vllm-radiance/affinity-proxy.py b/dankserver/.AGENTTRASH/20260916-vllm-radiance/affinity-proxy.py deleted file mode 100755 index e526e8c..0000000 --- a/dankserver/.AGENTTRASH/20260916-vllm-radiance/affinity-proxy.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python3 -"""HTTP front door for the frank-medium single-GPU replicas. - -Why two replicas instead of TP=2: one vLLM process spanning both R9700s needs -RCCL collectives, and RCCL's kernels require PCIe atomics on every -participating GPU. The second card sits behind chipset bridges advertising -`AtomicOpsCap: Routing-`, which no software can enable (see the runbook), so -no collective can launch on it. Two single-GPU replicas use both cards; this -service keeps the fleet contract of one endpoint on :7770, streaming included. - -Routing decisions live in routing.py — spread single-turn requests, pin -continuations to the replica whose prefix cache holds their context. This file -only moves bytes: parse enough of the request to route it, forward it verbatim, -stream the answer back, and report replica health on /health. -""" - -from __future__ import annotations - -import asyncio -import os -from contextlib import suppress -from aiohttp import ClientSession, ClientTimeout, web - -import routing - -PORT = int(os.environ.get("PORT", "7770")) -HEALTH_TTL = float(os.environ.get("HEALTH_TTL", "10")) -# Hop-by-hop headers must not be forwarded (RFC 9110 §7.6.1) — forwarding -# transfer-encoding or content-length of a rewritten body breaks clients. -HOP_BY_HOP = { - "host", "content-length", "transfer-encoding", "connection", - "keep-alive", "proxy-authenticate", "proxy-authorization", "te", - "trailers", "upgrade", -} - -UPSTREAMS = routing.UPSTREAMS -# Replica health, refreshed by the watcher. Module state rather than aiohttp app -# state: the application is frozen once the site is up. -_UP: list[bool] = [True] * len(UPSTREAMS) - - -async def health(session: ClientSession) -> web.Response: - results = await asyncio.gather( - *(session.get(u + "/health", timeout=ClientTimeout(total=5)) for u in UPSTREAMS), - return_exceptions=True, - ) - lines, up = [], 0 - for url, res in zip(UPSTREAMS, results): - if isinstance(res, Exception) or res.status != 200: - lines.append(f"{url} DOWN") - else: - up += 1 - lines.append(f"{url} UP") - return web.Response(status=200 if up else 503, text="\n".join(lines) + "\n") - - -async def watch_health() -> None: - """Refresh the replica up/down table so routing never targets a dead one. - - One HTTP request per replica per interval, in exchange for never handing a - client a 502 after a restart. - """ - global _UP - session = ClientSession() - try: - while True: - res = await asyncio.gather( - *(session.get(u + "/health", timeout=ClientTimeout(total=5)) for u in UPSTREAMS), - return_exceptions=True, - ) - _UP = [not isinstance(r, Exception) and r.status == 200 for r in res] - await asyncio.sleep(HEALTH_TTL) - finally: - await session.close() - - -async def proxy(request: web.Request) -> web.StreamResponse: - session: ClientSession = request.app["session"] - if request.path == "/health": - return await health(session) - - body = await request.read() - headers = {k: v for k, v in request.headers.items() if k.lower() not in HOP_BY_HOP} - - # Fail over before anything is written to the client. A replica that - # restarts is invisible to the health watcher for up to HEALTH_TTL, and a - # deploy restarts both replicas — without this retry a 10 s window of - # requests would 502 on a service that has a live card the whole time. - # vLLM's OpenAI endpoints are stateless per request, so re-sending a - # request that never reached an engine is safe; a request that failed - # mid-stream is not retried (that path is below). - tried: set[int] = set() - upstream = None - last_error = "no healthy frank-medium replica" - while len(tried) < len(UPSTREAMS): - idx = routing.route(body, request.remote, _UP) - if idx < 0 or idx in tried: - break - tried.add(idx) - target = UPSTREAMS[idx] + request.path_qs - try: - upstream = await session.request( - request.method, target, headers=headers, data=body or None, - timeout=ClientTimeout(total=None, sock_connect=10, sock_read=3600), - ) - break - except Exception as exc: - # Demote at once: the watcher will confirm, but every request until - # then should skip this replica rather than pay the same timeout. - # Logged because a silent failover hides a card that is down. - _UP[idx] = False - last_error = f"{target} unreachable: {exc}" - print(f"proxy: failover — {target} unreachable ({exc}); retrying " - f"another replica", flush=True) - if upstream is None: - # 503 when no replica was even reachable through the routing table, - # 502 when replicas were tried and all of them failed. - status = 502 if tried else 503 - return web.Response(status=status, text=f"proxy: {last_error}\n") - - out = web.StreamResponse(status=upstream.status, reason=upstream.reason) - for k, v in upstream.headers.items(): - if k.lower() not in HOP_BY_HOP: - out.headers[k] = v - await out.prepare(request) - try: - async for chunk in upstream.content.iter_any(): - await out.write(chunk) - except Exception: - # The client went away mid-stream (an aborted generation). Nothing to - # report: the engine sees its own cancellation. - pass - finally: - upstream.release() - with suppress(Exception): - await out.write_eof() - return out - - -async def main() -> None: - app = web.Application(client_max_size=1024 ** 3) - app["session"] = ClientSession() - app.router.add_route("*", "/{tail:.*}", proxy) - runner = web.AppRunner(app) - await runner.setup() - await web.TCPSite(runner, "0.0.0.0", PORT).start() - print(f"proxy: :{PORT} -> {', '.join(UPSTREAMS)} routing={routing.ROUTING} " - f"key={routing.AFFINITY_BYTES}B known_max={routing.KNOWN_MAX}", flush=True) - watcher = asyncio.create_task(watch_health()) - with suppress(asyncio.CancelledError): - await asyncio.gather(watcher) - - -if __name__ == "__main__": - with suppress(KeyboardInterrupt): - asyncio.run(main()) diff --git a/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-a.container b/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-a.container deleted file mode 100644 index 4f121cc..0000000 --- a/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-a.container +++ /dev/null @@ -1,98 +0,0 @@ -# Quadlet unit: frank-medium replica A (rootless, boot-persistent). -# -# Install path on dankserver: ~/.config/containers/systemd/frank-medium-a.container -# Unit name: frank-medium-a.service (container name: frank-medium-a) -# -# WHY TWO REPLICAS INSTEAD OF TP=2. The intended shape for frank-medium is one -# tensor-parallel process across both R9700s (see frank-medium.container, kept -# as the target). That shape cannot run on this host: RCCL's collective kernels -# carry a hidden hostcall-buffer parameter, and the ROCr runtime refuses to -# dispatch them on a device without PCIe atomics ("Pcie atomics not enabled, -# hostcall not supported" -> AQL dispatch failed -> hipErrorIllegalState). The -# second R9700 sits behind the chipset bridges 05:00.0/06:09.0, which advertise -# `AtomicOpsCap: Routing-` — a read-only capability, so no software can enable -# atomics on that path; the amdgpu driver's pci_enable_atomic_ops_to_root() -# therefore fails for that GPU and KFD marks the CPU<->GPU2 link NO_ATOMICS. -# The first R9700 sits on CPU root port 00:01.1 (Routing+) and does have -# atomics. Result: every collective that touches GPU 2 dies at launch. -# Two independent single-GPU replicas use both cards instead; the proxy unit -# keeps the single :7770 endpoint. Full evidence: the frank-medium runbook. -# -# SHAPE. vLLM binds 0.0.0.0:7771 on the host network directly; no PublishPort. -# MAXLEN is 114688, not the TP=2 deployment's 262144: one card carries the -# whole model (~20 GiB of weights + drafter), leaving ~5.2 GiB of KV, and -# 131072 does not fit (measured: needs 5.58 GiB). KV_MEM=0 lets vLLM profile -# this shape instead of using the 2x7551-32624 pin, which is only valid at -# TP=2/262144. -# -# DEVICES. /dev/kfd + /dev/dri; HIP_VISIBLE_DEVICES=0 (the first R9700). The -# 2 GiB Raphael iGPU enumerates as index 2 and is excluded by the pin. -# Only HIP_VISIBLE_DEVICES is set, never ROCR_VISIBLE_DEVICES: ROCR filters -# first and HIP indexes into the filtered list, so a single-valued pair -# (ROCR=1,HIP=1) leaves an empty device set. -# -# DO NOT name an environment variable TEMP, HOST or GPU_MAX_HEAP_SIZE: the -# HIP runtime reads bare TEMP for its JIT cache path (2026-08-13 incident). -[Unit] -Description=frank-medium replica A — vllm-radiance MXFP4 Qwen3.8-27B on R9700 #1 -After=network-online.target - -[Container] -ContainerName=frank-medium-a -Image=localhost/radiance-vllm:0.9.3 -AddDevice=/dev/kfd -AddDevice=/dev/dri -GroupAdd=keep-groups -# The upstream launcher's exact podman posture (rootless). -PodmanArgs=--privileged --ipc=host --security-opt seccomp=unconfined --cap-add SYS_PTRACE --network=host -Environment=HIP_VISIBLE_DEVICES=0 -Environment=SNAP=/models/Qwen3.8-27B-Uncensored-MXFP4-awq -Environment=DRAFTER=/models/Qwen3.8-27B-DFlash2-FP8 -Environment=CHAT_TEMPLATE=/patches/qwen-fixed-v22.3.jinja -Environment=SERVED_NAMES=frank-medium -Environment=PORT=7771 -Environment=TP=1 -Environment=MAXSEQS=4 -Environment=CHUNK=8192 -Environment=MAXLEN=114688 -Environment=GPU_UTIL=0.98 -Environment=KV_MEM=0 -Environment=SPEC_METHOD=dflash -Environment=SPEC=7 -# Checkpoints (rw: nothing downloads at runtime, but vLLM mmaps the shards). -Volume=/home/dank/models:/models -# Tokenizer files ride the HF cache; HF_HUB_OFFLINE=1 is set in the entrypoint. -Volume=/home/dank/.cache/huggingface:/root/.cache/huggingface -# Compile cache. Shared with the other replica and with the TP=2 target: the -# entries are content-addressed, so shapes simply add subdirectories. -Volume=/home/dank/.radiance-cache-w4a8-093-gdnm-nqft-fp8s-gnq:/cache -# The pinned radiance clone: patches applied at every container start. -Volume=/home/dank/GitPackages/radiance-vllm-mxfp4:/patches:z -# The patched libr4d (pin b9e42ab + repo extras, key b9e42ab-rx6). The image's -# shipped r4d.so predates the GDN overflow fix and NaNs this model. -Volume=/home/dank/.cache/radiance-libr4d/b9e42ab-rx6:/r4d:z -# Entrypoint + healthcheck from this checkout, mounted read-only. -Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/radiance-entrypoint.sh:/usr/local/bin/frank-medium-entrypoint.sh:ro -Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/frank-medium-healthcheck.py:/usr/local/bin/frank-medium-healthcheck.py:ro -# The entrypoint must be the container command; the image's own entrypoint -# (which the patched script execs at the end) must not run unpatched. -PodmanArgs=--entrypoint /usr/local/bin/frank-medium-entrypoint.sh -# 200 only once the engine serves; during first-start compilation the -# container is legitimately unhealthy. HealthCmd is split on spaces into an -# argv array by the podman 5.x v1 generator, hence the file-based probe. -HealthCmd=python3 /usr/local/bin/frank-medium-healthcheck.py -HealthInterval=60s -HealthTimeout=30s -HealthRetries=3 -HealthStartPeriod=1800s -Pull=never - -[Service] -# First start compiles Triton/inductor kernels (several minutes) before the -# engine loads ~20 GiB of weights. -TimeoutStartSec=1800 -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=default.target diff --git a/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-b.container b/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-b.container deleted file mode 100644 index 4839383..0000000 --- a/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-b.container +++ /dev/null @@ -1,98 +0,0 @@ -# Quadlet unit: frank-medium replica B (rootless, boot-persistent). -# -# Install path on dankserver: ~/.config/containers/systemd/frank-medium-b.container -# Unit name: frank-medium-b.service (container name: frank-medium-a) -# -# WHY TWO REPLICAS INSTEAD OF TP=2. The intended shape for frank-medium is one -# tensor-parallel process across both R9700s (see frank-medium.container, kept -# as the target). That shape cannot run on this host: RCCL's collective kernels -# carry a hidden hostcall-buffer parameter, and the ROCr runtime refuses to -# dispatch them on a device without PCIe atomics ("Pcie atomics not enabled, -# hostcall not supported" -> AQL dispatch failed -> hipErrorIllegalState). The -# second R9700 sits behind the chipset bridges 05:00.0/06:09.0, which advertise -# `AtomicOpsCap: Routing-` — a read-only capability, so no software can enable -# atomics on that path; the amdgpu driver's pci_enable_atomic_ops_to_root() -# therefore fails for that GPU and KFD marks the CPU<->GPU2 link NO_ATOMICS. -# The first R9700 sits on CPU root port 00:01.1 (Routing+) and does have -# atomics. Result: every collective that touches GPU 2 dies at launch. -# Two independent single-GPU replicas use both cards instead; the proxy unit -# keeps the single :7770 endpoint. Full evidence: the frank-medium runbook. -# -# SHAPE. vLLM binds 0.0.0.0:7772 on the host network directly; no PublishPort. -# MAXLEN is 114688, not the TP=2 deployment's 262144: one card carries the -# whole model (~20 GiB of weights + drafter), leaving ~5.2 GiB of KV, and -# 131072 does not fit (measured: needs 5.58 GiB). KV_MEM=0 lets vLLM profile -# this shape instead of using the 2x7551-32624 pin, which is only valid at -# TP=2/262144. -# -# DEVICES. /dev/kfd + /dev/dri; HIP_VISIBLE_DEVICES=1 (the second R9700). The -# 2 GiB Raphael iGPU enumerates as index 2 and is excluded by the pin. -# Only HIP_VISIBLE_DEVICES is set, never ROCR_VISIBLE_DEVICES: ROCR filters -# first and HIP indexes into the filtered list, so a single-valued pair -# (ROCR=1,HIP=1) leaves an empty device set. -# -# DO NOT name an environment variable TEMP, HOST or GPU_MAX_HEAP_SIZE: the -# HIP runtime reads bare TEMP for its JIT cache path (2026-08-13 incident). -[Unit] -Description=frank-medium replica B — vllm-radiance MXFP4 Qwen3.8-27B on R9700 #2 -After=network-online.target - -[Container] -ContainerName=frank-medium-b -Image=localhost/radiance-vllm:0.9.3 -AddDevice=/dev/kfd -AddDevice=/dev/dri -GroupAdd=keep-groups -# The upstream launcher's exact podman posture (rootless). -PodmanArgs=--privileged --ipc=host --security-opt seccomp=unconfined --cap-add SYS_PTRACE --network=host -Environment=HIP_VISIBLE_DEVICES=1 -Environment=SNAP=/models/Qwen3.8-27B-Uncensored-MXFP4-awq -Environment=DRAFTER=/models/Qwen3.8-27B-DFlash2-FP8 -Environment=CHAT_TEMPLATE=/patches/qwen-fixed-v22.3.jinja -Environment=SERVED_NAMES=frank-medium -Environment=PORT=7772 -Environment=TP=1 -Environment=MAXSEQS=4 -Environment=CHUNK=8192 -Environment=MAXLEN=114688 -Environment=GPU_UTIL=0.98 -Environment=KV_MEM=0 -Environment=SPEC_METHOD=dflash -Environment=SPEC=7 -# Checkpoints (rw: nothing downloads at runtime, but vLLM mmaps the shards). -Volume=/home/dank/models:/models -# Tokenizer files ride the HF cache; HF_HUB_OFFLINE=1 is set in the entrypoint. -Volume=/home/dank/.cache/huggingface:/root/.cache/huggingface -# Compile cache. Shared with the other replica and with the TP=2 target: the -# entries are content-addressed, so shapes simply add subdirectories. -Volume=/home/dank/.radiance-cache-w4a8-093-gdnm-nqft-fp8s-gnq:/cache -# The pinned radiance clone: patches applied at every container start. -Volume=/home/dank/GitPackages/radiance-vllm-mxfp4:/patches:z -# The patched libr4d (pin b9e42ab + repo extras, key b9e42ab-rx6). The image's -# shipped r4d.so predates the GDN overflow fix and NaNs this model. -Volume=/home/dank/.cache/radiance-libr4d/b9e42ab-rx6:/r4d:z -# Entrypoint + healthcheck from this checkout, mounted read-only. -Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/radiance-entrypoint.sh:/usr/local/bin/frank-medium-entrypoint.sh:ro -Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/frank-medium-healthcheck.py:/usr/local/bin/frank-medium-healthcheck.py:ro -# The entrypoint must be the container command; the image's own entrypoint -# (which the patched script execs at the end) must not run unpatched. -PodmanArgs=--entrypoint /usr/local/bin/frank-medium-entrypoint.sh -# 200 only once the engine serves; during first-start compilation the -# container is legitimately unhealthy. HealthCmd is split on spaces into an -# argv array by the podman 5.x v1 generator, hence the file-based probe. -HealthCmd=python3 /usr/local/bin/frank-medium-healthcheck.py -HealthInterval=60s -HealthTimeout=30s -HealthRetries=3 -HealthStartPeriod=1800s -Pull=never - -[Service] -# First start compiles Triton/inductor kernels (several minutes) before the -# engine loads ~20 GiB of weights. -TimeoutStartSec=1800 -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=default.target diff --git a/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-proxy.container b/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-proxy.container deleted file mode 100644 index 0fe4f8f..0000000 --- a/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-proxy.container +++ /dev/null @@ -1,61 +0,0 @@ -# Quadlet unit: frank-medium proxy (rootless, boot-persistent). -# -# Install path on dankserver: ~/.config/containers/systemd/frank-medium-proxy.container -# Unit name: frank-medium-proxy.service (container name: frank-medium-proxy) -# -# WHAT IT IS. A prefix-affinity reverse proxy in front of the two single-GPU -# frank-medium replicas (:7771 on R9700 #1, :7772 on R9700 #2). It preserves -# the fleet contract: one endpoint on host port 7770 answering as the served -# model name frank-medium, OpenAI /v1, streaming included. -# -# WHY IT EXISTS. TP=2 — one vLLM process spanning both cards — is impossible -# on this host (see frank-medium-a.container and the runbook: RCCL's kernels -# need PCIe atomics, and the second card's chipset path advertises -# AtomicOpsCap: Routing-). Two replicas are how both cards get used. -# -# ROUTING. Requests are keyed on a hash of their leading 4 KiB of prompt text, -# so a conversation family — an agent loop re-sending the same system prompt, -# tool list and file context every turn — keeps landing on the same replica, -# where --enable-prefix-caching has that prefix warm. Plain round-robin threw -# half of those cache hits away. A replica that is down is skipped, and -# ROUTING=roundrobin restores the old behaviour for A/B measurement. /health -# answers 200 while at least one replica is up and names the ones that are -# down. The proxy runs on host networking so it can bind 7770 and reach the -# replicas on loopback. -# -# The script is a few dozen lines of aiohttp, mounted read-only out of -# /home/dank/serverAppConfigs (deploy.sh pulls that checkout before dispatch). -# It reuses the same image as the replicas: aiohttp is already there and no -# extra image has to be pulled on this host. -[Unit] -Description=frank-medium proxy — prefix-affinity in front of the two single-GPU replicas -After=network-online.target -Wants=frank-medium-a.service frank-medium-b.service -After=frank-medium-a.service frank-medium-b.service - -[Container] -ContainerName=frank-medium-proxy -Image=localhost/radiance-vllm:0.9.3 -PodmanArgs=--network=host -Environment=PORT=7770 -Environment=UPSTREAMS=http://127.0.0.1:7771,http://127.0.0.1:7772 -# The proxy is two files: routing.py holds the replica-selection policy, the -# server holds the HTTP plumbing. Both are mounted because the server imports -# the policy by name from its own directory (/usr/local/bin is sys.path[0]). -Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/proxy/routing.py:/usr/local/bin/routing.py:ro -Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/proxy/affinity-proxy.py:/usr/local/bin/affinity-proxy.py:ro -Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/frank-medium-healthcheck.py:/usr/local/bin/frank-medium-healthcheck.py:ro -PodmanArgs=--entrypoint /usr/local/bin/affinity-proxy.py -HealthCmd=python3 /usr/local/bin/frank-medium-healthcheck.py -HealthInterval=60s -HealthTimeout=30s -HealthRetries=3 -HealthStartPeriod=60s -Pull=never - -[Service] -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=default.target diff --git a/dankserver/.AGENTTRASH/20260916-vllm-radiance/routing.py b/dankserver/.AGENTTRASH/20260916-vllm-radiance/routing.py deleted file mode 100755 index e5b38b0..0000000 --- a/dankserver/.AGENTTRASH/20260916-vllm-radiance/routing.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python3 -"""Routing policy for the frank-medium proxy: which replica serves a request. - -Kept apart from the HTTP plumbing in affinity-proxy.py so the decisions can be -tested without aiohttp, a network or a GPU — the first version of these rules -shipped with a bug that only appeared on a continuation request, and a test -that loads this module catches that class of mistake directly. - -Two cards, two behaviours, and the workload decides which is right: - -* A **single-turn** request is one member of a batch (a benchmark run, a burst - of parallel subagents). Spreading those over both replicas lets both cards - prefill at once; measured on this host, a 16-request burst of 15k-token - prompts finishes in 18.6 s spread versus 30.1 s pinned. -* A **continuation** re-sends a prefix the replica has already prefilled. Pinned - to that replica the turn costs a cache lookup; spread, half the turns pay a - cold prefill again (measured: 1096 ms versus 2676 ms mean warm-turn TTFT on - 90k-token contexts). - -ROUTING picks the strategy; the default `adaptive` applies both rules above. -`affinity` and `roundrobin` exist so the choice can be re-measured, and are not -production settings. -""" - -from __future__ import annotations - -import collections -import hashlib -import itertools -import json -import os - -UPSTREAMS = [u.strip() for u in os.environ.get( - "UPSTREAMS", "http://127.0.0.1:7771,http://127.0.0.1:7772").split(",") if u.strip()] -ROUTING = os.environ.get("ROUTING", "adaptive").strip().lower() -# Leading bytes of prompt text that identify a conversation's shared prefix. -# 4 KiB covers a system prompt plus the start of the first user turn, which is -# where agent loops keep the text they resend every turn. -AFFINITY_BYTES = int(os.environ.get("AFFINITY_BYTES", "4096")) -# Prefixes remembered for pinning. Bounded so a long-lived proxy cannot grow -# without limit; 4096 conversations is far beyond what this fleet runs at once. -KNOWN_MAX = int(os.environ.get("KNOWN_MAX", "4096")) - -_counter = itertools.count() -# digest -> replica index, insertion-ordered for O(1) LRU eviction. A prefix is -# remembered as soon as any request carries it; only a continuation is pinned -# there, because only a continuation can profit from the cache entry. -_known: "collections.OrderedDict[bytes, int]" = collections.OrderedDict() - - -def affinity_key(body: bytes, remote: str | None) -> tuple[bytes, bool]: - """Key a request by its shared prefix, and report whether it continues one. - - Chat and completions bodies both carry their prompt text in JSON, so the - leading bytes of that text are the prefix; a body we cannot parse (or an - endpoint with no prompt, e.g. /v1/models) falls back to the client address, - which still beats nothing for a single-client loop. - """ - fallback = (remote or "anonymous").encode() - try: - payload = json.loads(body) - except Exception: - return fallback, False - - text = "" - continuation = False - if isinstance(payload, dict): - if isinstance(payload.get("prompt"), str): - text = payload["prompt"] - elif isinstance(payload.get("messages"), list): - continuation = len(payload["messages"]) > 1 - parts = [] - for msg in payload["messages"]: - if isinstance(msg, dict): - content = msg.get("content") - if isinstance(content, str): - parts.append(content) - elif isinstance(content, list): # multimodal content blocks - parts.extend( - blk.get("text", "") for blk in content - if isinstance(blk, dict) and isinstance(blk.get("text"), str) - ) - text = "\n".join(parts) - if not text: - return fallback, continuation - return text[:AFFINITY_BYTES].encode("utf-8", "replace"), continuation - - -def pick(key: bytes, continuation: bool, up: list[bool]) -> int: - """Replica for a request, or -1 when none is available. - - Down replicas are never selected — a rollout must not send traffic to an - engine that is still compiling its graphs — and a pin whose replica is down - is re-established on a live one rather than kept pointing at a corpse. - """ - digest = hashlib.sha1(key).digest() - - if ROUTING == "roundrobin": - start = next(_counter) % len(UPSTREAMS) - elif ROUTING == "adaptive": - entry = _known.get(digest) - if continuation and entry is not None and up[entry]: - _known.move_to_end(digest) - return entry - # Anything that is not a continuation spreads by counter — including a - # batch of requests that happen to share one prefix. Hashing the prefix - # would send every member of such a batch to whichever card the hash - # names (measured: 29.8 s for a 16-request burst, against 18.6 s when - # spread). The counter is only advanced by these requests, so batch - # members alternate across the cards. - start = next(_counter) % len(UPSTREAMS) - else: # affinity - start = int.from_bytes(digest[:8], "big") % len(UPSTREAMS) - - for offset in range(len(UPSTREAMS)): - idx = (start + offset) % len(UPSTREAMS) - if up[idx]: - if ROUTING == "adaptive": - _known[digest] = idx - _known.move_to_end(digest) - if len(_known) > KNOWN_MAX: - _known.popitem(last=False) - return idx - return -1 - - -def route(body: bytes, remote: str | None, up: list[bool]) -> int: - """Convenience wrapper: decode the request body and choose a replica.""" - return pick(*affinity_key(body, remote), up) diff --git a/scripts/.AGENTTRASH/20260920-llama-layersplit/bench-spec-ab.py.llamacpp-layersplit b/scripts/.AGENTTRASH/20260920-llama-layersplit/bench-spec-ab.py.llamacpp-layersplit deleted file mode 100755 index e453cc9..0000000 --- a/scripts/.AGENTTRASH/20260920-llama-layersplit/bench-spec-ab.py.llamacpp-layersplit +++ /dev/null @@ -1,309 +0,0 @@ -#!/usr/bin/env python3 -"""bench-spec-ab.py — settle "which proposer should this endpoint use?" with data. - -Runs one payload through several speculative-decoding configurations of the same -binary, one at a time, with the unit stopped between runs: on a single-GPU host -there is no room for two copies of a 29 GiB model, and the benchmarking policy -applies. Every configuration gets identical prompts at temperature 0, and the -greedy text is compared across them — speculation may buy speed, never change -the answer. - - bench-spec-ab.py [options] - bench-spec-ab.py /home/dank/serverAppConfigs/scripts/frank-medium.env \ - /home/dank/serverAppConfigs/scripts/serve-frank-medium.sh \ - --configs "dflash=draft-dflash@7;ngram=ngram-mod@64;chain=draft-dflash,ngram-mod@7" - -Config syntax: `name=SPEC_TYPE[@n_max][|ENV=VAL|ENV=VAL]`, entries joined by `;`. -The systemd unit is restored in a finally block, so it comes back on whatever the -env file says even if a configuration crashes the harness. - -THREE SHAPES, because a single one answers the wrong question. A proposer can -only draft text it has seen, so the shape decides the verdict: - - prose a 1k prompt asking for one sentence. Nothing to copy and nothing - repetitive, so only a model-backed proposer can help: this is the - shape that compares drafters. - echo a ~1.5k passage of unique lines, "repeat it back verbatim". The - output IS the prompt, which is ngram-mod's home turf. - structured "25 JSON records, one per line" — repetitive OUTPUT with no prompt - overlap, which is what an ngram matcher is really for in an - agentic fleet (config edits, tables, record lists). - -echo and structured disable thinking (`chat_template_kwargs`), otherwise the -model thinks the answer into `reasoning_content`, `content` comes back empty and -the cross-configuration identity check passes vacuously — which is exactly how -this harness first "verified" an echo it never looked at. - -Results on Dankserver (frank-medium payload, 2026-09-16, temperature 0). Decode: - - shape dflash=7 ngram-mod(48/64) chain (tok/s) - prose 1k/512 43.8 17.7 (0 drafts) 43.7 - echo 1.5k/400 76.8 102.5 200.6 - structured/400 91.1 17.8 (0 drafts) 91.2 - identity 70.4 17.8 (0 drafts) 70.4 - -`draft-mtp,draft-dflash` fails at load (a chain has ONE draft context and MTP -builds against whatever model that context holds); ngram-mod has no such -constraint — it is draftless and joins the chain unconditionally. -""" -import argparse -import json -import os -import re -import subprocess -import sys -import time -import urllib.request - -FILLER = ("The quick brown fox jumps over the lazy dog near the river bank. " - "A cache is a component that stores data so future requests are faster. ") -TOKENS_PER_REP = 28 -UNIT = "frank-medium.service" -MODEL = "frank-medium" -SPEC_GREP = ("adding speculative implementation|- n_max=|- block_size=|creating MTP|" - "draft acceptance") - -DEFAULT_CONFIGS = [ - ("none", {"SPEC_TYPE": "none"}), - ("mtp-n3", {"SPEC_TYPE": "draft-mtp", "SPEC_DRAFT_N_MAX": "3"}), - ("mtp-n7", {"SPEC_TYPE": "draft-mtp", "SPEC_DRAFT_N_MAX": "7"}), - ("dflash-n7", {"SPEC_TYPE": "draft-dflash", "SPEC_DRAFT_N_MAX": "7"}), -] - -WORDS = ["latency", "throughput", "cache", "adapter", "scheduler", "allocator", - "queue", "kernel", "buffer", "shard", "replica", "window"] - - -def echo_passage(lines=90): - """Unique lines: no long span repeats inside the prompt, so an n-gram can - only fire when the OUTPUT copies the INPUT.""" - return "\n".join( - "Record %03d: the %s of subsystem %d reports state %d with checksum %05d." - % (i, WORDS[i % len(WORDS)], i % 7, i % 3, (i * 7919) % 100000) - for i in range(lines)) - - -ECHO = (echo_passage() + "\n\nRepeat the passage above verbatim, every line, exactly as " - "written, with no commentary before or after it.") -STRUCTURED = ("Output exactly 25 JSON records, one per line, no prose and no code fence. " - 'Each line must look like {"id": N, "state": "active", "region": "us-west-2"} ' - "with N running from 1 to 25.") -PROSE = FILLER * (1000 // TOKENS_PER_REP) + "\n\nReply with a single short sentence." -IDENTITY = "List the first five prime numbers, comma separated, then stop." -NO_THINK = {"chat_template_kwargs": {"enable_thinking": False}} - - -def log(msg, logfile): - line = "[" + time.strftime("%H:%M:%S") + "] " + str(msg) - print(line, flush=True) - with open(logfile, "a") as fh: - fh.write(line + "\n") - - -def sh(cmd): - return subprocess.run(cmd, shell=True, capture_output=True, text=True).stdout.strip() - - -def parse_configs(spec): - """`name=SPEC_TYPE[@n_max][|KEY=VALUE]...`, entries joined by ';'. - - KEY=VALUE parts override any env file key, which is how an impl's own window - (SPEC_NGRAM_N_MIN/MAX, for instance) gets swept without hand-editing env - files.""" - out = [] - for chunk in spec.split(";"): - chunk = chunk.strip() - if not chunk: - continue - name, _, rest = chunk.partition("=") - over = {} - if "|" in rest: - rest, _, extras = rest.partition("|") - for item in extras.split("|"): - if "=" in item: - k, _, v = item.partition("=") - over[k.strip()] = v.strip() - n_max = None - if "@" in rest: - rest, _, n_max = rest.partition("@") - over["SPEC_TYPE"] = rest.strip() - if n_max: - over["SPEC_DRAFT_N_MAX"] = n_max.strip() - out.append((name.strip(), over)) - return out - - -def write_env(src, name, overrides): - """The env file with this configuration's overrides applied.""" - path = "/tmp/spec-ab-" + name.replace("/", "_") + ".env" - with open(src) as fh: - lines = fh.read().splitlines() - out = [] - for line in lines: - for k, v in overrides.items(): - if line.startswith(k + "="): - line = k + "=" + v - break - out.append(line) - with open(path, "w") as fh: - fh.write("\n".join(out) + "\n") - return path - - -def post(base, body, timeout=1800): - req = urllib.request.Request( - base + "/v1/chat/completions", data=json.dumps(body).encode(), - headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(req, timeout=timeout) as r: - return json.load(r) - - -def vram(): - out = sh("rocm-smi --showmeminfo vram 2>/dev/null | grep 'Used Memory'") - return [round(int(x) / 1e9, 2) for x in re.findall(r":\s+(\d+)\s*$", out, re.M)][:2] - - -def wait_health(base, timeout=420): - t0 = time.time() - while time.time() - t0 < timeout: - if not sh("pgrep -x llama-server >/dev/null && echo up"): - return False - try: - with urllib.request.urlopen(base + "/health", timeout=5) as r: - if r.status == 200 and json.load(r).get("status") == "ok": - return True - except Exception: - time.sleep(3) - return False - - -def stop_unit(): - sh("sudo systemctl stop " + UNIT) - sh("pkill -x llama-server") - for _ in range(40): - if not sh("pgrep -x llama-server || true"): - return - time.sleep(1) - - -def restore_unit(base, logfile): - """Bring the endpoint back on the deployed configuration. Runs in a - finally, because a harness that can leave production down is a hazard.""" - log("restoring " + UNIT, logfile) - sh("sudo systemctl start " + UNIT) - for _ in range(120): - try: - with urllib.request.urlopen(base + "/health", timeout=5) as r: - if json.load(r).get("status") == "ok": - log(UNIT + " healthy again", logfile) - return True - except Exception: - time.sleep(5) - log("WARNING: " + UNIT + " did not come back healthy -- check it by hand", logfile) - return False - - -def run_shape(base, prompt, max_tokens, label, extra=None): - body = {"model": MODEL, "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, "temperature": 0.0} - if extra: - body.update(extra) - d = post(base, body) - tm = d["timings"] - msg = d["choices"][0]["message"] - content = (msg.get("content") or "").strip() - row = {"shape": label, - "prompt_n": tm["prompt_n"], - "prefill_tps": round(1000 * tm["prompt_n"] / tm["prompt_ms"], 1), - "predicted_n": tm["predicted_n"], - "decode_tps": round(tm["predicted_per_second"], 2), - "draft_n": tm.get("draft_n", 0), - "accepted": tm.get("draft_n_accepted", 0), - "answer": content, - "reasoning_len": len(msg.get("reasoning_content") or "")} - if label.startswith("echo"): - # How much of the answer really is the passage: a configuration that - # "wins" by answering something else is not a win. - want = {l.strip() for l in echo_passage().splitlines()} - got = [l.strip() for l in content.splitlines() if l.strip()] - row["copy_ratio"] = round(sum(1 for l in got if l in want) / len(got), 3) if got else 0.0 - return row - - -def measure(base): - run_shape(base, FILLER * (4000 // TOKENS_PER_REP) + - "\n\nReply with a single short sentence.", 32, "warmup") - return [ - run_shape(base, PROSE, 512, "prose 1k/512"), - run_shape(base, ECHO, 400, "echo 1.5k/400", NO_THINK), - run_shape(base, STRUCTURED, 400, "structured/400", NO_THINK), - run_shape(base, IDENTITY, 192, "identity"), - ] - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("env_file") - ap.add_argument("serve_script") - ap.add_argument("--base-url", default="http://127.0.0.1:7770") - ap.add_argument("--log", default="/tmp/bench-spec-ab.log") - ap.add_argument("--configs", default=None, - help="name=SPEC_TYPE[@n_max][|ENV=VAL] joined by ';' " - "(default: the four configs measured on 2026-09-16)") - ap.add_argument("--json", default="/tmp/bench-spec-ab.json") - args = ap.parse_args() - - configs = parse_configs(args.configs) if args.configs else DEFAULT_CONFIGS - # Absolute paths: the launcher must not depend on the caller's cwd, and a - # relative path plus a `cd` to its own directory double-counts the prefix. - env_file = os.path.abspath(args.env_file) - serve_script = os.path.abspath(args.serve_script) - base, logfile = args.base_url, args.log - results = {} - log("spec A/B start: " + ", ".join(n for n, _ in configs) + - " (unit " + UNIT + " stopped for the duration)", logfile) - try: - stop_unit() - for name, over in configs: - env_path = write_env(env_file, name, over) - logf = "/tmp/spec-ab-" + name.replace("/", "_") + ".log" - sh("FRANK_MEDIUM_ENV_FILE=" + env_path + " setsid bash " + serve_script + - " > " + logf + " 2>&1 < /dev/null &") - if not wait_health(base): - log(name + ": FAILED to become healthy -- " + sh("tail -3 " + logf), logfile) - results[name] = {"error": "not healthy", "tail": sh("tail -6 " + logf)} - stop_unit() - continue - rows = measure(base) - results[name] = {"vram_gb": vram(), "rows": rows, - "spec_log": sh("grep -E '" + SPEC_GREP + "' " + logf + " | tail -4")} - for r in rows: - log(name + " | " + r["shape"] + " prefill " + str(r["prefill_tps"]) + - " tok/s | decode " + str(r["decode_tps"]) + " tok/s | draft " + - str(r["draft_n"]) + "/" + str(r["accepted"]) + - (" (" + str(round(100.0 * r["accepted"] / r["draft_n"])) + " %)" - if r["draft_n"] else "") + - (" | copy " + str(r["copy_ratio"]) if "copy_ratio" in r else "") + - (" | EMPTY ANSWER" if not r["answer"] else ""), logfile) - log(name + " | vram " + str(results[name]["vram_gb"]), logfile) - stop_unit() - finally: - restore_unit(base, logfile) - - with open(args.json, "w") as fh: - json.dump(results, fh, indent=2) - for shape in ("prose 1k/512", "echo 1.5k/400", "structured/400", "identity"): - texts = [r["answer"] for v in results.values() if "rows" in v - for r in v["rows"] if r["shape"] == shape] - if any(not t for t in texts): - log("WARNING: '" + shape + "' produced an empty answer in some configuration -- " - "its identity check proves nothing", logfile) - elif len(set(texts)) > 1: - log("WARNING: configurations disagree on greedy text for '" + shape + - "' -- speculation is changing answers", logfile) - log("done; results in " + args.json, logfile) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file diff --git a/scripts/.AGENTTRASH/20260920-llama-layersplit/frank-medium.env.llamacpp-layersplit b/scripts/.AGENTTRASH/20260920-llama-layersplit/frank-medium.env.llamacpp-layersplit deleted file mode 100644 index f41afe8..0000000 --- a/scripts/.AGENTTRASH/20260920-llama-layersplit/frank-medium.env.llamacpp-layersplit +++ /dev/null @@ -1,318 +0,0 @@ -# Tunables for frank-medium (orcarouter/Qwen3.8-27B-Uncensored-GGUF Q8_0 + -# z-lab/Qwen3.8-27B-DFlash2-GGUF Q4_K_M drafter), mainline llama.cpp on -# Dankserver, 2x AMD Radeon AI PRO R9700 (gfx1201). Since 2026-09-16 this is -# the SOLE LLM runtime on the box: the vllm-radiance replica pair and its -# routing proxy are retired. -# -# Hardware: two R9700 (Navi 48, RDNA4, 32,624 MiB each), no XGMI, different -# IOMMU groups -- every cross-card byte goes over PCIe. The Raphael iGPU -# (gfx1036/Radeon 610M) enumerates as HIP device 2 and is hidden below so -# nothing can drift onto shared system RAM. -# -# WHY LAYER SPLIT AND NOT TENSOR (operator call, 2026-09-16): tensor parallel -# needs a cross-card allreduce per layer, and this host cannot dispatch one. -# RCCL's kernels carry a hidden hostcall-buffer parameter, and ROCr refuses it -# on a device without PCIe atomics -- the second card sits behind the B650 -# chipset bridges (AtomicOpsCap: Routing-, read-only), so every collective that -# touches it dies with hipErrorIllegalState. Measured on this box 2026-09-15 -# (llama-bench, pp4096 / tg128, f16 KV, fa on): -# -# single R9700 1160.9 / 19.42 -# layer 2x 1991.2 / 18.76 (859.1 / 17.64 at depth 32768) -# tensor 2x 271.8 / 26.26 (no RCCL: butterfly fallback, -7x prefill) -# -# Layer split distributes layers in order; only the activation at the boundary -# crosses the wire, once per forward pass. That is the "pipeline parallelism" -# this service wanted, and it needs no collective at all. -# -# Runtime: mainline llama.cpp master fb27a525 (2026-09-16), gfx1201, built by -# scripts/build-llama-rdna4.sh against a private ROCm 7.14.0a20260624 toolchain -# that is bundled next to the binary (RPATH $ORIGIN/../lib, no LD_LIBRARY_PATH). -# The ref is pinned by TWO features: DFlash2 (ggml-org/llama.cpp#27342 -- the -# drafter is detected from its GGUF) and qwen35 MTP. Do not float it. -# -# Model: orcarouter/Qwen3.8-27B-Uncensored-GGUF @ Q8_0, 27.0 GiB, arch qwen35 -# (verified from the GGUF header 2026-09-19: 262144 trained window, 24 heads / -# 4 KV heads, nextn_predict_layers=1). The file ALSO carries the model's MTP -# head (blk.64.nextn.*), so --spec-type draft-mtp is available without a second -# copy in VRAM -- one SPEC_TYPE change, the drafter below stays on disk. -# -# Drafter: z-lab/Qwen3.8-27B-DFlash2-GGUF @ Q4_K_M, 1.06 GiB (operator call -# 2026-09-19: the Q4 rung -- the drafter's job is throughput, not output -# quality, and the target verifies every drafted token). 5 layers, hidden -# 5120, a block-diffusion proposal (block_size 8) under a 2048-token sliding -# window (dflash.attention.sliding_window=2048, verified from the header). -# The window is what makes it cheap HERE: its KV is bounded at 2048 positions -# (~40 MiB) instead of spanning the context (see SIZING). -# -# Architecture (GGUF arch qwen35): 64 layers as 16 x (3 x (Gated DeltaNet -> -# FFN) + 1 x (Gated Attention -> FFN)) -- 48 linear-attention layers whose -# per-sequence state is CONSTANT-SIZE, 16 full-attention layers at 4 KV heads x -# 256 head_dim. A token therefore costs 2 x 4 x 256 x 2 B = 64 KiB of KV at f16 -# in the attention layers only, plus ~0.16 GiB per sequence of f32 DeltaNet -# state at any depth. -# -# SIZING, the deployed shape (32:33 / ctx 327680 / 6144x512 / two slots / -# DFlash2 Q4_K_M), measured 2026-09-19 after the model swap (rocm-smi): -# -# card 0 (x16) 26.9 GiB used, 5.7 GiB free -# card 1 (x1) 27.8 GiB used, 4.8 GiB free -# -# The predecessor's measured table (rocm-smi, 2026-09-16, UD-Q8_K_XL target) -# stays as the calibration history: -# -# CTX split batch slots drafter GPU 0 free GPU 1 free note (ubatch 1536) -# 262144 1,1 2048 1 DFlash2 8.64 GiB 4.43 GiB day's start -# 262144 39,26 6144 2 DFlash2 2.42 GiB 7.20 GiB -# 262144 38,27 6144 2 DFlash2 3.85 GiB 5.77 GiB -# 262144 37,28 6144 2 DFlash2 4.28 GiB 5.34 GiB -# 262144 36,29 6144 2 DFlash2 4.71 GiB 4.91 GiB -# 262144 35,30 6144 2 DFlash2 5.14 GiB 4.48 GiB -# 262144 40,25 6144 2 DSpark 1.87 GiB 1.86 GiB (retired) -# -# ...and the same deployed shape re-measured at ubatch 512, after llama-bench -# showed 512 is the throughput optimum and costs 3.71 GiB less in total (see the -# batch block below): -# -# 262144 36,29 6144 512 2 DFlash2 6.01 GiB 7.32 GiB ubatch only -# 262144 35,30 6144 512 2 DFlash2 6.44 GiB 6.90 GiB -# -# ONE -ts STEP IS ONE LAYER, AND LAYERS ARE NOT THE SAME SIZE. This is a hybrid -# stack -- 16 x (3 Gated DeltaNet + 1 Gated Attention) blocks -- so moving a -# DeltaNet layer shifts ~0.43 GiB while an attention layer shifts ~1.43 GiB -# (0.45 of weights plus ~1.0 GiB of its own f16 KV at 262144). Both step sizes -# were measured, which is why the balance is found by measurement rather than by -# interpolating between splits. -# -# WHY THE SPLIT IS 32:33 (operator call, 2026-09-19). The operator asked for -# the tensors BALANCED across the two cards, which for 65 layers is 32:33 -- -# the most even split that exists. The predecessor's 35:30 skew existed to lean -# work onto the x16 card (ROCm0), because ROCm1's Gen4 x1 chipset path made it -# the 98 % utilization bottleneck; the measured trade of the balanced point is -# that the x1 card runs hotter while the x16 card idles a little more, and the -# operator accepted that for symmetry. Do not re-skew without asking. -# -# THE TWO CARDS' PATHS ARE NOT SYMMETRIC (measured, 2026-09-17): ROCm0 -# (03:00.0) hangs off CPU root port 00:01.1 at Gen4 x16 end to end (~31.5 GB/s); -# ROCm1 (0b:00.0) rides the B650 chipset through bridges that each negotiate -# Gen4 x1 (~1.97 GB/s). The layer-split boundary itself is not the cost -# (~5.2 MB of activation per 512-token ubatch rides the wire); the x1 card is -# COMPUTE-bound, not PCIe-bound. -# -# A FAULT TO WATCH. A 39:26 split, at 1x batch / 1 slot, died on its eighth -# request: the amdgpu driver logged SDMA0 UTCL2 page faults and the engine -# surfaced `ROCm error: an illegal memory access was encountered` -> ggml_abort, -# i.e. a fault in the cross-card boundary copy, this layer split's hot path. It -# has not reproduced since, but the signature is worth recognising -- it is NOT -# an OOM, and no amount of headroom prevents it. -# -# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (HIP reads -# those bare vars; 2026-08-13 incident). llama.cpp takes --host/--port/--temp -# as flags only. - -# llama-server: mainline fb27a525 built for gfx1201 by scripts/build-llama-rdna4.sh. -LLAMA_BIN=/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server - -# Payload, fetched with scripts/download-frank-medium-gguf.sh. -MODEL_DIR=/home/dank/models/Qwen3.8-27B-Uncensored-GGUF -MODEL_FILE=Qwen3.8-27B-Uncensored-Q8_0.gguf -# The f16 vision projector, same repo as the target. llama-server loads it via -# the explicit --mmproj flag (no autoload in this build); the serve script -# refuses to start without it when vision is enabled, so a missing file is a -# deploy error, not a text-only server that pretends to be multimodal. -MM_PROJ_FILE=mmproj-Qwen3.8-27B-Uncensored-f16.gguf -DRAFT_MODEL_DIR=/home/dank/models/Qwen3.8-27B-DFlash2-GGUF -DRAFT_MODEL_FILE=Qwen3.8-27B-DFlash2-Q4_K_M.gguf - -# Fleet identity. 7770 is the fleet's dankserver endpoint; the alias is the id -# at /v1/models and what every client asks for. -ALIAS=frank-medium -BIND_ADDR=0.0.0.0 -LISTEN_PORT=7770 -# Device placement. HIP_VISIBLE_DEVICES hides the Raphael iGPU (HIP device 2) -# so nothing can drift onto shared system RAM; DEVICES is what llama.cpp splits -# across, and neither of the two cards has an XGMI link to the other. -HIP_VISIBLE_DEVICES=0,1 -DEVICES=ROCm0,ROCm1 -SPLIT_MODE=layer - -# 32:33 -- balanced (operator call 2026-09-19): 65 layers split as evenly as -# they can be, ROCm1 (the last-layers card, which also carries the drafter and -# the output layers) holding the extra one. See the WHY THE SPLIT IS 32:33 -# block above for the trade against the predecessor's 35:30 x16-lean. -TENSOR_SPLIT=32,33 - -NGL=999 -# Auto-fit has no business resizing a context we set explicitly. -FIT=off -FLASH_ATTN=on - -# 327680 positions over TWO slots, so each conversation gets 163840 -- -# --parallel DIVIDES --ctx-size, so more slots would shrink every -# conversation rather than grow the cache. 163840 is inside the checkpoint's -# trained 262144 window, so the whole pool is trained territory (no rope -# extrapolation, unlike the predecessor's 393216). -CTX_SIZE=327680 -PARALLEL=2 - -# MEASURED 2026-09-16 with scripts/bench-batch-ubatch.sh on this box: llama-bench, -# a 64k prefill plus a 128-token decode with 64k of KV already resident, 3 reps, -# one invocation per cell, the deployed split and f16 KV. --ubatch-size is the -# physical batch the graph is built at -- and the only one of the two knobs that -# costs VRAM: -# -# batch ubatch prefill tok/s GPU 0 / GPU 1 free -# 6144 1536 1164.96 +- 2.30 10.28 / 11.71 GiB -# 6144 768 1135.03 +- 1.09 11.29 / 13.28 GiB -# 6144 512 1192.93 +- 0.91 11.63 / 13.81 GiB <- deployed -# 6144 384 1068.04 +- 0.79 11.80 / 14.07 GiB -# 6144 256 1045.98 +- 1.69 11.97 / 14.33 GiB -# 4096 1536 1161.90 +- 0.96 10.28 / 11.71 GiB -# 2048 1536 1170.23 +- 0.86 10.28 / 11.71 GiB -# 2048 512 1193.88 +- 1.01 11.63 / 13.81 GiB -# -# THE PLATEAU STARTS AT 512. 512 -> 1536 buys ~2 % of prefill -- inside the -# run-to-run spread, and the 512 cells were the faster pair in BOTH batch -# columns -- while costing 1.35 GiB on ROCm0 and 2.10 GiB on ROCm1, so the -# deployment takes the cheap end. BELOW 512 the SMs are no longer saturated: -# 384 and 256 fall ~11 % and ~12 %. The logical batch is free either way -- -# 2048, 4096 and 6144 land within 2 % of each other at a fixed ubatch and use -# the SAME VRAM, because the compute buffers are sized by ubatch alone. -# -# The prefill figure is a property of the SHAPE, not of this knob: the same -# sweep with no depth in front of it (depth 0) runs 1543 tok/s at ubatch 256 and -# 1723 at 1536 -- the plateau is in the same place, and the ~32 % between the two -# tables is what attending over 64k of resident KV costs. Decode is 15.88 tok/s -# at depth 65536 on EVERY ubatch (17.80 at depth 0), so this knob cannot touch -# generation at all. -# -# 8192/2048 stays untested under this drafter (the "2020 MiB pp buffer fails on -# ROCm1" note was measured under DSpark, when it had 0.08 GiB free), and there is -# no longer a reason to want it: batch buys nothing measurable here. -BATCH_SIZE=6144 -UBATCH_SIZE=512 - -# Threads cover tokenization, sampling and HTTP only: every layer is on a GPU. -NUM_THREADS=8 -THREAD_BATCH=8 - -# f16 K/V: an agentic checkpoint is the wrong place to spend KV precision. -CACHE_TYPE_K=f16 -CACHE_TYPE_V=f16 -# Context checkpoints: 0 -- no per-slot KV snapshots at all. The build's -# default is 32 (this binary's --help), so the 0 below is a deliberate -# deviation, not an inherited default: a checkpoint is a VRAM->host-RAM copy -# of part of a slot's KV, taken so a partially-matching prompt can be restored -# instead of re-prefilled. This endpoint trades that away -- its reuse comes -# from the slot prompt and from the RAM-side prompt cache below. Stated -# explicitly so the tunable stays visible rather than merely absent. -CTX_CHECKPOINTS=0 -# PROMPT CACHE, HOST RAM -- this is the one cache tunable that is NOT VRAM. -# --cache-ram caps the total bytes of saved prompt states -# (server_prompt_cache, tools/server/server-task.cpp): update() evicts -# oldest-first past the cap, and alloc() REFUSES outright any single state -# bigger than the whole cap. A state is the serialized KV of a whole sequence -# plus its DeltaNet state plus the draft context's own KV, so at this shape it -# costs ~68 KiB per token -- -# 16 full-attention layers x 4 kv heads x 256 dim x 2 B x 2 (K,V) = 64 KiB -# plus the draft context's window-bounded KV (a ~40 MiB constant for DFlash2) -# -- which makes the cap a TOKEN budget in disguise: 16384 MiB holds ~240k -# tokens, i.e. a FULL 163840-token slot (~10.6 GiB) with ~5 GiB of the budget -# spare for a second entry. -# -# The cost is host RAM, allocated on demand and never pinned (plain heap -# vectors), so no VRAM number above moves. dankserver has 62 GiB beside the -# media-stack co-tenants, so the cache is bounded rather than unlimited (-1): -# the seam test refuses a budget over half the host's RAM. -CACHE_RAM=16384 - -# Sampling defaults for requests that send none -- the values the vLLM -# predecessor served on this endpoint (`--override-generation-config`), so a -# client sees no change across the runtime swap. The model card's own -# recommendation is temp 1.0 / top_p 0.95; the harness sets its own per model in -# models.yml, so this is the floor, not the contract. -TEMPERATURE=0.7 -TOP_P=0.95 -TOP_K=20 -MIN_P=0.0 -PRESENCE_PENALTY=0.0 - -# Reasoning on, projected into reasoning_content (deepseek format) with no -# budget: the model decides how long to think. -REASONING=on -REASONING_FORMAT=deepseek -REASONING_BUDGET=-1 - -# Chat template: empty = the one embedded in the GGUF (Qwen3.8's own). -CHAT_TEMPLATE_FILE= -CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true}' - -# Speculative decoding: the DFlash2 sidecar (operator call 2026-09-19). The -# target file also carries its own MTP head (nextn_predict_layers=1), so -# draft-mtp is one SPEC_TYPE change away -- the drafter file stays on disk. -# n_max 7 is the measured DFlash2 winner (see the table below); the drafter -# rides ROCm1, the card that holds the last layers. -SPEC_TYPE=draft-dflash -SPEC_DRAFT_N_MAX=7 -SPEC_DRAFT_NGL=999 -SPEC_DRAFT_DEVICE=ROCm1 -SPEC_DRAFT_TYPE_K=f16 -SPEC_DRAFT_TYPE_V=f16 - -# DFlash2 vs the model's own MTP head -- MEASURED 2026-09-16 on the predecessor -# payload (temperature 0, 1k prompt / 512 out; the autoregressive baseline and -# both MTP depths were run in one session, same binary, same prompts): -# -# none 17.7 tok/s card1 26.07 GB -# draft-mtp n=3 32.4 tok/s 58 % accepted card1 29.35 GB -# draft-mtp n=7 26.7 tok/s 33 % accepted card1 29.65 GB -# draft-dflash 7 43.5 tok/s 41 % accepted card1 29.42 GB -# -# All four produced IDENTICAL greedy text, so verification is lossless -# whichever proposer runs. DFlash2 is the faster proposer (2.46x over -# autoregressive, 34 % over the best MTP). The MTP head adds ~3.3 GB to the -# card that holds it, against 1.06 GB for the Q4_K_M drafter's file -- at the -# 327680 pool the drafter's window-bounded KV (~40 MiB) leaves headroom on -# both cards, so the speed wins. -# -# THE TWO CANNOT BE CHAINED. `--spec-type draft-mtp,draft-dflash` dies at load: -# context type MTP requested but model doesn't contain MTP layers -# failed to create MTP context -# A chain has ONE draft context, and MTP builds against whatever model that -# context holds -- here the DFlash2 file, which has no MTP layers. MTP's head is -# only loaded when MTP IS the draft model ("creating MTP draft context against -# the target model ..."), so a chain of the two is a structural dead end, not a -# tuning question. See serverAppConfigs-docs/LLM/frank-medium-llamacpp-dflash2.md. - -# ngram-mod: the draftless n-gram matcher (CPU hash table in host RAM, no VRAM, -# no model file, no draft context). It joins the chain unconditionally -- -# unlike MTP, which needs a draft context and therefore cannot share one with -# DFlash2. NOT CURRENTLY ACTIVE: SPEC_TYPE is draft-dflash alone, and the serve -# script only passes --spec-ngram-mod-* when the type contains "ngram". The -# window below is retained because the DFlash2 + ngram chain was measured free -# on prose and 2.6x on prompt-echo -- one SPEC_TYPE change brings it back. -# -# WINDOW MEASURED 2026-09-16 (chain at each window, decode tok/s): -# -# n_min/n_max prose 1k/512 prompt-echo 1.5k/400 -# 48/64 43.9 200.6 -# 16/32 36.0 161.0 -# 8/16 36.2 113.8 -# -# The long window is the right one, and not only because it wins on echo: a -# SHORT window makes the matcher propose on short fragments of ordinary prose, -# where the proposals are rejected and the verification still costs -- which is -# the "ngram hurts" result the fleet measured on other payloads twice before. -# With 48/64 the chain is byte-identical to DFlash2 alone on prose (same 119 -# drafts / 49 accepted) and on the identity prompt (70/53). -# -# Do not shorten these without re-running scripts/bench-spec-ab.py: the -# no-penalty property is what makes the chain safe as a default. -SPEC_NGRAM_N_MATCH=24 -SPEC_NGRAM_N_MIN=48 -SPEC_NGRAM_N_MAX=64 - -# Vision enabled: llama-server loads mmproj-Qwen3.8-27B-Uncensored-f16.gguf -# (931 MiB, lands on ROCm0 -- the fit charges it there) via the explicit -# --mmproj flag. This keeps frank-medium multimodal across the model swap. -NO_MMPROJ=0 diff --git a/scripts/.AGENTTRASH/20260920-llama-layersplit/frank-medium.service.llamacpp-layersplit b/scripts/.AGENTTRASH/20260920-llama-layersplit/frank-medium.service.llamacpp-layersplit deleted file mode 100644 index 5523942..0000000 --- a/scripts/.AGENTTRASH/20260920-llama-layersplit/frank-medium.service.llamacpp-layersplit +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=llama.cpp Qwen3.8-27B Uncensored UD-Q8_K_XL + DFlash2 drafter (dankserver, frank-medium, 2x Radeon AI PRO R9700) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=dank -Group=dank -# The engine pins staging buffers on the first card; keep the memlock budget -# unbounded so an RLIMIT that is smaller than the split buffers cannot fail the -# load. Parity with the other LLM units on this fleet. -LimitMEMLOCK=infinity -Environment="FRANK_MEDIUM_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-medium.env" -ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-medium.sh -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/scripts/.AGENTTRASH/20260920-llama-layersplit/serve-frank-medium.sh.llamacpp-layersplit b/scripts/.AGENTTRASH/20260920-llama-layersplit/serve-frank-medium.sh.llamacpp-layersplit deleted file mode 100755 index 339f600..0000000 --- a/scripts/.AGENTTRASH/20260920-llama-layersplit/serve-frank-medium.sh.llamacpp-layersplit +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env bash -# Serve ajgazin/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF @ UD-Q8_K_XL as -# frank-medium on Dankserver's two Radeon AI PRO R9700 (gfx1201), with the -# z-lab DFlash2 block-diffusion drafter, using mainline llama.cpp. -# -# Runtime: llama.cpp master fb27a525 (2026-09-16), a gfx1201 HIP build that -# bundles its own ROCm 7.14 runtime (RPATH $ORIGIN/../lib), produced by -# scripts/build-llama-rdna4.sh. Two features pin that ref: DFlash2 -# (ggml-org/llama.cpp#27342) and qwen35 MTP. The charlie12345/ROCmFPX fork -# this deployment replaced has neither (its tip is c49ebdb, 2026-08-22). -# -# Parallelism: --split-mode layer across both cards. Tensor split would need a -# cross-card allreduce and this host cannot dispatch one -- RCCL's kernels -# require a hostcall buffer that ROCr refuses on a card without PCIe atomics, -# and the second R9700 sits behind the chipset (AtomicOpsCap: Routing-). Layer -# split crosses the wire once per forward pass instead of once per layer. -# -# The vLLM replica pair and its routing proxy were retired with this cutover: -# one process, one model, both cards. -# -# Tunables come from frank-medium.env unless FRANK_MEDIUM_ENV_FILE overrides it. -set -euo pipefail - -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -ENV_FILE="${FRANK_MEDIUM_ENV_FILE:-$SCRIPT_DIR/frank-medium.env}" -if [[ -f "$ENV_FILE" ]]; then - set -a - # shellcheck disable=SC1090 - source "$ENV_FILE" - set +a -fi - -# --- defaults (used when the env file does not set them) ------------------ -: "${LLAMA_BIN:=/home/dank/GitPackages/llama.cpp-dflash2/build-rdna4-rocm714/bin/llama-server}" -: "${MODEL_DIR:=/home/dank/models/Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF}" -: "${MODEL_FILE:=Swift-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf}" -: "${DRAFT_MODEL_DIR:=/home/dank/models/Qwen3.8-27B-DFlash2-GGUF}" -: "${DRAFT_MODEL_FILE:=Qwen3.8-27B-DFlash2-Q8_0.gguf}" -: "${ALIAS:=frank-medium}" -: "${BIND_ADDR:=0.0.0.0}" -: "${LISTEN_PORT:=7770}" - -# Devices. HIP_VISIBLE_DEVICES hides the Raphael iGPU (HIP device 2) so nothing -# can drift onto shared system RAM; DEVICES is what llama.cpp splits across. -: "${HIP_VISIBLE_DEVICES:=0,1}" -: "${DEVICES:=ROCm0,ROCm1}" -: "${SPLIT_MODE:=layer}" -: "${TENSOR_SPLIT:=1,1}" -: "${NGL:=999}" -: "${FIT:=off}" -: "${FLASH_ATTN:=on}" - -: "${CTX_SIZE:=131072}" -: "${PARALLEL:=1}" -: "${BATCH_SIZE:=2048}" -: "${UBATCH_SIZE:=512}" -: "${NUM_THREADS:=8}" -: "${THREAD_BATCH:=8}" -: "${CACHE_TYPE_K:=f16}" -: "${CACHE_TYPE_V:=f16}" -: "${CTX_CHECKPOINTS:=8}" -: "${CACHE_RAM:=8192}" - -# Sampling defaults for silent clients: the values the vLLM predecessor served. -: "${TEMPERATURE:=0.7}" -: "${TOP_P:=0.95}" -: "${TOP_K:=20}" -: "${MIN_P:=0.0}" -: "${PRESENCE_PENALTY:=0.0}" - -: "${REASONING:=on}" -: "${REASONING_FORMAT:=deepseek}" -: "${REASONING_BUDGET:=-1}" -: "${CHAT_TEMPLATE_FILE:=}" -: "${CHAT_TEMPLATE_KWARGS:={\"enable_thinking\":true,\"preserve_thinking\":true}}" - -# Speculative decoding: draft-dflash or draft-dspark (both are MODEL drafters -- -# a file, not a flag), or draft-mtp (the target GGUF's own MTP head, no draft -# file). The drafter's own tuning knobs (n-min, p-min, p-split, poll) are left -# at the build's defaults: they were only ever passed for dspark, and dspark is -# not the deployed drafter. -: "${SPEC_TYPE:=draft-dflash}" -: "${SPEC_DRAFT_N_MAX:=7}" -: "${SPEC_DRAFT_NGL:=999}" -: "${SPEC_DRAFT_DEVICE:=}" -: "${SPEC_DRAFT_TYPE_K:=f16}" -: "${SPEC_DRAFT_TYPE_V:=f16}" -: "${SPEC_NGRAM_N_MATCH:=24}" -: "${SPEC_NGRAM_N_MIN:=48}" -: "${SPEC_NGRAM_N_MAX:=64}" - -NO_MMPROJ="${NO_MMPROJ:-0}" -: "${MM_PROJ_FILE:=mmproj-BF16.gguf}" - -export HIP_VISIBLE_DEVICES - -MODEL="$MODEL_DIR/$MODEL_FILE" -DRAFT_MODEL="$DRAFT_MODEL_DIR/$DRAFT_MODEL_FILE" - -if [[ ! -x "$LLAMA_BIN" ]]; then - echo "missing executable runtime: $LLAMA_BIN" >&2 - echo "build it with: bash $SCRIPT_DIR/build-llama-rdna4.sh" >&2 - exit 1 -fi -if [[ ! -r "$MODEL" ]]; then - echo "missing or unreadable target model: $MODEL" >&2 - echo "fetch it with: bash $SCRIPT_DIR/download-frank-medium-gguf.sh" >&2 - exit 1 -fi -if [[ "$NO_MMPROJ" != "1" ]]; then - MM_PROJ="$MODEL_DIR/$MM_PROJ_FILE" - if [[ ! -r "$MM_PROJ" ]]; then - echo "vision is enabled but the projector is missing: $MM_PROJ" >&2 - echo "fetch it with: bash $SCRIPT_DIR/download-frank-medium-gguf.sh" >&2 - exit 1 - fi -fi -if [[ -n "$CHAT_TEMPLATE_FILE" && ! -r "$CHAT_TEMPLATE_FILE" ]]; then - echo "missing or unreadable chat template: $CHAT_TEMPLATE_FILE" >&2 - exit 1 -fi - -# --- speculative decoding ------------------------------------------------- -SPEC_ARGS=() -if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then - SPEC_ARGS=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX") - if [[ "$SPEC_TYPE" == *dflash* || "$SPEC_TYPE" == *dspark* ]]; then - if [[ ! -r "$DRAFT_MODEL" ]]; then - echo "missing or unreadable draft model: $DRAFT_MODEL" >&2 - exit 1 - fi - SPEC_ARGS+=( - --spec-draft-model "$DRAFT_MODEL" - --spec-draft-ngl "$SPEC_DRAFT_NGL" - --spec-draft-type-k "$SPEC_DRAFT_TYPE_K" - --spec-draft-type-v "$SPEC_DRAFT_TYPE_V" - ) - fi - # ngram-mod is draftless: it needs no draft context and its window is its - # own (--spec-ngram-mod-*), so it can ride in the chain beside a model - # drafter. It also costs no VRAM -- a CPU hash table. - if [[ "$SPEC_TYPE" == *ngram* ]]; then - SPEC_ARGS+=( - --spec-ngram-mod-n-match "$SPEC_NGRAM_N_MATCH" - --spec-ngram-mod-n-min "$SPEC_NGRAM_N_MIN" - --spec-ngram-mod-n-max "$SPEC_NGRAM_N_MAX" - ) - fi - if [[ -n "$SPEC_DRAFT_DEVICE" ]]; then - SPEC_ARGS+=(--spec-draft-device "$SPEC_DRAFT_DEVICE") - fi -fi - -# --- template + placement ------------------------------------------------- -TEMPLATE_ARGS=() -if [[ -n "$CHAT_TEMPLATE_FILE" ]]; then - TEMPLATE_ARGS+=(--chat-template-file "$CHAT_TEMPLATE_FILE") -fi -TEMPLATE_ARGS+=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS") - -PLACEMENT_ARGS=(--device "$DEVICES" --split-mode "$SPLIT_MODE" --tensor-split "$TENSOR_SPLIT") - -MMPROJ_ARGS=() -if [[ "$NO_MMPROJ" == "1" ]]; then - MMPROJ_ARGS=(--no-mmproj) -else - # No autoload in this build: a server without the flag starts text-only - # and answers image requests with a 500, so name the projector. - MMPROJ_ARGS=(--mmproj "$MM_PROJ") -fi -exec "$LLAMA_BIN" \ - --model "$MODEL" \ - --alias "$ALIAS" \ - --host "$BIND_ADDR" \ - --port "$LISTEN_PORT" \ - --jinja \ - --gpu-layers "$NGL" \ - --fit "$FIT" \ - --parallel "$PARALLEL" \ - --ctx-size "$CTX_SIZE" \ - --batch-size "$BATCH_SIZE" \ - --ubatch-size "$UBATCH_SIZE" \ - --flash-attn "$FLASH_ATTN" \ - --cache-type-k "$CACHE_TYPE_K" \ - --cache-type-v "$CACHE_TYPE_V" \ - --threads "$NUM_THREADS" \ - --threads-batch "$THREAD_BATCH" \ - --ctx-checkpoints "$CTX_CHECKPOINTS" \ - --cache-ram "$CACHE_RAM" \ - --cache-prompt \ - --metrics \ - --temp "$TEMPERATURE" \ - --top-p "$TOP_P" \ - --top-k "$TOP_K" \ - --min-p "$MIN_P" \ - --presence-penalty "$PRESENCE_PENALTY" \ - --reasoning "$REASONING" \ - --reasoning-format "$REASONING_FORMAT" \ - --reasoning-budget "$REASONING_BUDGET" \ - "${TEMPLATE_ARGS[@]}" \ - "${PLACEMENT_ARGS[@]}" \ - "${MMPROJ_ARGS[@]}" \ - "${SPEC_ARGS[@]}" diff --git a/scripts/.AGENTTRASH/bench-decode-threads-batch.py.strix-rocmfp4 b/scripts/.AGENTTRASH/bench-decode-threads-batch.py.strix-rocmfp4 deleted file mode 100755 index 0087886..0000000 --- a/scripts/.AGENTTRASH/bench-decode-threads-batch.py.strix-rocmfp4 +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env python3 -""" -Decode throughput benchmark for frank-dragoon. -Starts the server with a given --threads-batch value, sends an 8K+ token prompt, -generates 256 tokens, measures generation tok/s. - -Each --threads-batch value requires a cold server load (~3 min). -""" -import json, time, urllib.request, urllib.error, subprocess, os, sys - -MODEL = "/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX/Qwen3.8-Flash-Next-Q4_0-ROCmFP4-STRIX.gguf" -SERVE = "/home/dank/GitPackages/ROCmFPX/build-qwen38/bin/llama-server" -PORT = 7770 -N_GEN = 256 -PROMPT_FILE = "/tmp/bench-prompts/prompt_code.txt" -ENV_FILE = os.path.join(os.path.dirname(__file__), "frank-dragoon.env") -SERVER_DIR = os.path.dirname(SERVE) -ROCM_PATH = "/opt/rocm" - -def make_env(): - """Build the same env as serve-frank-dragoon.sh.""" - env = {**os.environ, "HSA_OVERRIDE_GFX_VERSION": "11.5.1", "HIP_VISIBLE_DEVICES": "0"} - - # Source env file for tunables - if os.path.exists(ENV_FILE): - with open(ENV_FILE) as f: - for line in f: - line = line.strip() - if line and not line.startswith("#") and "=" in line: - k, v = line.split("=", 1) - env[k] = v - - # Build LD_LIBRARY_PATH - lib_path = f"{SERVER_DIR}:{ROCM_PATH}/lib:{ROCM_PATH}/lib/rocm_sysdeps/lib:{ROCM_PATH}/lib/llvm/lib" - if os.environ.get("LD_LIBRARY_PATH"): - lib_path += f":{os.environ['LD_LIBRARY_PATH']}" - env["LD_LIBRARY_PATH"] = lib_path - env["ROCM_PATH"] = ROCM_PATH - - return env - -ENV = make_env() - -def kill_all(): - subprocess.run(["pkill", "-9", "llama-server"], capture_output=True) - time.sleep(3) - -def wait_ready(): - for i in range(600): - try: - r = urllib.request.urlopen(f"http://127.0.0.1:{PORT}/health", timeout=2) - if json.loads(r.read()).get("status") == "ok": - return True - except: - pass - time.sleep(1) - return False - -def send_request(prompt, max_tokens=N_GEN): - """Send chat completion, measure generation throughput.""" - body = json.dumps({ - "model": "frank-dragoon", - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "stream": False, - "temperature": 0.7, - "top_p": 0.8 - }).encode() - - t0 = time.time() - req = urllib.request.Request( - f"http://127.0.0.1:{PORT}/v1/chat/completions", - data=body, - headers={"Content-Type": "application/json"}, - method="POST" - ) - r = urllib.request.urlopen(req, timeout=1200) - resp = json.loads(r.read()) - wall = time.time() - t0 - - usage = resp.get("usage", {}) - pt = usage.get("prompt_tokens", 0) - gt = usage.get("completion_tokens", 0) - gtps = gt / wall if wall > 0 else 0 - return pt, gt, wall, gtps - -def load_and_extend_prompt(): - """Load prompt and extend to ~8K tokens if needed.""" - with open(PROMPT_FILE) as f: - text = f.read() - - # Count approximate tokens (rough: ~4 chars per token for code) - approx_tokens = len(text) / 4.0 - needed = 8200 - - if approx_tokens < needed: - repeats = needed // max(int(approx_tokens), 1) + 1 - text = (text + "\n") * repeats - words = text.split()[:needed] - text = " ".join(words) - - return text - -def main(): - tb_values = [int(x) for x in sys.argv[1:]] if len(sys.argv) > 1 else [16, 32, 64, 128] - results = [] - - prompt = load_and_extend_prompt() - n_words = len(prompt.split()) - print(f"Prompt: {n_words} words (~{n_words} tokens), {len(prompt)} chars\n", flush=True) - - for tb in tb_values: - print(f"\n{'='*60}") - print(f"threads-batch = {tb}") - print(f"{'='*60}", flush=True) - - kill_all() - print(" Starting server...", flush=True) - - lib_path = ENV.get("LD_LIBRARY_PATH", "") - cmd = [ - "/usr/bin/env", - "HSA_OVERRIDE_GFX_VERSION=11.5.1", - "HIP_VISIBLE_DEVICES=0", - f"LD_LIBRARY_PATH={lib_path}", - f"ROCM_PATH={ROCM_PATH}", - SERVE, "--model", MODEL, "--port", str(PORT), "--jinja", - "--device", "ROCm0", "--gpu-layers", "999", "--fit", "off", - "--threads", "16", "--threads-batch", str(tb), - "--parallel", "2", "--ctx-size", "131072", - "--batch-size", "4096", "--ubatch-size", "4096", - "--flash-attn", "on", - "--cache-type-k", "f16", "--cache-type-v", "f16", - "--cache-prompt", "--cache-idle-slots", "--metrics" - ] - - srv = subprocess.Popen(cmd, env=ENV, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) - - ready = wait_ready() - if not ready: - print(" FAILED: server did not become ready", flush=True) - srv.terminate() - srv.wait() - for line in (srv.stdout.readlines() or [])[-20:]: - print(f" LOG: {line.rstrip()}", flush=True) - continue - - print(" Server ready.", flush=True) - - # Warmup (small request) - try: - send_request("Hi", 16) - except Exception as e: - print(f" Warmup failed: {e}", flush=True) - srv.terminate() - srv.wait() - continue - - # Benchmark - print(f" Generating {N_GEN} tokens...", flush=True) - try: - pt, gt, wall, gtps = send_request(prompt, N_GEN) - print(f" {gt}/{N_GEN} gen tokens in {wall:.1f}s = {gtps:.1f} tok/s") - print(f" Prompt tokens: {pt}") - results.append({"tb": tb, "wall": wall, "gtps": gtps, "gt": gt, "pt": pt}) - except Exception as e: - print(f" FAILED: {e}", flush=True) - - srv.terminate() - srv.wait() - time.sleep(2) - - # Summary - print(f"\n{'='*60}") - print("SUMMARY") - print(f"{'='*60}") - for r in results: - print(f" tb={r['tb']:>3}: {r['gt']} gen tokens, {r['wall']:.1f}s, {r['gtps']:.1f} tok/s") - - if results: - best = max(results, key=lambda r: r['gtps']) - print(f"\nBest: tb={best['tb']} at {best['gtps']:.1f} tok/s") - - with open("/tmp/bench_decode_results.txt", "w") as f: - f.write("tb,gen_tokens,wall_s,gen_tps,prompt_tokens\n") - for r in results: - f.write(f"{r['tb']},{r['gt']},{r['wall']:.1f},{r['gtps']:.1f},{r['pt']}\n") - - print(f"\nResults: /tmp/bench_decode_results.txt") - -if __name__ == "__main__": - main() diff --git a/scripts/.AGENTTRASH/bench-ngram-mod.py.ik-llama.cpp b/scripts/.AGENTTRASH/bench-ngram-mod.py.ik-llama.cpp deleted file mode 100755 index d60f1e5..0000000 --- a/scripts/.AGENTTRASH/bench-ngram-mod.py.ik-llama.cpp +++ /dev/null @@ -1,268 +0,0 @@ -#!/usr/bin/env python3 -""" -Benchmark n-gram speculative decoding vs no-spec for frank-dragoon. - -Tests prompt-processing throughput with 16K token prompts (manageable request size). -Code-heavy prompt (high n-gram redundancy) vs prose prompt (low redundancy). - -Usage: - # Generate prompts: python3 scripts/bench-ngram-mod.py generate /tmp/prompts/ - # Run: python3 scripts/bench-ngram-mod.py /tmp/prompts/ -""" - -import json -import os -import sys -import time -import subprocess -import urllib.request -import urllib.error -from pathlib import Path - -MODEL = "/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX/Qwen3.8-Flash-Next-Q4_0-ROCmFP4-STRIX.gguf" -CHAT_TEMPLATE = "/home/dank/serverAppConfigs/scripts/chat-template.jinja" -BUILD_DIR = "/home/dank/GitPackages/ROCmFPX/build-qwen38/bin" -LLAMA_BIN = os.path.join(BUILD_DIR, "llama-server") - -# 16K tokens ~= 64KB text -TARGET_PROMPT_TOKENS = 16000 -TOKEN_CHAR_RATIO = 4 -TARGET_PROMPT_BYTES = TARGET_PROMPT_TOKENS * TOKEN_CHAR_RATIO - -CONTEXT = 65536 -BATCH = 4096 -UBATCH = 4096 -THREADS = 16 -THREAD_BATCH = 32 -NGRAM_N_MAX = 24 -NGRAM_N_MIN = 4 -TEST_TOKENS = 128 -SERVER_WAIT = 300 - - -def start_server(port, spec_type): - cmd = [ - LLAMA_BIN, - "--model", MODEL, - "--chat-template-file", CHAT_TEMPLATE, - "--jinja", - "--device", "ROCm0", - "--gpu-layers", "999", - "--parallel", "1", - "--ctx-size", str(CONTEXT), - "--batch-size", str(BATCH), - "--ubatch-size", str(UBATCH), - "--fit", "off", - "--flash-attn", "on", - "--cache-type-k", "f16", - "--cache-type-v", "f16", - "--threads", str(THREADS), - "--threads-batch", str(THREAD_BATCH), - "--cache-prompt", - "--cache-idle-slots", - "--metrics", - "--host", "127.0.0.1", - "--port", str(port), - "--temp", "0", - ] - if spec_type == "ngram-mod": - cmd.extend([ - "--spec-type", "ngram-mod", - "--spec-ngram-mod-n-max", str(NGRAM_N_MAX), - "--spec-ngram-mod-n-min", str(NGRAM_N_MIN), - ]) - else: - cmd.extend(["--spec-type", "none"]) - - env = os.environ.copy() - env["ROCM_PATH"] = "/opt/rocm" - env["LD_LIBRARY_PATH"] = f"{BUILD_DIR}:/opt/rocm/lib:/opt/rocm/lib/rocm_sysdeps/lib:/opt/rocm/lib/llvm/lib" - env["HSA_OVERRIDE_GFX_VERSION"] = "11.5.1" - env["HIP_VISIBLE_DEVICES"] = "0" - - proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - print(f" Waiting for server on port {port} (spec={spec_type})...") - for i in range(SERVER_WAIT): - try: - req = urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=2) - resp = json.loads(req.read().decode()) - if resp.get("status") == "ok": - print(f" Server ready after {i+1}s") - return proc - except: - time.sleep(1) - - proc.terminate() - out, err = proc.communicate() - print(f"ERROR: Server failed to start in {SERVER_WAIT}s") - print(err.decode()[-2000:]) - sys.exit(1) - - -def run_test(prompt_text, spec_type, port): - """Send a request and extract prompt processing metrics.""" - data = json.dumps({ - "model": "frank-dragoon", - "messages": [{"role": "user", "content": prompt_text}], - "max_tokens": TEST_TOKENS, - "stream": False, - "temperature": 0, - }).encode() - - req = urllib.request.Request( - f"http://127.0.0.1:{port}/v1/chat/completions", - data=data, - headers={"Content-Type": "application/json"}, - ) - - start = time.monotonic() - resp = urllib.request.urlopen(req, timeout=600) - elapsed = time.monotonic() - start - - result = json.loads(resp.read().decode()) - timings = result.get("timings", {}) - usage = result.get("usage", {}) - - return { - "prompt_tokens": usage.get("prompt_tokens", 0), - "prompt_ms": timings.get("prompt_ms", 0), - "prompt_per_second": timings.get("prompt_per_second", 0), - "prompt_per_token_ms": timings.get("prompt_per_token_ms", 0), - "predicted_per_second": timings.get("predicted_per_second", 0), - "total_time_s": elapsed, - "cache_n": timings.get("cache_n", 0), - } - - -def main(): - if len(sys.argv) < 2: - print(f"Usage: {sys.argv[0]} ") - sys.exit(1) - - prompt_dir = Path(sys.argv[1]) - prompt_dir.mkdir(parents=True, exist_ok=True) - - code_prompt_file = prompt_dir / "prompt_code.txt" - prose_prompt_file = prompt_dir / "prompt_prose.txt" - - # Generate prompts if they don't exist - if not code_prompt_file.exists() or not prose_prompt_file.exists(): - print("Generating prompts...") - # Generate deterministic prompts - # Code prompt: structured algorithms + data generation + config schemas - code_lines = [] - for i in range(50): - code_lines.append(f"# Algorithm {i}\n") - code_lines.append(f"def algo_{i}(x, y):\n") - code_lines.append(f" \"\"\"Algorithm {i} processing.\"\"\"\n") - code_lines.append(f" result = 0\n") - code_lines.append(f" for j in range({i * 10}):\n") - code_lines.append(f" result += x * y + j\n") - code_lines.append(f" return result\n\n") - code_text = "".join(code_lines) - # Pad/truncate to target - while len(code_text) < TARGET_PROMPT_BYTES: - code_text += "# Additional algorithm block\n" - code_text += f"def extra_{len(code_text)//100}():\n" - code_text += " x = 1\n" - code_text += " y = 2\n" - code_text += " return x + y\n\n" - code_text = code_text[:TARGET_PROMPT_BYTES] - - # Prose prompt: historical/scientific/philosophical text - prose_text = "" - topics = [ - "The Renaissance brought profound transformation to European civilization. Artists and scholars pushed the boundaries of human expression.", - "Quantum mechanics describes the behavior of matter and energy at atomic scales. The uncertainty principle limits measurement precision.", - "Ethical philosophy grapples with questions of right and wrong. Utilitarianism judges actions by their consequences and the happiness they produce.", - "Mountains rise like ancient sentinels, their peaks lost in clouds that swirl and dance across the sky above the valleys below.", - "The ocean whispers secrets to the shore, each wave carrying fragments of distant lands across the water before retreating back to sea.", - ] - for _ in range(500): - prose_text += f"{topics[len(prose_text) % len(topics)]} " - prose_text = prose_text[:TARGET_PROMPT_BYTES] - - code_prompt_file.write_text(code_text) - prose_prompt_file.write_text(prose_text) - print(f" Code: {code_prompt_file.stat().st_size:,} bytes") - print(f" Prose: {prose_prompt_file.stat().st_size:,} bytes") - - print("=" * 60) - print(" N-gram Mod Benchmark — frank-dragoon") - print(f" STRIX ROCmFP4 · 64K context · 16K prompts") - print("=" * 60) - - results = [] - port_base = 7772 - port_ngram = 7773 - - tests = [ - ("code", "none", port_base, code_prompt_file), - ("prose", "none", port_ngram, prose_prompt_file), - ("code", "ngram-mod", port_base, code_prompt_file), - ("prose", "ngram-mod", port_ngram, prose_prompt_file), - ] - - for prompt_type, spec_type, port, pfile in tests: - print(f"\n=== {spec_type} ({prompt_type}) ===") - print(f" File: {pfile} ({pfile.stat().st_size:,} bytes)") - print(f" Spec: {spec_type}") - if spec_type == "ngram-mod": - print(f" n_max={NGRAM_N_MAX}, n_min={NGRAM_N_MIN}") - - proc = start_server(port, spec_type) - - try: - # Warm-up - print(" Warm-up...") - run_test("Hello test", spec_type, port) - - # Full prompt - print(f" Sending {TARGET_PROMPT_TOKENS} token prompt...") - with open(pfile) as f: - prompt_text = f.read() - result = run_test(prompt_text, spec_type, port) - - tps = result["prompt_per_second"] - print(f" Tokens: {result['prompt_tokens']}") - print(f" Prompt: {result['prompt_ms']:.0f}ms") - print(f" TPS: {tps:.1f} tok/s") - print(f" Total: {result['total_time_s']:.1f}s") - - results.append({ - "prompt_type": prompt_type, - "spec_type": spec_type, - "prompt_tokens": result["prompt_tokens"], - "prompt_ms": result["prompt_ms"], - "tps": tps, - "total_time_s": result["total_time_s"], - }) - - finally: - proc.terminate() - try: - proc.wait(timeout=5) - except: - proc.kill() - - # Summary - print("\n" + "=" * 60) - print(" RESULTS SUMMARY") - print("=" * 60) - - for ptype in ["code", "prose"]: - baseline = next((r for r in results if r["prompt_type"] == ptype and r["spec_type"] == "none"), None) - ngram = next((r for r in results if r["prompt_type"] == ptype and r["spec_type"] == "ngram-mod"), None) - if baseline and ngram: - gain = ((ngram["tps"] - baseline["tps"]) / baseline["tps"]) * 100 if baseline["tps"] > 0 else 0 - print(f"\n{ptype.upper()} PROMPT:") - print(f" Baseline TPS: {baseline['tps']:.1f} tok/s ({baseline['prompt_ms']:.0f}ms)") - print(f" N-gram TPS: {ngram['tps']:.1f} tok/s ({ngram['prompt_ms']:.0f}ms)") - print(f" Speedup: {gain:+.1f}%") - - print("\n" + "=" * 60) - - -if __name__ == "__main__": - main() diff --git a/scripts/.AGENTTRASH/bench-ngram-mod.sh.ik-llama.cpp b/scripts/.AGENTTRASH/bench-ngram-mod.sh.ik-llama.cpp deleted file mode 100755 index dc5a512..0000000 --- a/scripts/.AGENTTRASH/bench-ngram-mod.sh.ik-llama.cpp +++ /dev/null @@ -1,304 +0,0 @@ -#!/usr/bin/env bash -# Benchmark n-gram speculative decoding vs no-spec for frank-dragoon. -# -# This tests prompt-processing throughput at 64k context with two distinct -# prompts: one code-heavy and one prose-heavy. Uses llama-server on scratch -# ports with the OpenAI-compatible API to measure prompt_eval_duration_ms. -# -# Run on Dankclaw after stopping the production service: -# sudo systemctl stop frank-dragoon -# ./scripts/bench-ngram-mod.sh -# sudo systemctl start frank-dragoon - -set -euo pipefail - -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}" -if [[ -f "$ENV_FILE" ]]; then - set -a; source "$ENV_FILE"; set +a -fi - -: "${LLAMA_BIN:=/home/dank/GitPackages/ROCmFPX/build-qwen38/bin/llama-server}" -: "${MODEL_DIR:=/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX}" -: "${MODEL_FILE:=Qwen3.8-Flash-Next-Q4_0-ROCmFP4-STRIX.gguf}" -: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}" -: "${CHAT_TEMPLATE_KWARGS:={}}" -: "${BATCH_SIZE:=4096}" -: "${UBATCH_SIZE:=4096}" -: "${CTX_SIZE:=65536}" -: "${NUM_THREADS:=16}" -: "${THREAD_BATCH:=32}" -: "${CACHE_TYPE_K:=f16}" -: "${CACHE_TYPE_V:=f16}" -: "${FIT:=off}" -: "${TEMPERATURE:=0.7}" -: "${TOP_P:=0.8}" -: "${TOP_K:=20}" -: "${MIN_P:=0.0}" -: "${REASONING:=on}" -: "${REASONING_FORMAT:=deepseek}" - -MODEL="$MODEL_DIR/$MODEL_FILE" -HOSTNAME=$(hostname 2>/dev/null || echo "dankclaw") - -# Prompt files (generated by generate-prompts.py or passed via args) -PROMPT_CODE="${1:-}" -PROMPT_PROSE="${2:-}" - -if [[ -z "$PROMPT_CODE" || -z "$PROMPT_PROSE" ]]; then - echo "ERROR: Need two prompt file paths." - echo "Usage: $0 " - echo "Generate with: python3 scripts/generate-prompts.py" - exit 1 -fi - -if [[ ! -r "$MODEL" ]]; then - echo "missing or unreadable model: $MODEL" >&2 - exit 1 -fi - -SERVER_DIR="$(cd -- "$(dirname -- "$LLAMA_BIN")" && pwd)" -ROCM_PATH="${ROCM_PATH:-/opt/rocm}" -LIBRARY_PATH="$SERVER_DIR:$ROCM_PATH/lib:$ROCM_PATH/lib/rocm_sysdeps/lib:$ROCM_PATH/lib/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - -# Scratch ports -PORT_NGRAM=7771 -PORT_BASELINE=7772 - -cleanup() { - echo "Cleaning up servers..." - # Kill any leftover llama-server on our ports - for port in $PORT_NGRAM $PORT_BASELINE; do - fuser -k "$port"/tcp 2>/dev/null || true - done -} -trap cleanup EXIT - -wait_for_server() { - local port=$1 - local max_wait=120 - local elapsed=0 - while (( elapsed < max_wait )); do - if curl -s "http://127.0.0.1:$port/health" | grep -q '"running"'; then - return 0 - fi - sleep 1 - (( elapsed++ )) - done - echo "ERROR: Server on port $port did not start within ${max_wait}s" >&2 - return 1 -} - -get_prompt_tokens() { - local file=$1 - # Count tokens approximately (1 token ~ 4 chars for English/code) - wc -c < "$file" | awk '{printf "%d", $1/4}' -} - -run_single_test() { - local port=$1 - local spec_type="$2" - local prompt_file=$3 - local label=$4 - local ngram_n_max="${5:-24}" - local ngram_n_min="${6:-4}" - - echo "=== $label ===" - echo " spec-type: $spec_type" - echo " ngram-mod n-max: $ngram_n_max (if enabled)" - echo " ngram-mod n-min: $ngram_n_min (if enabled)" - echo " prompt: $prompt_file ($(wc -c < "$prompt_file") bytes)" - - # Build command - local cmd=( - /usr/bin/env - ROCM_PATH="$ROCM_PATH" - HIP_VISIBLE_DEVICES=0 - LD_LIBRARY_PATH="$LIBRARY_PATH" - HSA_OVERRIDE_GFX_VERSION="${HSA_OVERRIDE_GFX_VERSION:-11.5.1}" - "$LLAMA_BIN" - --model "$MODEL" - --chat-template-file "$CHAT_TEMPLATE_FILE" - --chat-template-kwargs "$CHAT_TEMPLATE_KWARGS" - --host 127.0.0.1 - --port "$port" - --jinja - --device ROCm0 - --gpu-layers 999 - --parallel 1 - --ctx-size "$CTX_SIZE" - --batch-size "$BATCH_SIZE" - --ubatch-size "$UBATCH_SIZE" - --fit "$FIT" - --flash-attn on - --cache-type-k "$CACHE_TYPE_K" - --cache-type-v "$CACHE_TYPE_V" - --threads "$NUM_THREADS" - --threads-batch "$THREAD_BATCH" - --cache-prompt - --cache-idle-slots - --metrics - --temp "$TEMPERATURE" - --top-p "$TOP_P" - --top-k "$TOP_K" - --min-p "$MIN_P" - --reasoning "$REASONING" - --reasoning-format "$REASONING_FORMAT" - --reasoning-budget -1 - ) - - # Add spec flags - if [[ "$spec_type" == "ngram-mod" ]]; then - cmd+=( - --spec-type "ngram-mod" - --spec-ngram-mod-n-max "$ngram_n_max" - --spec-ngram-mod-n-min "$ngram_n_min" - ) - else - cmd+=( - --spec-type "none" - ) - fi - - # Start server with stderr to log - local log_file="/tmp/ngram-bench-${label//[^a-zA-Z0-9]/-}.log" - echo " Starting server (log: $log_file)..." - - "${cmd[@]}" 2> "$log_file" & - local server_pid=$! - - if ! wait_for_server "$port"; then - echo "ERROR: Server failed to start" >&2 - kill "$server_pid" 2>/dev/null || true - return 1 - fi - - echo " Server ready." - - # Send prompt via API - echo " Sending prompt..." - local prompt_text - prompt_text=$(cat "$prompt_file") - - local response - response=$(curl -s -X POST "http://127.0.0.1:$port/v1/completions" \ - -H "Content-Type: application/json" \ - -d "{ - \"model\": \"frank-dragoon\", - \"prompt\": \"$prompt_text\", - \"max_tokens\": 128, - \"temperature\": 0, - \"stream\": false - }") - - # Extract metrics - local prompt_tokens prompt_eval_duration_ms total_duration_ms - prompt_tokens=$(echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['prompt_eval_count'])" 2>/dev/null || echo "0") - prompt_eval_duration_ms=$(echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['prompt_eval_duration_ms'])" 2>/dev/null || echo "0") - total_duration_ms=$(echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['total_duration_ms'])" 2>/dev/null || echo "0") - - # Calculate throughput - local prompt_tps=0 - if [[ "$prompt_eval_duration_ms" -gt 0 ]]; then - prompt_tps=$(python3 -c "print(f'{($prompt_tokens / $prompt_eval_duration_ms) * 1000:.1f}')") - fi - - echo " Prompt tokens: $prompt_tokens" - echo " Prompt eval: ${prompt_eval_duration_ms}ms" - echo " Prompt TPS: $prompt_tps tok/s" - echo " Total duration: ${total_duration_ms}ms" - - # Kill server - kill "$server_pid" 2>/dev/null || true - sleep 2 - - echo " Done." - echo "---" - - # Output structured results for parsing - echo "RESULT|$label|$prompt_file|$prompt_tokens|$prompt_eval_duration_ms|$total_duration_ms|$prompt_tps" -} - -echo "╔══════════════════════════════════════════════════════════╗" -echo "║ N-gram Mod Benchmark — frank-dragoon ║" -echo "║ STRIX ROCmFP4 · 64k context · two prompts ║" -echo "╚══════════════════════════════════════════════════════════╝" -echo "" -echo "Host: $HOSTNAME" -echo "Model: $MODEL" -echo "Build: $(basename "$LLAMA_BIN")" -echo "Context: $CTX_SIZE" -echo "" - -# Run baseline (no spec) -echo "=== BASELINE (no speculative decoding) ===" -echo "" - -BASELINE_CODE_RESULT=$(run_single_test $PORT_BASELINE "none" "$PROMPT_CODE" "Baseline-Code" 24 4) -BASELINE_PROSE_RESULT=$(run_single_test $PORT_NGRAM "none" "$PROMPT_PROSE" "Baseline-Prose" 24 4) - -echo "" -echo "=== NGRAM-MOD (speculative decoding enabled) ===" -echo "" - -# Run n-gram mod -NGRAM_CODE_RESULT=$(run_single_test $PORT_NGRAM "ngram-mod" "$PROMPT_CODE" "Ngram-Code" 24 4) -NGRAM_PROSE_RESULT=$(run_single_test $PORT_BASELINE "ngram-mod" "$PROMPT_PROSE" "Ngram-Prose" 24 4) - -echo "" -echo "╔══════════════════════════════════════════════════════════╗" -echo "║ RESULTS SUMMARY ║" -echo "╚══════════════════════════════════════════════════════════╝" -echo "" - -# Parse and display results -python3 << 'PYEOF' -import sys - -results = [] -for line in sys.stdin: - line = line.strip() - if not line.startswith("RESULT|"): - continue - parts = line.split("|") - if len(parts) != 8: - continue - label, prompt_file, prompt_tokens, prompt_eval_ms, total_ms, tps = parts[1], parts[2], int(parts[3]), int(parts[4]), int(parts[5]), float(parts[6]) - results.append({ - 'label': label, - 'prompt_file': prompt_file, - 'prompt_tokens': prompt_tokens, - 'prompt_eval_ms': prompt_eval_ms, - 'total_ms': total_ms, - 'tps': tps - }) - -# Compare baseline vs ngram for each prompt type -for code_result in results: - if not code_result['label'].startswith('Ngram-'): - continue - prompt_type = code_result['label'].replace('Ngram-', '').replace('Ngram', 'Baseline').replace('-', '-') - baseline = next((r for r in results if r['label'] == 'Baseline-' + prompt_type), None) - if not baseline: - continue - - spec = code_result['label'] == 'Ngram-' + prompt_type - - print(f"Prompt: {prompt_type}") - print(f" Tokens: {code_result['prompt_tokens']}") - if spec: - print(f" Spec: ngram-mod (n_max=24, n_min=4)") - print(f" Prompt: {code_result['prompt_eval_ms']}ms | {code_result['tps']:>10.1f} tok/s") - baseline_tps = baseline['tps'] - if baseline_tps > 0: - improvement = ((code_result['tps'] - baseline_tps) / baseline_tps) * 100 - print(f" Speedup: {improvement:+.1f}% vs baseline ({baseline_tps:.1f} tok/s)") - else: - print(f" Spec: none (baseline)") - print(f" Prompt: {baseline['prompt_eval_ms']}ms | {baseline['tps']:>10.1f} tok/s") - print() -PYEOF - -echo "" -echo "Raw data lines above (RESULT|... format) can be parsed for automation." -echo "" diff --git a/scripts/.AGENTTRASH/bench-spec-decode.py.ik-llama.cpp b/scripts/.AGENTTRASH/bench-spec-decode.py.ik-llama.cpp deleted file mode 100755 index 68f3313..0000000 --- a/scripts/.AGENTTRASH/bench-spec-decode.py.ik-llama.cpp +++ /dev/null @@ -1,229 +0,0 @@ -#!/usr/bin/env python3 -"""Benchmark decode throughput across speculative decoding configs. - -Sweeps 4 configs: none, ngram-mod, draft-mtp, draft-mtp+ngram-mod. -Each config generates 16k tokens from a minimal prompt at temp 0. - -Runs ON the target host (Dankclaw). Service management uses local sudo. -Usage: - python3 bench-spec-decode.py [--max-tokens N] [--host H] [--port P] -""" -import json -import subprocess -import sys -import time -import urllib.request -import urllib.error - -SERVICE = "frank-dragoon" -ENV_FILE = "/home/dank/serverAppConfigs/scripts/frank-dragoon.env" -ALIAS = "frank-dragoon" -DEFAULT_HOST = "localhost" -DEFAULT_PORT = 7770 -DEFAULT_MAX_TOKENS = 16384 -HEALTH_TIMEOUT = 1200 # 20 min for cold load -HEALTH_INTERVAL = 5 - -PROMPT = ( - "Write a comprehensive, detailed essay about the complete history of " - "computing, from the earliest mechanical calculators through modern AI. " - "Cover every major era in detail, including key figures, machines, and " - "breakthroughs. Be as thorough as possible." -) - -CONFIGS = [ - ("none", "none"), - ("ngram-mod", "ngram-mod"), - ("draft-mtp", "draft-mtp"), - ("draft-mtp+ngram-mod", "draft-mtp,ngram-mod"), -] - - -def local(cmd): - """Run a command locally (the script runs on the target host).""" - r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=120) - if r.returncode != 0: - print(f" WARNING: {cmd} -> exit {r.returncode}: {r.stderr.strip()}", - file=sys.stderr) - return r - - -def sed_spec_type(value): - local(f"sudo sed -i 's/^SPEC_TYPE=.*/SPEC_TYPE={value}/' {ENV_FILE}") - - -def wait_health(host, port, timeout=HEALTH_TIMEOUT): - start = time.time() - while time.time() - start < timeout: - try: - resp = urllib.request.urlopen( - f"http://{host}:{port}/health", timeout=10 - ) - if resp.status == 200: - return True - except (urllib.error.URLError, urllib.error.HTTPError, ConnectionError, - OSError): - pass - time.sleep(HEALTH_INTERVAL) - return False - - -def benchmark(host, port, alias, max_tokens): - body = json.dumps({ - "model": alias, - "messages": [{"role": "user", "content": PROMPT}], - "max_tokens": max_tokens, - "temperature": 0, - "stream": False, - }).encode() - req = urllib.request.Request( - f"http://{host}:{port}/v1/chat/completions", - data=body, - headers={"Content-Type": "application/json"}, - ) - t0 = time.time() - resp = urllib.request.urlopen(req, timeout=1200) - wall = time.time() - t0 - data = json.loads(resp.read()) - return data, wall - - -def main(): - import argparse - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--host", default=DEFAULT_HOST) - ap.add_argument("--port", type=int, default=DEFAULT_PORT) - ap.add_argument("--max-tokens", type=int, default=DEFAULT_MAX_TOKENS) - args = ap.parse_args() - - # Read original SPEC_TYPE from the env file - r = local(f"grep '^SPEC_TYPE=' {ENV_FILE}") - original_spec = r.stdout.strip() - orig_val = (original_spec.split("=", 1)[1] - if "=" in original_spec else "draft-mtp,ngram-mod") - print(f"Original: {original_spec}") - print(f"Max tokens: {args.max_tokens}") - print(f"Configs: {[c[0] for c in CONFIGS]}") - print() - - # Stop + mask the production service - print("Stopping + masking production service...") - local(f"sudo systemctl stop {SERVICE}") - local(f"sudo systemctl mask {SERVICE}") - time.sleep(2) - - results = [] - for name, spec_type in CONFIGS: - print(f"\n{'='*60}") - print(f"Config: {name} (SPEC_TYPE={spec_type})") - print(f"{'='*60}") - - # Set SPEC_TYPE - sed_spec_type(spec_type) - print(f"Set SPEC_TYPE={spec_type}") - - # Start the service - local(f"sudo systemctl unmask {SERVICE}") - local(f"sudo systemctl start {SERVICE}") - print("Starting service...") - - # Wait for health - print(f"Waiting for health (up to {HEALTH_TIMEOUT}s)...") - if not wait_health(args.host, args.port): - print(f"FAILED: service did not become healthy for {name}") - results.append({ - "config": name, "spec_type": spec_type, - "error": "health timeout", - }) - local(f"sudo systemctl stop {SERVICE}") - local(f"sudo systemctl mask {SERVICE}") - continue - - print("Healthy. Running benchmark...") - - # Run benchmark - try: - data, wall = benchmark( - args.host, args.port, ALIAS, args.max_tokens - ) - except Exception as e: - print(f"FAILED: {e}") - results.append({ - "config": name, "spec_type": spec_type, - "error": str(e), - }) - local(f"sudo systemctl stop {SERVICE}") - local(f"sudo systemctl mask {SERVICE}") - continue - - timings = data.get("timings", {}) - usage = data.get("usage", {}) - pred_n = timings.get("predicted_n", usage.get("completion_tokens", 0)) - pred_ms = timings.get("predicted_ms", 0) - pred_ps = timings.get("predicted_per_second", 0) - prompt_n = timings.get("prompt_n", usage.get("prompt_tokens", 0)) - prompt_ms = timings.get("prompt_ms", 0) - - print(f" Generated: {pred_n} tokens") - print(f" Wall time: {wall:.1f}s") - print(f" Decode: {pred_ps:.2f} tok/s") - print(f" Prefill: {prompt_n} tokens in {prompt_ms:.0f}ms" - f" ({prompt_n / max(prompt_ms, 1) * 1000:.1f} tok/s)") - - results.append({ - "config": name, - "spec_type": spec_type, - "predicted_tokens": pred_n, - "predicted_ms": pred_ms, - "predicted_per_second": pred_ps, - "prompt_tokens": prompt_n, - "prompt_ms": prompt_ms, - "wall_time": wall, - }) - - # Stop + mask between configs - local(f"sudo systemctl stop {SERVICE}") - local(f"sudo systemctl mask {SERVICE}") - time.sleep(2) - - # Restore original SPEC_TYPE - sed_spec_type(orig_val) - print(f"\nRestoring SPEC_TYPE={orig_val}") - print("Unmasking + starting production service...") - local(f"sudo systemctl unmask {SERVICE}") - local(f"sudo systemctl start {SERVICE}") - - # Health check - if wait_health(args.host, args.port): - print("Production service healthy.") - else: - print("WARNING: production service not healthy after restore!") - - # Print results table - print(f"\n{'='*70}") - print("RESULTS") - print(f"{'='*70}") - print(f"{'Config':<30} {'Tokens':>8} {'Wall(s)':>10} {'Decode tok/s':>14} " - f"{'Error':>20}") - print(f"{'-'*30} {'-'*8} {'-'*10} {'-'*14} {'-'*20}") - for r in results: - if "error" in r: - print(f"{r['config']:<30} {'':>8} {'':>10} {'':>14} " - f"{r['error']:>20}") - else: - print(f"{r['config']:<30} {r['predicted_tokens']:>8} " - f"{r['wall_time']:>10.1f} {r['predicted_per_second']:>14.2f} " - f"{'':>20}") - print(f"{'='*70}") - - # Save JSON results - ts = time.strftime("%Y%m%d-%H%M%S") - out = f"/tmp/bench-spec-decode-{ts}.json" - with open(out, "w") as f: - json.dump({"results": results, "max_tokens": args.max_tokens, - "timestamp": ts}, f, indent=2) - print(f"Results saved: {out}") - - -if __name__ == "__main__": - main() diff --git a/scripts/.AGENTTRASH/bench-vllm-corpus.py b/scripts/.AGENTTRASH/bench-vllm-corpus.py deleted file mode 100755 index df0e7cf..0000000 --- a/scripts/.AGENTTRASH/bench-vllm-corpus.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python3 -"""bench-vllm-corpus.py — build a code-shaped benchmark corpus from the repo. - -Why this exists: `vllm bench serve --dataset-name random` produces synthetic -token soup, and speculative decoding accepts it nothing like real code — measured -on frank-medium, random prompts accepted 2.9-3.6 tokens/update against the ~5.8 -the upstream reference reports on a code mix, which makes the random corpus -under-report throughput by a wide margin and would pick the wrong SPEC depth. - -The prompts are the shape agents actually send: a repository artifact plus an -instruction that requires reading it. Output length is fixed at bench time -(`--custom-output-len`), so the corpus carries prompts only. - -Usage: - bench-vllm-corpus.py [repo-root] [out.jsonl] - -Then feed it to the endpoint (see scripts/bench-vllm-serve.sh, which passes the -tokenizer the client needs and salts prompts so no run inherits a warm prefix -cache). Methodology and the numbers these corpora produced are in -serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md. -""" - -from __future__ import annotations - -import json -import pathlib -import sys - -REPO = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else "/home/dank/serverAppConfigs") -OUT = pathlib.Path(sys.argv[2] if len(sys.argv) > 2 else "/home/dank/bench/corpus-code.jsonl") - -TARGETS = [ - "scripts/deploy-remote.sh", - "scripts/deploy-plan.py", - "scripts/deploy.sh", - "dankserver/frank-medium/radiance-entrypoint.sh", - "dankserver/frank-medium/quadlet/frank-medium-a.container", - "dankserver/frank-medium/proxy/affinity-proxy.py", - "dankserver/frank-medium/proxy/routing.py", - "tests/frank_medium_quadlet.bats", - "tests/deployment_scoping.bats", - "dankclaw/hermes/quadlet/hermes.container", - "dankclaw/anubis/quadlet/anubis.container", - "shared/prometheus/prometheus.yml", - "shared/prometheus/rules/llm-fleet.yml", - "shared/loki/loki-config.yaml", - "dankclaw/compose.yaml", - "dankserver/copyparty/quadlet/copyparty.container", -] - -PROMPTS = [ - "Read this repository artifact and summarise what it does, then list every " - "assumption it makes about the host it runs on.\n\n```\n{body}\n```", - "Here is a file from our homelab repository. Identify any bug, race or " - "missing error path, and show the minimal patch that fixes it.\n\n```\n{body}\n```", - "Explain this configuration to a new operator, then rewrite it with " - "comments that state the constraint each line protects.\n\n```\n{body}\n```", - "This file changed in a recent deploy. Write the changelog entry and a test " - "that would fail if the change regressed.\n\n```\n{body}\n```", -] - -MAX_LINES = 220 - - -def main() -> int: - samples = [] - for i, rel in enumerate(TARGETS): - path = REPO / rel - if not path.is_file(): - continue - body = "\n".join(path.read_text(errors="replace").splitlines()[:MAX_LINES]) - samples.append({"prompt": PROMPTS[i % len(PROMPTS)].format(body=body)}) - if not samples: - print(f"no corpus sources found under {REPO}", file=sys.stderr) - return 1 - OUT.parent.mkdir(parents=True, exist_ok=True) - with OUT.open("w") as fh: - for sample in samples: - fh.write(json.dumps(sample) + "\n") - print(f"wrote {len(samples)} prompts to {OUT}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/.AGENTTRASH/bench-vllm-serve.sh b/scripts/.AGENTTRASH/bench-vllm-serve.sh deleted file mode 100755 index 97cae60..0000000 --- a/scripts/.AGENTTRASH/bench-vllm-serve.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# bench-vllm-serve.sh — concurrency sweep against a vLLM endpoint. -# -# Usage: bench-vllm-serve.sh [concurrency ...] -# e.g. bench-vllm-serve.sh baseline 7770 1 4 8 -# -# Runs `vllm bench serve` from inside the serving image (so the client matches -# the server build) through an OpenAI-compatible endpoint, on the code corpus -# scripts/bench-vllm-corpus.py writes. -# -# Two traps this script exists to encode, both of which produced silent garbage -# during the 2026-09-16 frank-medium work: -# -# 1. The tokenizer must be passed explicitly and /models mounted. With only -# `--model frank-medium` the client tries to resolve that alias on -# HuggingFace and dies before sending a request — a run that looks like a -# benchmark and reports nothing. -# 2. `--ignore-eos` is deliberate: it measures decode throughput at a fixed -# output length. Drop it when you care about real stop behaviour, but then -# do not compare the numbers with anything measured under it. -# -# Also note the corpora are code-shaped on purpose (see the generator's header): -# synthetic random prompts misreport speculative acceptance badly. -# -# Methodology, and the numbers these runs produced on 2x R9700, are in -# serverAppConfigs-docs/LLM/frank-medium-radiance-mxfp4.md. -set -u - -TAG=${1:?usage: bench-vllm-serve.sh [concurrency ...]} -PORT=${2:?usage: bench-vllm-serve.sh [concurrency ...]} -shift 2 -CONCS=("$@") -[ "${#CONCS[@]}" -eq 0 ] && CONCS=(1 4 8) - -IMG=${IMG:-localhost/radiance-vllm:0.9.3} -MODELS_DIR=${MODELS_DIR:-/home/dank/models} -TOK=${TOK:-/models/Qwen3.8-27B-Uncensored-MXFP4-awq} -BENCH_DIR=${BENCH_DIR:-/home/dank/bench} -CORPUS=${CORPUS:-$BENCH_DIR/corpus-code.jsonl} -OUT=${OUT:-$BENCH_DIR/$TAG} -mkdir -p "$OUT" - -if [ ! -f "$CORPUS" ]; then - echo "corpus $CORPUS is missing; run scripts/bench-vllm-corpus.py first" >&2 - exit 1 -fi - -for c in "${CONCS[@]}"; do - echo "== $TAG conc=$c (port $PORT)" - podman run --rm --network=host -v "$MODELS_DIR":/models:ro -v "$BENCH_DIR":/bench:ro \ - -e HF_HUB_OFFLINE=1 --entrypoint bash "$IMG" -lc " - vllm bench serve --backend openai-chat \ - --base-url http://127.0.0.1:$PORT --endpoint /v1/chat/completions \ - --model frank-medium --tokenizer $TOK \ - --dataset-name custom --dataset-path /bench/$(basename "$CORPUS") \ - --custom-output-len 512 --num-prompts 16 --max-concurrency $c \ - --disable-shuffle --seed 0 --ignore-eos \ - --percentile-metrics ttft,tpot --metric-percentiles 50,99 \ - --disable-tqdm 2>&1 | grep -E 'Successful requests|Benchmark duration|Output token throughput|Total Token throughput|Mean TTFT|P50 TTFT|P50 TPOT|Acceptance rate|Acceptance length' - " 2>&1 | sed 's/^/ /' - echo -done -echo "results for $TAG written under $OUT" diff --git a/scripts/.AGENTTRASH/chat-template.jinja.ik-llama-qwen-sharp b/scripts/.AGENTTRASH/chat-template.jinja.ik-llama-qwen-sharp deleted file mode 100644 index 0a2996e..0000000 --- a/scripts/.AGENTTRASH/chat-template.jinja.ik-llama-qwen-sharp +++ /dev/null @@ -1,453 +0,0 @@ -{%- set template_version = "qwen3.8-froggeric-v22.4.0" %} -{%- set _tool_format = tool_call_format if tool_call_format is defined else 'xml' %} -{%- set image_count = namespace(value=0) %} -{%- set video_count = namespace(value=0) %} -{%- set add_vision_id = add_vision_id if add_vision_id is defined else false %} -{%- set enable_thinking = enable_thinking if enable_thinking is defined else true %} -{%- set auto_disable_thinking_with_tools = auto_disable_thinking_with_tools if auto_disable_thinking_with_tools is defined else false %} -{%- if preserve_reasoning is defined and preserve_reasoning is not none %} - {%- set _preserve_thinking = preserve_reasoning %} -{%- elif preserve_thinking is defined and preserve_thinking is not none %} - {%- set _preserve_thinking = preserve_thinking %} -{%- else %} - {%- set _preserve_thinking = true %} -{%- endif %} -{%- set max_tool_arg_chars = max_tool_arg_chars if max_tool_arg_chars is defined else 0 %} -{%- set max_tool_response_chars = max_tool_response_chars if max_tool_response_chars is defined else 0 %} -{%- set _default_reasoning_effort = 'medium' %} -{%- set _has_tools = (tools is defined and tools and tools is iterable and tools is not mapping) %} -{%- set _effort_raw = (reasoning_effort | string | lower) if reasoning_effort is defined and reasoning_effort is not none else _default_reasoning_effort %} -{%- set _initial_thinking = enable_thinking %} -{%- if _effort_raw in ('none', 'off') %} - {%- set _initial_thinking = false %} - {%- set _initial_effort = 'medium' %} -{%- elif _effort_raw in ('minimal', 'low') %} - {%- set _initial_effort = 'low' %} -{%- elif _effort_raw in ('high', 'xhigh', 'max', 'ultracode', 'extreme') %} - {%- set _initial_effort = 'xhigh' %} -{%- else %} - {%- set _initial_effort = 'medium' %} -{%- endif %} -{%- set ns_state = namespace(thinking=_initial_thinking, effort=_initial_effort) %} -{%- if auto_disable_thinking_with_tools and _has_tools %} - {%- set ns_state.thinking = false %} -{%- endif %} -{%- for msg in messages %} - {%- if msg.role == 'system' or msg.role == 'developer' or msg.role == 'user' %} - {%- if msg.content is string %} - {%- if '<|think_off|>' in msg.content %} - {%- set ns_state.thinking = false %} - {%- elif '<|think_on|>' in msg.content %} - {%- set ns_state.thinking = true %} - {%- elif '<|think_xhigh|>' in msg.content or '<|think_high|>' in msg.content or '<|think_ultracode|>' in msg.content or '<|think_extreme|>' in msg.content or '<|think_max|>' in msg.content %} - {%- set ns_state.thinking = true %} - {%- set ns_state.effort = 'xhigh' %} - {%- elif '<|think_low|>' in msg.content or '<|think_minimal|>' in msg.content %} - {%- set ns_state.thinking = true %} - {%- set ns_state.effort = 'low' %} - {%- elif '<|think_medium|>' in msg.content %} - {%- set ns_state.thinking = true %} - {%- set ns_state.effort = 'medium' %} - {%- endif %} - {%- elif msg.content is iterable and msg.content is not mapping %} - {%- for item in msg.content %} - {%- if item is string %} - {%- set _item_text = item %} - {%- elif item is mapping and 'text' in item and item.text is string %} - {%- set _item_text = item.text %} - {%- else %} - {%- set _item_text = '' %} - {%- endif %} - {%- if _item_text %} - {%- if '<|think_off|>' in _item_text %} - {%- set ns_state.thinking = false %} - {%- elif '<|think_on|>' in _item_text %} - {%- set ns_state.thinking = true %} - {%- elif '<|think_xhigh|>' in _item_text or '<|think_high|>' in _item_text or '<|think_ultracode|>' in _item_text or '<|think_extreme|>' in _item_text or '<|think_max|>' in _item_text %} - {%- set ns_state.thinking = true %} - {%- set ns_state.effort = 'xhigh' %} - {%- elif '<|think_low|>' in _item_text or '<|think_minimal|>' in _item_text %} - {%- set ns_state.thinking = true %} - {%- set ns_state.effort = 'low' %} - {%- elif '<|think_medium|>' in _item_text %} - {%- set ns_state.thinking = true %} - {%- set ns_state.effort = 'medium' %} - {%- endif %} - {%- endif %} - {%- endfor %} - {%- endif %} - {%- endif %} -{%- endfor %} -{%- set reasoning_instructions = '' %} -{%- if ns_state.thinking %} - {%- if ns_state.effort == 'xhigh' %} - {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %} - {%- elif ns_state.effort == 'low' %} - {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %} - {%- endif %} -{%- endif %} -{%- macro render_content(content, do_vision_count, is_system_content=false) %} - {%- if content is string %} - {{- content }} - {%- elif content is iterable and content is not mapping %} - {%- for item in content %} - {%- if item is mapping %} - {%- if item.type == 'image' or 'image' in item or 'image_url' in item %} - {%- if is_system_content %} - {{- raise_exception('System message cannot contain images.') }} - {%- endif %} - {%- if do_vision_count %} - {%- set image_count.value = image_count.value + 1 %} - {%- endif %} - {%- if add_vision_id %} - {{- 'Picture ' ~ image_count.value ~ ': ' }} - {%- endif %} - {{- '<|vision_start|><|image_pad|><|vision_end|>' }} - {%- elif item.type == 'video' or 'video' in item %} - {%- if is_system_content %} - {{- raise_exception('System message cannot contain videos.') }} - {%- endif %} - {%- if do_vision_count %} - {%- set video_count.value = video_count.value + 1 %} - {%- endif %} - {%- if add_vision_id %} - {{- 'Video ' ~ video_count.value ~ ': ' }} - {%- endif %} - {{- '<|vision_start|><|video_pad|><|vision_end|>' }} - {%- elif 'text' in item %} - {{- item.text }} - {%- else %} - {{- raise_exception('Unexpected item type in content.') }} - {%- endif %} - {%- else %} - {{- item | string }} - {%- endif %} - {%- endfor %} - {%- elif content is none or content is undefined %} - {{- '' }} - {%- else %} - {{- raise_exception('Unexpected content type.') }} - {%- endif %} -{%- endmacro %} -{%- if not messages %} - {{- raise_exception('No messages provided.') }} -{%- endif %} -{%- set head = namespace(count=0, seen_non_system=false) %} -{%- for message in messages %} - {%- set _is_sys = (message.role == 'system' or message.role == 'developer') %} - {%- if _is_sys and not head.seen_non_system %} - {%- set head.count = head.count + 1 %} - {%- else %} - {%- set head.seen_non_system = true %} - {%- endif %} -{%- endfor %} -{%- set sys_state = namespace(content='') %} -{%- for message in messages[:head.count] %} - {%- set _part = render_content(message.content, false, true) | trim %} - {%- if '<|think_off|>' in _part %}{%- set _part = _part.split('<|think_off|>') | join('') | trim %}{%- endif %} - {%- if '<|think_on|>' in _part %}{%- set _part = _part.split('<|think_on|>') | join('') | trim %}{%- endif %} - {%- if '<|think_xhigh|>' in _part %}{%- set _part = _part.split('<|think_xhigh|>') | join('') | trim %}{%- endif %} - {%- if '<|think_high|>' in _part %}{%- set _part = _part.split('<|think_high|>') | join('') | trim %}{%- endif %} - {%- if '<|think_ultracode|>' in _part %}{%- set _part = _part.split('<|think_ultracode|>') | join('') | trim %}{%- endif %} - {%- if '<|think_extreme|>' in _part %}{%- set _part = _part.split('<|think_extreme|>') | join('') | trim %}{%- endif %} - {%- if '<|think_max|>' in _part %}{%- set _part = _part.split('<|think_max|>') | join('') | trim %}{%- endif %} - {%- if '<|think_medium|>' in _part %}{%- set _part = _part.split('<|think_medium|>') | join('') | trim %}{%- endif %} - {%- if '<|think_low|>' in _part %}{%- set _part = _part.split('<|think_low|>') | join('') | trim %}{%- endif %} - {%- if '<|think_minimal|>' in _part %}{%- set _part = _part.split('<|think_minimal|>') | join('') | trim %}{%- endif %} - {%- if _part %} - {%- if sys_state.content %} - {%- set sys_state.content = sys_state.content ~ '\n\n' ~ _part %} - {%- else %} - {%- set sys_state.content = _part %} - {%- endif %} - {%- endif %} -{%- endfor %} -{%- set _sc = sys_state.content %} -{%- set _msgs = messages[head.count:] %} -{%- if ns_state.thinking %} - {%- set _terse_lead = 'Answer directly, after thinking. Lead with the answer, then only what it needs to be correct and usable.' %} -{%- else %} - {%- set _terse_lead = 'Answer directly and concisely. Give the answer with only what it needs to be correct and usable.' %} -{%- endif %} -{%- set _terse_core %} -Never: open with preamble or pleasantries; restate the question; add filler transitions; hedge with niceties; or repeat a point you've already made. -Always: keep essential steps, caveats, uncertainties, and specifics — never drop correctness or a needed warning for brevity. Keep the final answer lean. Use the least structure that conveys it (plain prose when short; lists or code only when they earn their place). If genuinely uncertain, say so and explain why — never omit uncertainty for the sake of brevity. -If a user request is genuinely ambiguous, ask a sharp question, don't guess. -{%- endset %} -{%- set _terse = _terse_lead ~ '\n' ~ (_terse_core | trim) %} -{#- `terse` chat-template kwarg: default true, so behaviour is unchanged unless a caller opts out. - Pass {"terse": false} via chat_template_kwargs to serve the model with only its own system - prompt -- useful for A/B-ing the terseness effect, or when a downstream prompt conflicts. #} -{%- set _terse_on = terse if terse is defined else true %} -{%- if _terse_on %} - {%- if not _sc %} - {%- set _sc = _terse | trim %} - {%- else %} - {%- set _sc = (_sc | trim) ~ '\n\n' ~ (_terse | trim) %} - {%- endif %} -{%- endif %} -{%- if _has_tools %} - {{- '<|im_start|>system\n' }} - {%- if reasoning_instructions %} - {{- reasoning_instructions + '\n\n' }} - {%- endif %} - {{- '# Tools\n\nYou have access to the following functions:\n\n' }} - {%- for tool in tools %} - {{- '\n' }} - {{- tool | tojson }} - {%- endfor %} - {{- '\n' }} - {%- if _tool_format == 'json' %} - {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n' }}{%- if ns_state.thinking %}{{- '\nBrief explanation of tool call\n\n' }}{%- endif %}{{- '\n{"name": "example_function_name", "arguments": {"example_parameter_1": "value_1", "example_parameter_2": "This is the value for the second parameter"}}\n\n\n\nReminder:\n' }}{%- if ns_state.thinking %}{{- '- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the block.\n' }}{%- endif %}{{- '- Function calls MUST follow the specified format: a single JSON object with "name" and "arguments" keys inside XML tags.\n- If you choose to call a tool, you MUST output the block IMMEDIATELY' }}{%- if ns_state.thinking %}{{- ' after thinking' }}{%- endif %}{{- ', with NO conversational text before it.\n- The tag MUST be at the very beginning of a new line, with NO spaces or indentation before it.\n- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n' }} - {%- else %} - {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n' }}{%- if ns_state.thinking %}{{- '\nBrief explanation of tool call\n\n' }}{%- endif %}{{- '\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n' }}{%- if ns_state.thinking %}{{- '- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the block.\n' }}{%- endif %}{{- '- Function calls MUST follow the specified format: an inner block must be nested within XML tags.\n- If you choose to call a tool, you MUST output the block IMMEDIATELY' }}{%- if ns_state.thinking %}{{- ' after thinking' }}{%- endif %}{{- ', with NO conversational text before it.\n- The and tags MUST be at the very beginning of a new line, with NO spaces or indentation before them.\n- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n' }} - {%- endif %} - {%- if _sc %} - {{- '\n\n' + _sc }} - {%- endif %} - {{- '<|im_end|>\n' }} -{%- else %} - {%- if _sc %} - {{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '') + _sc + '<|im_end|>\n' }} - {%- elif reasoning_instructions %} - {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }} - {%- endif %} -{%- endif %} -{%- set _last_idx = _msgs | length - 1 %} -{%- set ns = namespace(multi_step_tool=true, last_query_index=_last_idx) %} -{%- for message in _msgs[::-1] %} - {%- set index = (_msgs | length - 1) - loop.index0 %} - {%- if ns.multi_step_tool and message.role == 'user' %} - {%- set _rc = render_content(message.content, false) | trim %} - {%- if not (_rc.startswith('') and _rc.endswith('')) %} - {%- set ns.multi_step_tool = false %} - {%- set ns.last_query_index = index %} - {%- endif %} - {%- endif %} -{%- endfor %} -{%- if ns.multi_step_tool %} - {%- if _last_idx > 50 %} - {%- set ns.last_query_index = _last_idx %} - {%- else %} - {%- set ns.last_query_index = 0 %} - {%- endif %} -{%- endif %} -{%- set ns2 = namespace(prev_role='', consecutive_failures=0) %} -{%- for message in _msgs %} - {%- set is_system = (message.role == "system" or message.role == "developer") %} - {%- set content = render_content(message.content, true, is_system) | trim %} - {%- if is_system or message.role == 'user' %} - {%- if '<|think_off|>' in content %}{%- set content = content.split('<|think_off|>') | join('') | trim %}{%- endif %} - {%- if '<|think_on|>' in content %}{%- set content = content.split('<|think_on|>') | join('') | trim %}{%- endif %} - {%- if '<|think_xhigh|>' in content %}{%- set content = content.split('<|think_xhigh|>') | join('') | trim %}{%- endif %} - {%- if '<|think_high|>' in content %}{%- set content = content.split('<|think_high|>') | join('') | trim %}{%- endif %} - {%- if '<|think_ultracode|>' in content %}{%- set content = content.split('<|think_ultracode|>') | join('') | trim %}{%- endif %} - {%- if '<|think_extreme|>' in content %}{%- set content = content.split('<|think_extreme|>') | join('') | trim %}{%- endif %} - {%- if '<|think_max|>' in content %}{%- set content = content.split('<|think_max|>') | join('') | trim %}{%- endif %} - {%- if '<|think_medium|>' in content %}{%- set content = content.split('<|think_medium|>') | join('') | trim %}{%- endif %} - {%- if '<|think_low|>' in content %}{%- set content = content.split('<|think_low|>') | join('') | trim %}{%- endif %} - {%- if '<|think_minimal|>' in content %}{%- set content = content.split('<|think_minimal|>') | join('') | trim %}{%- endif %} - {%- endif %} - {%- if is_system %} - {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} - {%- elif message.role == 'user' %} - {%- set ns2.consecutive_failures = 0 %} - {{- '<|im_start|>user\n' + content + '<|im_end|>\n' }} - {%- elif message.role == 'assistant' %} - {%- set reasoning_content = '' %} - {%- set _explicit_reasoning = '' %} - {%- if message.reasoning_content is defined and message.reasoning_content is not none %} - {%- if message.reasoning_content is string %} - {%- set _explicit_reasoning = message.reasoning_content %} - {%- else %} - {%- set _explicit_reasoning = message.reasoning_content | string %} - {%- endif %} - {%- elif message.thinking is defined and message.thinking is not none %} - {%- if message.thinking is string %} - {%- set _explicit_reasoning = message.thinking %} - {%- else %} - {%- set _explicit_reasoning = message.thinking | string %} - {%- endif %} - {%- elif message.reasoning is defined and message.reasoning is not none %} - {%- if message.reasoning is string %} - {%- set _explicit_reasoning = message.reasoning %} - {%- else %} - {%- set _explicit_reasoning = message.reasoning | string %} - {%- endif %} - {%- endif %} - {%- if _explicit_reasoning %} - {%- set _lead_end = '' %} - {%- if content.startswith('') and '' in content %} - {%- set _lead_end = '' %} - {%- elif content.startswith('') and '' in content %} - {%- set _lead_end = '' %} - {%- elif content.startswith('') %} - {%- set _lead_end = '' %} - {%- elif content.startswith('') %} - {%- set _lead_end = '' %} - {%- endif %} - {%- if _lead_end %} - {%- set content = content.split(_lead_end)[-1].lstrip('\n') %} - {%- endif %} - {%- set reasoning_content = _explicit_reasoning %} - {%- else %} - {%- set _think_end = '' %} - {%- if content.startswith('') %} - {%- set _think_end = '' %} - {%- elif content.startswith('') %} - {%- set _think_end = '' %} - {%- elif '\n' in content %} - {%- set _think_end = '\n' %} - {%- elif '\n' in content %} - {%- set _think_end = '\n' %} - {%- elif '\n' in content %} - {%- set _think_end = '\n' %} - {%- elif '\n' in content %} - {%- set _think_end = '\n' %} - {%- elif content.startswith('') and '' in content %} - {%- set _think_end = '' %} - {%- elif content.startswith('') and '' in content %} - {%- set _think_end = '' %} - {%- endif %} - {%- if _think_end %} - {%- if 'thinking' in _think_end %} - {%- set _think_start = '' %} - {%- else %} - {%- set _think_start = '' %} - {%- endif %} - {%- set reasoning_content = content.split(_think_end)[0].rstrip('\n') %} - {%- if _think_start in reasoning_content %} - {%- set reasoning_content = reasoning_content.split(_think_start)[-1].lstrip('\n') %} - {%- endif %} - {%- set content = content.split(_think_end)[-1].lstrip('\n') %} - {%- endif %} - {%- endif %} - {%- set reasoning_content = reasoning_content | trim %} - {%- if (_preserve_thinking or loop.index0 > ns.last_query_index) %} - {{- '<|im_start|>assistant\n\n' + reasoning_content + '\n\n\n' + content }} - {%- else %} - {{- '<|im_start|>assistant\n' + content }} - {%- endif %} - {%- if message.tool_calls is defined and message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} - {%- for tool_call in message.tool_calls %} - {%- if tool_call.function is defined and tool_call.function is not none %} - {%- set tc = tool_call.function %} - {%- else %} - {%- set tc = tool_call %} - {%- endif %} - {%- set tc_name = tc.name if (tc.name is defined and tc.name is not none) else '' %} - {%- if _tool_format == 'json' %} - {%- if loop.first %} - {%- if content | trim %} - {{- '\n\n' }} - {%- endif %} - {%- else %} - {{- '\n' }} - {%- endif %} - {%- set _args = '{}' %} - {%- if tc.arguments is defined and tc.arguments is not none %} - {%- if tc.arguments is mapping %} - {%- set _args = tc.arguments | tojson %} - {%- elif tc.arguments is string %} - {%- if tc.arguments %} - {%- set _args = tc.arguments %} - {%- endif %} - {%- else %} - {%- set _args = tc.arguments | tojson %} - {%- endif %} - {%- endif %} - {{- '\n{"name": ' }}{{- tc_name | tojson }}{{- ', "arguments": ' }}{{- _args }}{{- '}\n' }} - {%- else %} - {%- if loop.first %} - {%- if content | trim %} - {{- '\n\n\n\n' }} - {%- else %} - {{- '\n\n' }} - {%- endif %} - {%- else %} - {{- '\n\n\n' }} - {%- endif %} - {%- if tc.arguments is defined and tc.arguments is not none %} - {%- if tc.arguments is mapping %} - {%- for args_name, args_value in tc.arguments.items() %} - {{- '\n' }} - {%- if args_value is string %} - {%- set _av = args_value %} - {%- else %} - {%- set _av = args_value | tojson %} - {%- endif %} - {%- if max_tool_arg_chars > 0 and _av | length > max_tool_arg_chars %} - {{- _av[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_av | length | string) ~ ' chars]' }} - {%- else %} - {{- _av }} - {%- endif %} - {{- '\n\n' }} - {%- endfor %} - {%- else %} - {%- if tc.arguments is string %} - {%- set _raw_args = tc.arguments %} - {%- else %} - {%- set _raw_args = tc.arguments | tojson %} - {%- endif %} - {%- if _raw_args %} - {%- if max_tool_arg_chars > 0 and _raw_args | length > max_tool_arg_chars %} - {{- _raw_args[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_raw_args | length | string) ~ ' chars]' }} - {%- else %} - {{- _raw_args }} - {%- endif %} - {%- endif %} - {%- endif %} - {%- endif %} - {{- '\n' }} - {%- endif %} - {%- endfor %} - {%- endif %} - {{- '<|im_end|>\n' }} - {%- elif message.role == 'tool' %} - {%- set _content_lower = content | lower %} - {%- set _content_head = _content_lower[:120] %} - {%- set _is_code_or_grep = ('throw new ' in _content_lower or 'throw error' in _content_lower or 'console.error' in _content_lower or 'logger.error' in _content_lower or 'logging.error' in _content_lower or 'import ' in _content_head or 'def ' in _content_head or 'function ' in _content_head) %} - {%- set _exit_code_zero = ('exit code: 0' in _content_head or 'process exited with code 0' in _content_head) %} - {%- set _error_field_ok = ('"error": null' in _content_head or '"error":null' in _content_head or '"error": false' in _content_head or '"error":false' in _content_head or '"error": ""' in _content_head or '"error":""' in _content_head) %} - {%- set _strong_error = (('"error":' in _content_head and not _error_field_ok) or '"status": "error"' in _content_head or '"status":"error"' in _content_head or 'traceback (most recent call last):' in _content_head or 'command not found' in _content_head or 'invalid syntax' in _content_head or 'fatal:' in _content_head or (('exit code: ' in _content_head or 'process exited with code' in _content_head) and not _exit_code_zero) or _content_head.startswith('exception:') or _content_head.startswith('failed to ')) %} - {%- set _weak_error = ('error:' in _content_head or 'err!' in _content_head) %} - {%- set _weak_suppressed = ('$ ' in _content_head or 'took ' in _content_head or content | length >= 600) %} - {%- if not _is_code_or_grep and (_strong_error or (_weak_error and not _weak_suppressed)) %} - {%- set ns2.consecutive_failures = ns2.consecutive_failures + 1 %} - {%- else %} - {%- set ns2.consecutive_failures = 0 %} - {%- endif %} - {%- if ns2.prev_role != 'tool' %} - {{- '<|im_start|>user' }} - {%- endif %} - {%- if _tool_format != 'json' and max_tool_response_chars > 0 and content | length > max_tool_response_chars %} - {%- set content = content[:max_tool_response_chars] + '\n[TRUNCATED - original length ' ~ (content | length | string) ~ ' chars]' %} - {%- endif %} - {{- '\n\n' + content }} - {%- if ns2.consecutive_failures >= 2 %} - {{- '\n\n⚠️ SYSTEM WARNING: ' ~ ns2.consecutive_failures ~ ' consecutive tool errors detected. Your previous approach is incorrect. You MUST use a fundamentally different approach or corrected arguments.' }} - {%- elif ns2.consecutive_failures == 1 %} - {{- '\n\n⚠️ SYSTEM WARNING: The previous tool call returned an error. Diagnose the failure and retry with completely corrected arguments.' }} - {%- endif %} - {{- '\n' }} - {%- if loop.last %} - {{- '<|im_end|>\n' }} - {%- else %} - {%- set _next_role = _msgs[loop.index0 + 1].role %} - {%- if _next_role != 'tool' %} - {{- '<|im_end|>\n' }} - {%- endif %} - {%- endif %} - {%- else %} - {{- '<|im_start|>user\n[' + message.role + ']: ' + content + '<|im_end|>\n' }} - {%- endif %} - {%- set ns2.prev_role = message.role %} -{%- endfor %} -{%- if add_generation_prompt %} - {{- '<|im_start|>assistant\n' }} - {%- if not ns_state.thinking %} - {{- '\n\n\n\n' }} - {%- else %} - {{- '\n' }} - {%- endif %} -{%- endif %} \ No newline at end of file diff --git a/scripts/.AGENTTRASH/download-q8agent.sh.rocmfpx b/scripts/.AGENTTRASH/download-q8agent.sh.rocmfpx deleted file mode 100755 index d27d6f7..0000000 --- a/scripts/.AGENTTRASH/download-q8agent.sh.rocmfpx +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash -# Download the kingjones Qwen3.8-27B-ROCmFPX-Q8_0-AGENT payload for -# frank-medium from HuggingFace into -# /home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT. -# -# Run on Dankserver (the box that serves it). The repo is public, so the -# download works anonymously; a token in ~/.cache/huggingface/token is used -# when present, which keeps the ~28 GB main file clear of the anonymous -# rate limit at full speed. -# -# bash scripts/download-q8agent.sh -# bash scripts/download-q8agent.sh --verify-only -# -# The script is resumable (curl -C -) and verifies every artifact's size -# afterwards, so a truncated file fails loudly instead of load-crashing. -set -euo pipefail - -REPO="kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF" -DIR="/home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT" -EXPECTED=( - "Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf:28214425952" - "mmproj-Qwen3.8-27B-BF16.gguf:931145888" - "mtp-Qwen3.8-27B-Q4_0.gguf:1680271648" -) -# The model card's MTP prompt-cache fix ships in the same repo as a patch; -# fetched for reference only. This fork's main already carries the -# speculative boundary in the context checkpoint, so nothing applies it. -EXTRA=( - "patches/mtp-prompt-cache-fix.patch" -) - -cd -- "$(dirname -- "$0")/.." - -main() { - local mode="download" - [[ "${1:-}" == "--verify-only" ]] && mode="verify" - mkdir -p "$DIR" - cd "$DIR" - - local token="" auth=() - if [[ -r "$HOME/.cache/huggingface/token" ]]; then - token="$(cat "$HOME/.cache/huggingface/token")" - auth=(-H "Authorization: Bearer $token") - else - echo "no HF token at $HOME/.cache/huggingface/token; downloading anonymously" - fi - - dl() { - local f="$1" - mkdir -p "$(dirname "$f")" - curl -fsSL -C - -o "$f" --retry 10 --retry-delay 5 --retry-max-time 10800 \ - "${auth[@]}" \ - "https://huggingface.co/$REPO/resolve/main/$f" - echo " done: $f ($(du -h "$f" | cut -f1))" - } - - if [[ "$mode" == "download" ]]; then - echo "downloading $REPO into $DIR" - for entry in "${EXPECTED[@]}"; do - dl "${entry%%:*}" - done - for f in "${EXTRA[@]}"; do - dl "$f" - done - fi - - echo "verifying sizes" - local ok=1 f want act - for entry in "${EXPECTED[@]}"; do - f="${entry%%:*}"; want="${entry##*:}" - act="$(stat -c%s "$f" 2>/dev/null || echo 0)" - if [[ "$act" -eq "$want" ]]; then - echo " OK $f ($act bytes)" - else - echo " BAD $f ($act bytes, want $want)" >&2 - ok=0 - fi - done - if [[ -z "$token" ]]; then - echo "note: no token was used; sizes above are the integrity check" - fi - [[ "$ok" -eq 1 ]] || exit 1 - echo "all files present and sized correctly" -} - -main "$@" diff --git a/scripts/.AGENTTRASH/frank-default-npu.env b/scripts/.AGENTTRASH/frank-default-npu.env deleted file mode 100644 index fc6662f..0000000 --- a/scripts/.AGENTTRASH/frank-default-npu.env +++ /dev/null @@ -1,17 +0,0 @@ -# Tunables for frank-default-npu (FastFlowLM NPU endpoint, Dankclaw) -# Runtime: FLM 1.0.1 on the XDNA2 NPU (50 TOPS). Model: Gemma4-E4B-IT -# (8B, Q4_1, vision + audio + reasoning). The NPU allocates from the -# unified pool shared with frank-default-large (the CIRU llama.cpp -# service on the iGPU). See serverAppConfigs-docs/FastFlowLM NPU Inference.md. -# -# 2026-08-20: gemma4-it:e4b deployed. Footprint ~9.1 GB. 64k context. - -FLM_MODEL=gemma4-it:e4b -FLM_HOST=0.0.0.0 -FLM_PORT=52626 -FLM_PMODE=performance -FLM_ASR=0 -# 64K context (native max for gemma4-it is 32768, but NPU supports up to 256k). -# The model's max_prefill_len is 4096, so prompts longer than that get -# truncated at prefill time. -FLM_CTX_LEN=65536 diff --git a/scripts/.AGENTTRASH/frank-default-npu.service b/scripts/.AGENTTRASH/frank-default-npu.service deleted file mode 100644 index 14a7af5..0000000 --- a/scripts/.AGENTTRASH/frank-default-npu.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=FastFlowLM NPU inference (frank-default-npu) on Dankclaw -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=dank -Group=dank -Environment="FLM_NPU_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-default-npu.env" -ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-default-npu.sh -LimitMEMLOCK=infinity -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/scripts/.AGENTTRASH/frank-dragoon.env.ik-llama.cpp b/scripts/.AGENTTRASH/frank-dragoon.env.ik-llama.cpp deleted file mode 100644 index 8a67d46..0000000 --- a/scripts/.AGENTTRASH/frank-dragoon.env.ik-llama.cpp +++ /dev/null @@ -1,105 +0,0 @@ -# Tunables for frank-dragoon (mradermacher i1-Q4_K_M of -# orcarouter/Qwen3.8-Flash-Next-Uncensored on ik_llama.cpp, Dankclaw, -# RTX 3090 CUDA + Strix Halo CPU path, since 2026-09-15). -# -# Runtime: ik_llama.cpp master d85a1d7 (2026-09-15) with full qwen4exp -# support (upstream PRs #2365/#2369/#2373 merged). Build recipe (CUDA for -# the 3090 at sm_86, Zen5 AVX-512 IQK kernels for the CPU expert path): -# cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=ON \ -# -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86 \ -# -DGGML_AVX512=ON -DGGML_AVX512_VBMI=ON -DGGML_AVX512_VNNI=ON \ -# -DGGML_AVX512_BF16=ON -DLLAMA_CURL=OFF -# The startup banner must print HAVE_FANCY_SIMD is defined — without those -# macros the quantized GEMM silently falls back to AVX2 (no warning). -# -# Model: mradermacher imatrix quant of the uncensored Flash-Next (qwen4exp -# MoE: 512 experts / 10 used, 48 layers = 12 attention + 36 GDN/ssm, QSA -# indexer, hyper-connections, PLE n-gram table ~320M rows). 111 GiB shipped -# as 3 parts, concatenated on disk; each part's sha256 was verified against -# its HF LFS oid at download time. -# -# Placement (ikawrakow's hybrid recipe; upstream issue #2402 for the PLE): -# - -ot "exps=CPU": ALL 512-expert tensors (121B elems, ~84 GB) live in -# the Strix Halo's LPDDR5X via the plain CPU backend buffer — mmap-backed -# page cache, ik's fastest known configuration for this shape (the -# author's 2x3090 + Threadripper reference box measures ~50 t/s on the -# same model family). The shared experts (shexp) keep their GPU-side -# default: they run on every token. -# - -ot "per_layer_token_embd=CPU": the ~32 GB PLE n-gram table follows — -# only a tiny slice is touched per token, so it streams from page cache. -# - NEVER use --cpu-moe on this host: it routes the CPU-overridden -# weights through the CUDA_Host PINNED buffer (measured 2026-09-15: -# 108.12 GiB page-locked, evicted every co-tenant into swap and starved -# sshd into unreachability until a power cycle). -ot ...=CPU is the -# mmap-backed path with identical DDR5 bandwidth for resident pages, -# and the kernel degrades gracefully under pressure (page cache evict -# + re-read) instead of stranding anyone. -# - GGML_CUDA_NO_PINNED_WEIGHTS=1 (src/llama-load-tensors.cpp) keeps the -# CPU-overridden weights mmapped and leaves only the small staging -# buffers pinned. (GGML_CUDA_NO_PINNED would also unpin staging — -# unnecessary here and slower for prefill transfers.) -# -# Sampling: greedy-by-default to preserve the halogen-era client contract -# (a request without temperature decodes greedy; temp 0.0 = argmax). -# Clients that send their own temperature get their own values. -# -# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE (see -# the 2026-08-13 HIP incident in AGENTS.md). llama.cpp reads --host and -# --temp as CLI flags, never as environment variables. - -# ik_llama.cpp runtime binary (master d85a1d7 build). -LLAMA_BIN=/home/dank/GitPackages/ik_llama.cpp/build/bin/llama-server - -# Model package from -# https://huggingface.co/mradermacher/Qwen3.8-Flash-Next-Uncensored-i1-GGUF -# (parts concatenated). The mmproj is the matching vision tower from the -# static repo mradermacher/Qwen3.8-Flash-Next-Uncensored-GGUF — ik_llama.cpp -# links examples/mtmd, so image input survives the runtime swap. -MODEL_DIR=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M -MODEL_FILE=Qwen3.8-Flash-Next-Uncensored.i1-Q4_K_M.gguf -MMPROJ_PATH=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M/Qwen3.8-Flash-Next-Uncensored.mmproj-f16.gguf - -# Qwen-Sharp chat template (tracked in the repo, staged on the host). The -# server emits reasoning in DeepSeek format so clients receive -# reasoning_content separately instead of literal think tags in content. -CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja - -# Alias + network. -ALIAS=frank-dragoon -BIND_ADDR=0.0.0.0 -LISTEN_PORT=7770 - -# Serving: single 262144-token slot (the model's full context window), -# batch 4096 / ubatch 512, 16+16 threads (the Halo CPU has 16 cores, no -# SMT), f16 KV. The KV cache scales with the 12 full-attention layers only -# (~6 GB f16 at 262k); the GDN/ssm layers keep constant-size state. If -# VRAM headroom is needed, --cache-type-k/v q8_0 halves the KV footprint. -CTX_SIZE=262144 -PARALLEL=1 -BATCH_SIZE=4096 -UBATCH_SIZE=512 -NUM_THREADS=16 -THREAD_BATCH=16 -CACHE_TYPE_K=f16 -CACHE_TYPE_V=f16 - -# GPU/MoE placement: everything on the 3090 except the experts and the PLE -# table, which both live in host RAM via the CPU backend buffer (see the -# placement notes above). -NGL=999 -# ik_llama.cpp CUDA host-memory policy: CPU-overridden weights stay mmapped; -# only the small staging buffers are pinned (see the placement notes above). -GGML_CUDA_NO_PINNED_WEIGHTS=1 - -# Prompt cache (RAM budget in MiB) and slot state dir. -CACHE_RAM=8192 -SLOT_SAVE_PATH=/home/dank/serverAppConfigs/scripts/slot-state - -# Greedy-by-default (halogen contract). See sampling note above. -TEMPERATURE=0.0 - -# Reasoning on: DeepSeek format projects thinking into the API's -# reasoning_content field instead of literal think tags in content; -# unlimited reasoning budget (the model decides). -REASONING_FORMAT=deepseek -REASONING_BUDGET=-1 diff --git a/scripts/.AGENTTRASH/frank-dragoon.service.ik-llama.cpp b/scripts/.AGENTTRASH/frank-dragoon.service.ik-llama.cpp deleted file mode 100644 index eae0d10..0000000 --- a/scripts/.AGENTTRASH/frank-dragoon.service.ik-llama.cpp +++ /dev/null @@ -1,21 +0,0 @@ -[Unit] -Description=ik_llama.cpp Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M 177B MoE (dankclaw, frank-dragoon) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=dank -Group=dank -# Staging buffers for the CUDA transfers are page-locked; the weights -# themselves stay mmapped (GGML_CUDA_NO_PINNED_WEIGHTS=1), but the staging -# budget still needs room. Parity with the halogen quadlet's -# Ulimit=memlock=-1:-1. -LimitMEMLOCK=infinity -Environment="FRANK_DRAGOON_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-dragoon.env" -ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-dragoon.sh -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/scripts/.AGENTTRASH/frank-medium.env.rocmfpx b/scripts/.AGENTTRASH/frank-medium.env.rocmfpx deleted file mode 100644 index a9fdd09..0000000 --- a/scripts/.AGENTTRASH/frank-medium.env.rocmfpx +++ /dev/null @@ -1,138 +0,0 @@ -# Tunables for frank-medium (kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF, -# charlie12345/ROCmFPX llama.cpp, Dankserver, 2x AMD Radeon AI PRO R9700). -# -# Hardware (2026-09-15): the RTX 3090 left Dankserver and two Radeon AI PRO -# R9700 cards (Navi 48, gfx1201, RDNA4, 32,624 MiB each, PCIe 5.0 x16) took -# its place. The Raphael iGPU (gfx1036) is still enumerated by HIP as the -# third device. There is no XGMI link between the two cards and they sit in -# different IOMMU groups, so any cross-GPU traffic goes over PCIe. -# -# Runtime: charlie12345/ROCmFPX (llama.cpp fork, ROCmFPX quant family; the -# Q8_0_ROCMFPX_AGENT tensor type exists nowhere else - stock llama.cpp cannot -# load this GGUF). Built for gfx1201 with the fork's self-contained ROCm -# 7.14 toolchain so the host's stale /opt/rocm-6.3.3 install is never used: -# -# cd /home/dank/GitPackages/charlie-ROCmFPX -# env JOBS=8 ./scripts/build-rocmfp4-rocm714-local.sh -# -# gfx1201 must be the build target: a gfx1200 binary links, loads the model -# and then segfaults on this card (fork's docs/BUILD-AMD-ARCHITECTURES.md). -# -# Model: kingjones Qwen3.8-27B-ROCmFPX-Q8_0-AGENT - Q8_0_ROCMFPX_AGENT -# (ftype 115), 26.28 GiB, 8.39 bpw, arch qwen35 (MoE), quantized from a -# 51.3 GiB BF16 source. The AGENT routing keeps 12 more attention-output -# projections at full Q8_0 than the plain 8-bit build, which buys a higher -# MTP draft acceptance (0.953 vs 0.911) and that is the whole point of this -# checkpoint. Model card documents 131,072 context. -# -# Parallelism (--split-mode layer, operator call 2026-09-15 "pipeline -# parallelism"): layers are distributed across the two cards in order and -# activations hop PCIe at the boundary. Measured with llama-bench -# (pp4096 / tg128 / same at depth 32768, f16 KV, fa on, 3 reps): -# -# single R9700 1160.9 / 19.42 (fits: 26.27 GiB weights, ~5 GiB left) -# layer 2x 1991.2 / 18.76 859.1 / 17.64 -# tensor 2x 271.8 / 26.26 228.6 / 25.09 -# -# Tensor parallel wins decode (+40 %) but collapses prefill 7x: this build -# has no RCCL (llama.cpp logs "AllReduce init failed; falling back to -# meta-backend butterfly"), so every layer pays a slow cross-card reduction. -# Agentic traffic is prefill-heavy, so layer split wins the workload. Revisit -# if the fork ships a working RCCL tensor path. -# -# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The -# ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP -# incident). llama.cpp reads --host/--port/--temp as CLI flags, never as -# env vars. - -# charlie12345/ROCmFPX llama-server (self-contained ROCm 7.14 build, RPATH -# points at the bundled runtime libs; no LD_LIBRARY_PATH needed). -LLAMA_BIN=/home/dank/GitPackages/charlie-ROCmFPX/build-rdna4-rocm714/bin/llama-server - -# Model package, fetched with scripts/download-q8agent.sh: -# Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf 26.28 GiB main weights -# mtp-Qwen3.8-27B-Q4_0.gguf 1.68 GiB MTP draft head -# mmproj-Qwen3.8-27B-BF16.gguf 0.93 GiB vision tower (unused) -MODEL_DIR=/home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT -MODEL_FILE=Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf -DRAFT_MODEL_FILE=mtp-Qwen3.8-27B-Q4_0.gguf - -# Alias + network. The port is the fleet's dankserver endpoint; the alias is -# what clients ask for, and it replaces the retired `frank-small`. -ALIAS=frank-medium -BIND_ADDR=0.0.0.0 -LISTEN_PORT=7770 - -# Device placement. HIP_VISIBLE_DEVICES hides the iGPU (device 2) so nothing -# can drift onto shared system RAM; DEVICES is what llama.cpp splits across. -HIP_VISIBLE_DEVICES=0,1 -DEVICES=ROCm0,ROCm1 -SPLIT_MODE=layer - -# TENSOR_SPLIT empty = layers distributed by free VRAM (equal here). -TENSOR_SPLIT= - -NGL=999 -# Auto-fit has no business resizing a context we set explicitly. -FIT=off -FLASH_ATTN=on - -# 131,072 tokens, one slot (the model card's documented window; the GGUF -# reports n_ctx_train 262144, untested by the model author). Layer split -# keeps each layer's KV on the card that owns the layer, so the window is -# bounded by per-card free memory: ~5.6 GiB of KV per card at 131072 f16. -CTX_SIZE=131072 -PARALLEL=1 - -# 26.28 GiB of weights + MTP draft + KV cache leaves ~15 GiB per card at this -# context, so the batch shape is chosen for prefill throughput, not memory. -BATCH_SIZE=2048 -UBATCH_SIZE=512 -NUM_THREADS=8 -THREAD_BATCH=8 - -# f16 K/V: an agent checkpoint is the wrong place to spend KV precision. -CACHE_TYPE_K=f16 -CACHE_TYPE_V=f16 -CTX_CHECKPOINTS=8 -CACHE_RAM=8192 - -# Sampling: the checkpoint's instruct table. -TEMPERATURE=0.7 -TOP_P=0.8 -TOP_K=20 -MIN_P=0.0 -PRESENCE_PENALTY=1.5 - -# Reasoning on; DeepSeek format projects the trace into reasoning_content -# instead of literal thinking tags in content. -REASONING=on -REASONING_FORMAT=deepseek -REASONING_BUDGET=-1 - -# Chat template: empty = the template embedded in the GGUF. Point this at -# /home/dank/serverAppConfigs/scripts/chat-template.jinja to force the repo's -# Sharp agent template instead. -CHAT_TEMPLATE_FILE= -CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true}' - -# MTP speculative decoding. n_max 6 / p_min 0.6 is the fork's measured -# sweet spot for a MTP head with high acceptance; p_min 0.0 drafts -# low-probability tokens that get rejected and makes MTP a net loss. -SPEC_TYPE=draft-mtp -SPEC_DRAFT_N_MAX=6 -SPEC_DRAFT_N_MIN=0 -SPEC_DRAFT_P_MIN=0.6 -SPEC_DRAFT_P_SPLIT=0.10 -SPEC_DRAFT_NGL=999 -SPEC_DRAFT_TYPE_K=f16 -SPEC_DRAFT_TYPE_V=f16 -SPEC_DRAFT_POLL=1 -SPEC_DRAFT_POLL_BATCH=1 -# The draft head rides the card that holds the target model's LAST layers: -# the MTP head consumes the final hidden state, so pinning it to ROCm1 keeps -# the per-token hop off the wire. (Under the abandoned tensor split the -# draft must also be pinned to exactly one device or the meta-device KV -# allocation aborts: GGML_ASSERT(buffer) in ggml-backend.cpp, measured -# 2026-09-15.) -SPEC_DRAFT_DEVICE=ROCm1 diff --git a/scripts/.AGENTTRASH/frank-medium.service.rocmfpx b/scripts/.AGENTTRASH/frank-medium.service.rocmfpx deleted file mode 100644 index 35ce5ae..0000000 --- a/scripts/.AGENTTRASH/frank-medium.service.rocmfpx +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=charlie12345/ROCmFPX llama.cpp serving kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT qwen35 MoE (dankserver, frank-medium, 2x Radeon AI PRO R9700) -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=dank -Group=dank -Environment="FRANK_MEDIUM_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-medium.env" -ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-medium.sh -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/scripts/.AGENTTRASH/serve-frank-default-npu.sh b/scripts/.AGENTTRASH/serve-frank-default-npu.sh deleted file mode 100755 index 17c2168..0000000 --- a/scripts/.AGENTTRASH/serve-frank-default-npu.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -# Serve the FastFlowLM (FLM) NPU endpoint — frank-default-npu. -# -# Runs flm serve with the model tag in frank-default-npu.env. The NPU -# (XDNA2, 50 TOPS) is a single-lane device: generations serialize in FIFO -# order, so this is a background/triage endpoint. The NPU shares the ~30 GiB -# unified pool with frank-default-large (the CIRU llama.cpp service on the -# iGPU); see serverAppConfigs-docs/FastFlowLM NPU Inference.md for the memory budget. -# -# NOTE: FLM requires unlimited memlock. The unit sets LimitMEMLOCK=infinity. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ENV_FILE="${FLM_NPU_ENV_FILE:-$SCRIPT_DIR/frank-default-npu.env}" -if [[ -f "$ENV_FILE" ]]; then - set -a - # shellcheck disable=SC1090 - source "$ENV_FILE" - set +a -fi - -# --- defaults (used when the env file does not set them) ------------------- -: "${FLM_BIN:=/usr/bin/flm}" -: "${FLM_MODEL:=gemma4-it:e4b}" -: "${FLM_HOST:=0.0.0.0}" -: "${FLM_PORT:=52626}" -: "${FLM_PMODE:=performance}" -: "${FLM_ASR:=0}" -: "${FLM_CTX_LEN:=-1}" - -if [[ ! -x "$FLM_BIN" ]]; then - echo "missing flm binary: $FLM_BIN" >&2 - exit 1 -fi - -# --- optional flags ---------------------------------------------------------- -OPT_ARGS=() -if [[ "${FLM_ASR}" != "0" ]]; then - OPT_ARGS+=(-a "${FLM_ASR}") -fi -if [[ "${FLM_CTX_LEN}" != "-1" ]]; then - OPT_ARGS+=(-c "${FLM_CTX_LEN}") -fi - -exec "$FLM_BIN" serve "$FLM_MODEL" \ - --host "$FLM_HOST" \ - -p "$FLM_PORT" \ - --pmode "$FLM_PMODE" \ - "${OPT_ARGS[@]}" diff --git a/scripts/.AGENTTRASH/serve-frank-dragoon.sh.ik-llama.cpp b/scripts/.AGENTTRASH/serve-frank-dragoon.sh.ik-llama.cpp deleted file mode 100755 index e6e09c7..0000000 --- a/scripts/.AGENTTRASH/serve-frank-dragoon.sh.ik-llama.cpp +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash -# Serve mradermacher/Qwen3.8-Flash-Next-Uncensored i1-Q4_K_M as frank-dragoon -# via ik_llama.cpp on Dankclaw. -# -# Topology (ikawrakow's hybrid recipe; upstream issue #2402 for the PLE): -# the RTX 3090 (CUDA, sm_86) takes the dense/attention/output weights and -# the KV cache (-ngl 999); ALL MoE expert tensors and the ~32 GB PLE -# n-gram table stay in the Strix Halo's LPDDR5X via the plain CPU backend -# buffer (-ot ...=CPU — mmap-backed page cache; only a tiny slice of PLE is -# touched per token). NEVER use --cpu-moe on this host: it routes the -# CPU-overridden weights through the CUDA_Host PINNED buffer (measured -# 108.12 GiB page-locked on 2026-09-15, which evicted every co-tenant into -# swap and starved sshd into unreachability until a power cycle). NEVER add -# --no-mmap — the STRIX-era lesson transfers directly (the cgroup killer -# takes the process with nothing in the server log). -# -# Runtime: ik_llama.cpp master d85a1d7 built with CUDA (sm_86) + Zen5 -# AVX-512 IQK kernels; the startup banner must show HAVE_FANCY_SIMD -# is defined. Flag set verified against this binary's --help before this -# script was written. ik has no --cache-prompt/--cache-idle-slots (prompt -# caching is its default behavior) and no --alias: it is a single-model -# server and reports the GGUF's general.name at /v1/models. -# -# Sampling: greedy-by-default (halogen client contract) — clients that -# send their own temperature get their own values. -set -euo pipefail - -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -ENV_FILE="${FRANK_DRAGOON_ENV_FILE:-$SCRIPT_DIR/frank-dragoon.env}" -if [[ -f "$ENV_FILE" ]]; then - set -a - # shellcheck disable=SC1090 - source "$ENV_FILE" - set +a -fi - -# --- defaults (used when the env file does not set them) ------------------- -: "${LLAMA_BIN:=/home/dank/GitPackages/ik_llama.cpp/build/bin/llama-server}" -: "${MODEL_DIR:=/home/dank/models/mradermacher-Qwen3.8-Flash-Next-Uncensored-i1-Q4_K_M}" -: "${MODEL_FILE:=Qwen3.8-Flash-Next-Uncensored.i1-Q4_K_M.gguf}" -: "${MMPROJ_PATH:=$MODEL_DIR/Qwen3.8-Flash-Next-Uncensored.mmproj-f16.gguf}" -: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}" -: "${BIND_ADDR:=0.0.0.0}" -: "${LISTEN_PORT:=7770}" -: "${CTX_SIZE:=262144}" -: "${PARALLEL:=1}" -: "${BATCH_SIZE:=4096}" -: "${UBATCH_SIZE:=512}" -: "${NUM_THREADS:=16}" -: "${THREAD_BATCH:=16}" -: "${CACHE_TYPE_K:=f16}" -: "${CACHE_TYPE_V:=f16}" -: "${NGL:=999}" -: "${CPU_MOE:=on}" -: "${FLASH_ATTN:=on}" -: "${TEMPERATURE:=0.0}" -: "${REASONING:=on}" -: "${REASONING_FORMAT:=deepseek}" -: "${REASONING_BUDGET:=-1}" -: "${CACHE_RAM:=8192}" -: "${SLOT_SAVE_PATH:=$SCRIPT_DIR/slot-state}" -# Hard guard: on this host CUDA pinned host memory strands the co-tenants -# (measured 2026-09-15: a default-policy start tried to page-lock 108.12 GiB, -# evicted every co-tenant into swap and starved sshd into unreachability -# until a power cycle). GGML_CUDA_NO_PINNED_WEIGHTS=1 keeps the -# CPU-overridden weights mmapped (src/llama-load-tensors.cpp); the env file -# sets it. Refuse to run without it. -if [[ "${GGML_CUDA_NO_PINNED_WEIGHTS:-}" != "1" ]]; then - echo "frank-dragoon: refusing to start — GGML_CUDA_NO_PINNED_WEIGHTS=1 is mandatory on this host" >&2 - exit 1 -fi - -mkdir -p "$SLOT_SAVE_PATH" - -# The UVM device node is created lazily by nvidia-modprobe on the first -# CUDA open after boot; make sure it exists before the server needs it. -nvidia-modprobe -u -c=0 2>/dev/null || true - -CPU_MOE_ARGS=() -if [[ "$CPU_MOE" == "on" ]]; then - # -ot ...=CPU (plain CPU backend buffer) is the mmap-backed path. NEVER - # swap this for --cpu-moe: that routes the weights through the CUDA_Host - # PINNED buffer (108.12 GiB measured) and strands the co-tenants. - CPU_MOE_ARGS=(-ot "exps=CPU" -ot "per_layer_token_embd=CPU") -fi - -MMPROJ_ARGS=() -if [[ -n "${MMPROJ_PATH:-}" && -f "$MMPROJ_PATH" ]]; then - MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH") -fi - -TEMPLATE_ARGS=() -if [[ -n "${CHAT_TEMPLATE_FILE:-}" && -f "$CHAT_TEMPLATE_FILE" ]]; then - TEMPLATE_ARGS=(--chat-template-file "$CHAT_TEMPLATE_FILE") -else - TEMPLATE_ARGS=(--jinja) -fi - -exec "$LLAMA_BIN" \ - --model "$MODEL_DIR/$MODEL_FILE" \ - "${TEMPLATE_ARGS[@]}" \ - --host "$BIND_ADDR" \ - --port "$LISTEN_PORT" \ - --gpu-layers "$NGL" \ - "${CPU_MOE_ARGS[@]}" \ - --flash-attn "$FLASH_ATTN" \ - --parallel "$PARALLEL" \ - --ctx-size "$CTX_SIZE" \ - --batch-size "$BATCH_SIZE" \ - --ubatch-size "$UBATCH_SIZE" \ - --threads "$NUM_THREADS" \ - --threads-batch "$THREAD_BATCH" \ - --cache-type-k "$CACHE_TYPE_K" \ - --cache-type-v "$CACHE_TYPE_V" \ - --cache-ram "$CACHE_RAM" \ - --slot-save-path "$SLOT_SAVE_PATH" \ - --ctx-checkpoints 16 \ - --metrics \ - --temp "$TEMPERATURE" \ - --reasoning "$REASONING" \ - --reasoning-format "$REASONING_FORMAT" \ - --reasoning-budget "$REASONING_BUDGET" \ - "${MMPROJ_ARGS[@]}" diff --git a/scripts/.AGENTTRASH/serve-frank-medium.sh.rocmfpx b/scripts/.AGENTTRASH/serve-frank-medium.sh.rocmfpx deleted file mode 100755 index 8b64445..0000000 --- a/scripts/.AGENTTRASH/serve-frank-medium.sh.rocmfpx +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/env bash -# Serve kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF as frank-medium via -# the charlie12345/ROCmFPX llama.cpp runtime on Dankserver -# (2x AMD Radeon AI PRO R9700, Navi 48, gfx1201, 32 GB each). -# -# Runtime: charlie12345/ROCmFPX main (c49ebdb, 2026-08-22), built for -# gfx1201 with the fork's self-contained ROCm toolchain: -# -# cd /home/dank/GitPackages/charlie-ROCmFPX -# env JOBS=8 ./scripts/build-rocmfp4-rocm714-local.sh -# -> build-rdna4-rocm714/bin/llama-server -# -# That script downloads the gfx120X-all TheRock ROCm 7.14 toolchain into -# ~/.local/share/rocm-7.14.0a20260624, builds with -# -DCMAKE_HIP_ARCHITECTURES="gfx1200;gfx1201", copies the ROCm runtime -# (hip/hsa/rocblas/hipblaslt + their kernel libraries) next to the binary and -# sets RPATH=$ORIGIN/../lib on it. The resulting service is self-contained: -# no LD_LIBRARY_PATH, and the host's stale /opt/rocm-6.3.3 cannot leak in. -# gfx1201 is NOT interchangeable with gfx1200 on this card (the fork's -# docs/BUILD-AMD-ARCHITECTURES.md: a gfx1200 build links, loads and then -# segfaults), and the build verifies the emitted code objects afterwards. -# -# Model: kingjones Qwen3.8-27B-ROCmFPX-Q8_0-AGENT (Q8_0_ROCMFPX_AGENT, -# ftype 115, arch qwen35 MoE), 26.28 GiB main file, plus a separate -# MTP draft head and an mmproj vision tower. 8.39 bpw; the AGENT routing -# keeps 13 of the 17 attention-output projections at full Q8_0, which is -# what makes the MTP draft land ~0.95 of its guesses. A stock llama.cpp -# cannot load this GGUF at all - Q8_0_ROCMFPX exists only in this fork. -# -# Parallelism: --split-mode layer across both cards (operator call -# 2026-09-15). Layers are distributed in order and activations hop PCIe at -# the boundary; measured llama-bench: layer 1991/18.8 tok/s (pp4096/tg128) -# vs tensor 272/26.3 and single-card 1161/19.4. Tensor wins decode +40 % -# but its prefill collapses 7x on this build (no RCCL: "AllReduce init -# failed; falling back to meta-backend butterfly"), so layer split wins the -# prefill-heavy agentic workload. -# -# Tunables come from frank-medium.env unless FRANK_MEDIUM_ENV_FILE -# overrides that path. -set -euo pipefail - -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -ENV_FILE="${FRANK_MEDIUM_ENV_FILE:-$SCRIPT_DIR/frank-medium.env}" -if [[ -f "$ENV_FILE" ]]; then - set -a - # shellcheck disable=SC1090 - source "$ENV_FILE" - set +a -fi - -# --- defaults (used when the env file does not set them) ----------------- -: "${LLAMA_BIN:=/home/dank/GitPackages/charlie-ROCmFPX/build-rdna4-rocm714/bin/llama-server}" -: "${MODEL_DIR:=/home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT}" -: "${MODEL_FILE:=Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf}" -: "${DRAFT_MODEL_FILE:=mtp-Qwen3.8-27B-Q4_0.gguf}" -: "${ALIAS:=frank-medium}" -: "${BIND_ADDR:=0.0.0.0}" -: "${LISTEN_PORT:=7770}" - -# Devices: ROCm0/ROCm1 are the two R9700s; the Raphael iGPU enumerates as -# ROCm2 (gfx1036) and is hidden from HIP so nothing can drift onto it. -: "${HIP_VISIBLE_DEVICES:=0,1}" -: "${DEVICES:=ROCm0,ROCm1}" -: "${SPLIT_MODE:=tensor}" -: "${TENSOR_SPLIT:=}" -: "${NGL:=999}" -: "${FIT:=off}" -: "${FLASH_ATTN:=on}" - -: "${CTX_SIZE:=131072}" -: "${PARALLEL:=1}" -: "${BATCH_SIZE:=2048}" -: "${UBATCH_SIZE:=512}" -: "${NUM_THREADS:=8}" -: "${THREAD_BATCH:=8}" -: "${CACHE_TYPE_K:=f16}" -: "${CACHE_TYPE_V:=f16}" -: "${CTX_CHECKPOINTS:=8}" -: "${CACHE_RAM:=8192}" - -# Sampling: the checkpoint's instruct table. -: "${TEMPERATURE:=0.7}" -: "${TOP_P:=0.8}" -: "${TOP_K:=20}" -: "${MIN_P:=0.0}" -: "${PRESENCE_PENALTY:=1.5}" - -# Reasoning: the model is an agent checkpoint; keep the trace and project it -# into the API's reasoning_content field rather than literal thinking tags. -: "${REASONING:=on}" -: "${REASONING_FORMAT:=deepseek}" -: "${REASONING_BUDGET:=-1}" - -# Chat template: empty means "use the template embedded in the GGUF". -: "${CHAT_TEMPLATE_FILE:=}" -: "${CHAT_TEMPLATE_KWARGS:={\"enable_thinking\":true,\"preserve_thinking\":true}}" - -# MTP draft head (separate GGUF, --spec-type draft-mtp). SPEC_DRAFT_DEVICE -# must name exactly one device: under tensor mode an unsplit draft joins the -# meta-device KV allocation and aborts while building the KV cache -# (GGML_ASSERT(buffer) in ggml-backend.cpp, 2026-09-15). -: "${SPEC_TYPE:=draft-mtp}" -: "${SPEC_DRAFT_N_MAX:=6}" -: "${SPEC_DRAFT_N_MIN:=0}" -: "${SPEC_DRAFT_P_MIN:=0.6}" -: "${SPEC_DRAFT_P_SPLIT:=0.10}" -: "${SPEC_DRAFT_NGL:=999}" -: "${SPEC_DRAFT_DEVICE:=}" -: "${SPEC_DRAFT_TYPE_K:=f16}" -: "${SPEC_DRAFT_TYPE_V:=f16}" -: "${SPEC_DRAFT_POLL:=1}" -: "${SPEC_DRAFT_POLL_BATCH:=1}" - -# Text only: the server must not try to autoload the vision tower. Vision -# stays on frank-dragoon (Dankclaw), which is where the fleet routes images. -NO_MMPROJ="${NO_MMPROJ:-1}" - -export HIP_VISIBLE_DEVICES - -MODEL="$MODEL_DIR/$MODEL_FILE" -if [[ ! -x "$LLAMA_BIN" ]]; then - echo "missing executable runtime: $LLAMA_BIN" >&2 - echo "build it with: cd $(dirname "$(dirname "$LLAMA_BIN")")/.. && env JOBS=8 ./scripts/build-rocmfp4-rocm714-local.sh" >&2 - exit 1 -fi -if [[ ! -r "$MODEL" ]]; then - echo "missing or unreadable required artifact: $MODEL" >&2 - echo "fetch it with: bash $SCRIPT_DIR/download-q8agent.sh" >&2 - exit 1 -fi -if [[ -n "$CHAT_TEMPLATE_FILE" && ! -r "$CHAT_TEMPLATE_FILE" ]]; then - echo "missing or unreadable chat template: $CHAT_TEMPLATE_FILE" >&2 - exit 1 -fi - -# --- speculative decoding (MTP draft head) -------------------------------- -SPEC_ARGS=() -if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then - SPEC_ARGS=( - --spec-type "$SPEC_TYPE" - --spec-draft-n-max "$SPEC_DRAFT_N_MAX" - --spec-draft-n-min "$SPEC_DRAFT_N_MIN" - --spec-draft-p-min "$SPEC_DRAFT_P_MIN" - --spec-draft-p-split "$SPEC_DRAFT_P_SPLIT" - ) - if [[ "$SPEC_TYPE" == *draft-mtp* ]]; then - DRAFT_MODEL="$MODEL_DIR/$DRAFT_MODEL_FILE" - if [[ ! -r "$DRAFT_MODEL" ]]; then - echo "missing or unreadable draft model: $DRAFT_MODEL" >&2 - exit 1 - fi - SPEC_ARGS+=( - --spec-draft-model "$DRAFT_MODEL" - --spec-draft-ngl "$SPEC_DRAFT_NGL" - --spec-draft-type-k "$SPEC_DRAFT_TYPE_K" - --spec-draft-type-v "$SPEC_DRAFT_TYPE_V" - --spec-draft-poll "$SPEC_DRAFT_POLL" - --spec-draft-poll-batch "$SPEC_DRAFT_POLL_BATCH" - ) - if [[ -n "$SPEC_DRAFT_DEVICE" ]]; then - SPEC_ARGS+=(--spec-draft-device "$SPEC_DRAFT_DEVICE") - fi - fi -fi - -# --- template + placement ------------------------------------------------- -TEMPLATE_ARGS=() -if [[ -n "$CHAT_TEMPLATE_FILE" ]]; then - TEMPLATE_ARGS+=(--chat-template-file "$CHAT_TEMPLATE_FILE") - TEMPLATE_ARGS+=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS") -fi - -PLACEMENT_ARGS=(--device "$DEVICES" --split-mode "$SPLIT_MODE") -if [[ -n "$TENSOR_SPLIT" ]]; then - PLACEMENT_ARGS+=(--tensor-split "$TENSOR_SPLIT") -fi - -MMPROJ_ARGS=() -if [[ "$NO_MMPROJ" == "1" ]]; then - MMPROJ_ARGS=(--no-mmproj) -fi - -exec "$LLAMA_BIN" \ - --model "$MODEL" \ - --alias "$ALIAS" \ - --host "$BIND_ADDR" \ - --port "$LISTEN_PORT" \ - --jinja \ - --gpu-layers "$NGL" \ - --fit "$FIT" \ - --parallel "$PARALLEL" \ - --ctx-size "$CTX_SIZE" \ - --batch-size "$BATCH_SIZE" \ - --ubatch-size "$UBATCH_SIZE" \ - --flash-attn "$FLASH_ATTN" \ - --cache-type-k "$CACHE_TYPE_K" \ - --cache-type-v "$CACHE_TYPE_V" \ - --threads "$NUM_THREADS" \ - --threads-batch "$THREAD_BATCH" \ - --ctx-checkpoints "$CTX_CHECKPOINTS" \ - --cache-ram "$CACHE_RAM" \ - --cache-prompt \ - --metrics \ - --temp "$TEMPERATURE" \ - --top-p "$TOP_P" \ - --top-k "$TOP_K" \ - --min-p "$MIN_P" \ - --presence-penalty "$PRESENCE_PENALTY" \ - --reasoning "$REASONING" \ - --reasoning-format "$REASONING_FORMAT" \ - --reasoning-budget "$REASONING_BUDGET" \ - "${TEMPLATE_ARGS[@]}" \ - "${PLACEMENT_ARGS[@]}" \ - "${MMPROJ_ARGS[@]}" \ - "${SPEC_ARGS[@]}" diff --git a/tests/.AGENTTRASH/frank_medium_proxy.bats b/tests/.AGENTTRASH/frank_medium_proxy.bats deleted file mode 100755 index f50bc36..0000000 --- a/tests/.AGENTTRASH/frank_medium_proxy.bats +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env bats -# Seam: the frank-medium proxy's routing decisions. -# The proxy is the only thing between the fleet's clients and the two -# single-GPU replicas, and its routing rules carry the performance work: spread -# single-turn requests (batch members, one-shot questions) so both cards prefill -# in parallel, and pin continuations so an agent loop keeps hitting the replica -# whose prefix cache holds its context. This seam drives the routing policy -# directly; the first version of these rules shipped a bug that only appeared on -# a continuation request, which is exactly the class of mistake a test this -# cheap should catch. - -load helpers - -ROUTING="$REPO_ROOT/dankserver/frank-medium/proxy/routing.py" -PROXY="$REPO_ROOT/dankserver/frank-medium/proxy/affinity-proxy.py" - -setup() { - DRIVER="$BATS_TEST_TMPDIR/route.py" - touch "$DRIVER" -} - -@test "the proxy and its routing policy exist and compile" { - [ -f "$ROUTING" ] - [ -f "$PROXY" ] - python3 -m py_compile "$ROUTING" - python3 -m py_compile "$PROXY" - # The policy must not drag the HTTP stack in: it is imported by tests and - # reasoned about without aiohttp, a network or a GPU. - ! grep -q 'import aiohttp' "$ROUTING" -} - -@test "a batch of single-turn requests spreads across both cards" { - cat > "$DRIVER" < "$DRIVER" < "$DRIVER" < replica 1; both down -> -1, and the proxy answers - # 503 rather than forwarding into the void. - [ "$output" = "1 -1" ] - done -} - -@test "affinity is deterministic per prefix, roundrobin is not" { - cat > "$DRIVER" <