diff --git a/scripts/deploy.sh b/scripts/deploy.sh index d7993eb..c6008d1 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -17,8 +17,8 @@ # FRANK_SUDO_PW=... ./deploy.sh # # Targets (2026-08-18 topology): -# dankserver - llama.cpp FastMTP (frank-default-large.service, port 7770) -# dankclaw - CIRU ROCmFPX llama.cpp (frank-default-small.service, port 52625) +# dankserver - llama.cpp Qwen3.6-35B-A3B Q6_K_P (frank-default-small.service, port 7770) +# dankclaw - CIRU ROCmFPX llama.cpp Qwen3.8-27B (frank-default-large.service, port 52625) set -euo pipefail # --------------------------------------------------------------------------- @@ -29,16 +29,17 @@ declare -A REMOTE_USER REMOTE_HOST REMOTE_DIR SERVICE SERVICE_HEALTH DROP_CACHES REMOTE_USER[dankserver]=dank REMOTE_HOST[dankserver]=dankserver REMOTE_DIR[dankserver]="/home/dank/vLLM" -SERVICE[dankserver]="frank-default-large" +SERVICE[dankserver]="frank-default-small" SERVICE_HEALTH[dankserver]="http://127.0.0.1:7770/health" DROP_CACHES[dankserver]=0 -# Cold load: --no-mmap reads 15.7 GB through RAM; allow 10 minutes. -HEALTH_WAIT[dankserver]=300 +# Cold load: --no-mmap reads the 30.6 GB Q6_K_P through RAM (MoE experts on +# CPU); allow 20 minutes. +HEALTH_WAIT[dankserver]=600 REMOTE_USER[dankclaw]=dank REMOTE_HOST[dankclaw]=dankclaw REMOTE_DIR[dankclaw]="/home/dank/vllmScripts" -SERVICE[dankclaw]="frank-default-small" +SERVICE[dankclaw]="frank-default-large" SERVICE_HEALTH[dankclaw]="http://127.0.0.1:52625/health" DROP_CACHES[dankclaw]=0 # CIRU llama.cpp (ROCm 7.15) loads the 34.9 GiB ActiveFPX payload + MTP diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index bbb8f45..95c6415 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -1,62 +1,72 @@ -# Tunables for frank-default-large (llama.cpp, dankserver, RTX 3090). +# Tunables for frank-default-large (CIRU llama.cpp, dankclaw, Strix Halo +# gfx1151). # -# Runtime: patched llama-server (HauhauCS FastMTP build) serving -# HauhauCS/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-MTP-GGUF IQ4_XS with -# the FastMTP 32K draft sidecar and the ngram-mod fallback. Single machine, -# no pipeline split. Alias frank-default-large, port 7770. +# Runtime: the PromptForge llama-server built from ciru-ai/ROCmFPX tag +# qwen3.8-activefpx-promptforge-v2.3 serving +# jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge (v2). ActiveFPX compact +# GGUF with PromptForge companions for CIRU DualView prompt acceleration and +# native depth-4 MTP. Alias frank-default-large, port 52625. +# +# This runtime replaced the retired vLLM service that previously occupied +# this slot. See docs/ciru-activefpx-promptforge.md. # # WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The ROCm # HIP runtime reads those bare env vars (see the 2026-08-13 TEMP incident). -# This box is NVIDIA/CUDA, but the rule still applies to keep the env files -# portable. +# llama.cpp reads --host/--port/--temp as CLI flags, never as env vars. -# Model files (downloaded to this directory from the HF repo). -MODEL_DIR=/home/dank/models/hauhaucs-qwen38 -MODEL_FILE=Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-IQ4_XS.gguf -DRAFT_FILE=Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-FastMTP-32K.gguf -# Patched llama.cpp binary. The FastMTP sidecar needs the patch; see AGENTS.md. -# REQUIRED build flags: -DGGML_CUDA_FA=ON -DGGML_CUDA_FA_ALL_QUANTS=ON. -# Without FA_ALL_QUANTS the q8_0 KV + flash attention fall back to a slow -# path and prefill collapses ~12x (95 t/s vs 1300+ t/s at 8K prompt, -# measured 2026-08-17). Do not rebuild without it. -LLAMA_BIN=/home/dank/llama.cpp-fastmtp/build/bin/llama-server +# CIRU llama-server binary (ROCmFPX v2.3 build, see docs/ciru runtime page). +# Built against the private TheRock ROCm 7.15 SDK kept in the retired +# ling3-ciru venv. ROCm 7.15 is required for the advertised Strix Halo +# performance; the system ROCm 7.2.4 selects slower/fallback kernels. +LLAMA_BIN=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server + +# Model files. The three-file payload (v2 GGUF + FFN + GDN) enables CIRU +# DualView prompt acceleration; omit the two .pfs files to run the GGUF alone +# and save ~19.7 GiB of memory at the cost of prompt throughput. +MODEL_DIR=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX +MODEL_FILE=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v2.gguf +FFN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-FFN.pfs +GDN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-GDN.pfs + +# ROCm 7.15 SDK paths (inside the retired ling3-ciru venv). The serve script +# exports ROCM_PATH and LD_LIBRARY_PATH from these so the 7.15 libraries +# precede any system ROCm 7.2.4. +ROCM_DEVEL=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel +ROCM_LIB=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries +ROCM_CORE=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core ALIAS=frank-default-large BIND_ADDR=0.0.0.0 -LISTEN_PORT=7770 +LISTEN_PORT=52625 -# Context, single slot. 131072 (128K) leaves ~1.4 GiB VRAM headroom on the -# 3090 with IQ4_XS + FastMTP + K q8_0 / V f16 KV. Lower to 122880 (120K) for -# ~1.8 GiB headroom. KV per token = 16 attention layers x 4 KV heads x 256 -# dim: K q8_0 (1088 B/token) + V f16 (2048 B/token) = 3136 B/token/layer. -CTX_SIZE=131072 -PARALLEL=1 - -# Speculative decoding. FastMTP (MTP depth 2) with the ngram-mod fallback. -SPEC_DEFAULT=1 -MTP=1 -SPEC_DRAFT_N_MAX=2 -SPEC_DRAFT_NGL=all -SPEC_DRAFT_P_MIN=0 -SPEC_NGRAM_MOD_N_MIN=48 -SPEC_NGRAM_MOD_N_MAX=64 -SPEC_NGRAM_MOD_N_MATCH=24 - -# KV quantization on GPU: K q8_0 + V f16 (user request: K=8, V=16). -CACHE_TYPE_K=q8_0 -CACHE_TYPE_V=f16 - -# Batch sizing (user request). +# Serving: 256K context (native max, n_ctx_train = 262144), 2 slots, f16 KV, +# context checkpoints, prompt cache. Native MTP depth-4. The Strix Halo is +# unified memory; the addressable pool is 96 GiB (VRAM is a pointer). KV at +# 262144 x 2 slots is ~32 GiB (64 KiB/cell, 16 full-attn layers); total +# sits well inside the 96 GiB pool. +CTX_SIZE=262144 +PARALLEL=2 BATCH_SIZE=2048 -UBATCH_SIZE=1024 +UBATCH_SIZE=2048 +CACHE_TYPE_K=f16 +CACHE_TYPE_V=f16 +CTX_CHECKPOINTS=32 +CACHE_RAM=8192 -# Sampling - Qwen3.8 thinking-mode defaults. -TEMPERATURE=1.0 -TOP_K=20 +# Native MTP draft (depth-4, greedy, ROCm-safe CPU argmax fast path). +SPEC_DRAFT_N_MAX=4 +SPEC_DRAFT_N_MIN=0 +SPEC_DRAFT_P_MIN=0.0 +SPEC_DRAFT_P_SPLIT=0.10 + +# Sampling (model card defaults). +TEMPERATURE=0.7 TOP_P=0.95 -MIN_P=0.0 -THREADS=8 -THREADS_HTTP=1 +# Thinking enabled (user request, 2026-08-18). deepseek format moves the +# reasoning into message.reasoning_content (OpenAI-compatible); +# none would leave the thought tags unparsed in content. +REASONING=on +REASONING_FORMAT=deepseek VERBOSITY=4 diff --git a/scripts/frank-default-large.service b/scripts/frank-default-large.service index 78ac03f..70cdd45 100644 --- a/scripts/frank-default-large.service +++ b/scripts/frank-default-large.service @@ -1,5 +1,5 @@ [Unit] -Description=llama.cpp Qwen3.8-27B Aggressive IQ4_XS + FastMTP (dankserver, frank-default-large) +Description=CIRU llama.cpp Qwen3.8-27B ActiveFPX PromptForge MTP (dankclaw, frank-default-large) After=network-online.target Wants=network-online.target @@ -7,8 +7,8 @@ Wants=network-online.target Type=simple User=dank Group=dank -Environment="FRANK_LARGE_ENV_FILE=/home/dank/vLLM/scripts/frank-default-large.env" -ExecStart=/home/dank/vLLM/scripts/serve-frank-large.sh +Environment="FRANK_LARGE_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-large.env" +ExecStart=/home/dank/vllmScripts/scripts/serve-frank-large.sh Restart=on-failure RestartSec=10 diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index 7843057..7528922 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -1,71 +1,61 @@ -# Tunables for frank-default-small (CIRU llama.cpp, dankclaw, Strix Halo -# gfx1151). +# Tunables for frank-default-small (HauhauCS Qwen3.6-35B-A3B Q6_K_P, +# ngram-mod spec, Dankserver, RTX 3090). # -# Runtime: the PromptForge llama-server built from ciru-ai/ROCmFPX tag -# qwen3.8-activefpx-promptforge-v2.3 serving -# jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge (v2). ActiveFPX compact -# GGUF with PromptForge companions for CIRU DualView prompt acceleration and -# native depth-4 MTP. Alias frank-default-small, port 52625. +# Runtime: upstream llama.cpp llama-server (b10121). RTX 3090 24 GB; +# Ryzen 7 7700X; 62 GB RAM. MoE experts offloaded to CPU (--n-cpu-moe 36). +# Restored 2026-08-18 (was the pre-2026-08-17-redo frank-default-small). # -# This runtime replaced the retired vLLM service that previously occupied -# this slot. See docs/ciru-activefpx-promptforge.md. -# -# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The ROCm -# HIP runtime reads those bare env vars (see the 2026-08-13 TEMP incident). -# llama.cpp reads --host/--port/--temp as CLI flags, never as env vars. +# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The +# ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP +# incident). This box is NVIDIA/CUDA, but the rule keeps the env portable. -# CIRU llama-server binary (ROCmFPX v2.3 build, see docs/ciru runtime page). -# Built against the private TheRock ROCm 7.15 SDK kept in the retired -# ling3-ciru venv. ROCm 7.15 is required for the advertised Strix Halo -# performance; the system ROCm 7.2.4 selects slower/fallback kernels. -LLAMA_BIN=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server - -# Model files. The three-file payload (v2 GGUF + FFN + GDN) enables CIRU -# DualView prompt acceleration; omit the two .pfs files to run the GGUF alone -# and save ~19.7 GiB of memory at the cost of prompt throughput. -MODEL_DIR=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX -MODEL_FILE=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v2.gguf -FFN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-FFN.pfs -GDN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-GDN.pfs - -# ROCm 7.15 SDK paths (inside the retired ling3-ciru venv). The serve script -# exports ROCM_PATH and LD_LIBRARY_PATH from these so the 7.15 libraries -# precede any system ROCm 7.2.4. -ROCM_DEVEL=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel -ROCM_LIB=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries -ROCM_CORE=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core +# Model files (already on disk; Q6_K_P GGUF + f16 mmproj vision encoder). +MODEL_DIR=/home/dank/models +MODEL_FILE=Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-Q6_K_P.gguf +MMPROJ_FILE=mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf +# Upstream llama.cpp binary (b10121). +LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server ALIAS=frank-default-small BIND_ADDR=0.0.0.0 -LISTEN_PORT=52625 +LISTEN_PORT=7770 -# Serving: 128K context (user request, 2026-08-18; model card default is -# 64K), single slot, f16 KV, context checkpoints, prompt cache. Native MTP -# depth-4. KV at 128K is 8 GiB (64 KiB/token); estimated total ~43 GiB of -# the 64 GiB SVM pool, leaving ~21 GiB headroom. -CTX_SIZE=131072 -PARALLEL=1 -BATCH_SIZE=2048 -UBATCH_SIZE=2048 +# 4 parallel slots x ~175104 ctx each = ~700000 total context pool. +# Dual-f16 KV at 700000 cells leaves ~2.3 GiB free on the 3090. +CTX_SIZE=700000 +PARALLEL=4 + +# Spec: ngram-mod (n_min=48, n_max=64, n_match=24). +SPEC_TYPE=ngram-mod +NGRAM_MOD_N_MIN=48 +NGRAM_MOD_N_MAX=64 +NGRAM_MOD_N_MATCH=24 + +# Expert offload: 36 of 40 layers keep MoE weights on CPU; 4 on GPU. +N_CPU_MOE=36 +N_GPU_LAYERS=all + +# KV cache on GPU, both K and V at f16. CACHE_TYPE_K=f16 CACHE_TYPE_V=f16 -CTX_CHECKPOINTS=32 -CACHE_RAM=8192 - -# Native MTP draft (depth-4, greedy, ROCm-safe CPU argmax fast path). -SPEC_DRAFT_N_MAX=4 -SPEC_DRAFT_N_MIN=0 -SPEC_DRAFT_P_MIN=0.0 -SPEC_DRAFT_P_SPLIT=0.10 - -# Sampling (model card defaults). -TEMPERATURE=0.7 -TOP_P=0.95 - -# Thinking enabled (user request, 2026-08-18). deepseek format moves the -# reasoning into message.reasoning_content (OpenAI-compatible); -# none would leave the thought tags unparsed in content. -REASONING=on -REASONING_FORMAT=deepseek VERBOSITY=4 +# Store SWA/context checkpoints in RAM to hold peak GPU memory down. +CTX_CHECKPOINTS=4 + +# Big prefill batches (ubatch 3072 is the top prefill tier). +BATCH_SIZE=8192 +UBATCH_SIZE=3072 + +# Eager RAM load (MoE experts on CPU; ~31 GB RAM). +NO_MMAP=1 + +# Sampling (HauhauCS Qwen3.6-35B-A3B model card, thinking-general). +TEMPERATURE=1.0 +TOP_K=20 +TOP_P=0.95 +MIN_P=0.0 + +# Compute threads lowered to 7 so other processes keep a core. +THREADS=7 +THREADS_HTTP=8 diff --git a/scripts/frank-default-small.service b/scripts/frank-default-small.service index 4946b1f..0a27cd2 100644 --- a/scripts/frank-default-small.service +++ b/scripts/frank-default-small.service @@ -1,5 +1,5 @@ [Unit] -Description=CIRU llama.cpp Qwen3.8-27B ActiveFPX PromptForge ngram MTP (dankclaw, frank-default-small) +Description=llama.cpp Qwen3.6-35B-A3B Q6_K_P + ngram-mod (dankserver, frank-default-small) After=network-online.target Wants=network-online.target @@ -7,8 +7,8 @@ Wants=network-online.target Type=simple User=dank Group=dank -Environment="FRANK_SMALL_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-small.env" -ExecStart=/home/dank/vllmScripts/scripts/serve-frank-small.sh +Environment="FRANK_SMALL_ENV_FILE=/home/dank/vLLM/scripts/frank-default-small.env" +ExecStart=/home/dank/vLLM/scripts/serve-frank-small.sh Restart=on-failure RestartSec=10 diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 38eb5d2..25b5eee 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -1,14 +1,16 @@ #!/usr/bin/env bash -# Serve the Qwen3.8-27B HauhauCS Aggressive IQ4_XS on Dankserver (RTX 3090). +# Serve jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge as frank-default-large +# via the CIRU ROCmFPX llama.cpp runtime on Dankclaw (Strix Halo gfx1151). # -# Runs a patched llama-server (HauhauCS FastMTP build, commit 4df29be + -# HauhauCS-FastMTP-llama.cpp.patch) with the FastMTP draft sidecar and the -# ngram-mod fallback. There is no pipeline split: the whole model lives on -# the 3090. +# Runtime: the PromptForge llama-server built from ciru-ai/ROCmFPX tag +# qwen3.8-activefpx-promptforge-v2.3 (commit 103af8b41) against the private +# TheRock ROCm 7.15 SDK (kept in the retired ling3-ciru venv). The model is +# a GGUF with optional PromptForge companion views (FFN/GDN .pfs) that enable +# CIRU DualView prompt acceleration. Native depth-4 MTP speculative decoding. # -# The FastMTP draft sidecar requires the patched executable. Loading the -# draft with the stock binary fails with "expected 5120, 248320, got 5120, -# 32768" (see the HauhauCS repo README). +# The model card requires ROCm 7.15 (not the system ROCm 7.2.4) for the +# advertised Strix Halo performance. LD_LIBRARY_PATH therefore points at the +# 7.15 SDK libraries first. An older system ROCm must not precede them. # # Tunables come from frank-default-large.env in this directory unless # FRANK_LARGE_ENV_FILE overrides that path. @@ -24,44 +26,52 @@ if [[ -f "$ENV_FILE" ]]; then fi # --- defaults (used when the env file does not set them) ------------------- -: "${MODEL_DIR:=/home/dank/models/hauhaucs-qwen38}" -: "${MODEL_FILE:=Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-IQ4_XS.gguf}" -: "${DRAFT_FILE:=Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-FastMTP-32K.gguf}" -: "${LLAMA_BIN:=/home/dank/llama.cpp-fastmtp/build/bin/llama-server}" +: "${LLAMA_BIN:=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server}" +: "${MODEL_DIR:=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX}" +: "${MODEL_FILE:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v2.gguf}" +: "${FFN_PFS:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-FFN.pfs}" +: "${GDN_PFS:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-GDN.pfs}" +: "${ROCM_DEVEL:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel}" +: "${ROCM_LIB:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries}" +: "${ROCM_CORE:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core}" : "${ALIAS:=frank-default-large}" : "${BIND_ADDR:=0.0.0.0}" -: "${LISTEN_PORT:=7770}" -: "${CTX_SIZE:=131072}" -: "${PARALLEL:=1}" -: "${SPEC_DEFAULT:=1}" -: "${MTP:=1}" -: "${SPEC_DRAFT_N_MAX:=2}" -: "${SPEC_DRAFT_NGL:=all}" -: "${SPEC_DRAFT_P_MIN:=0}" -: "${SPEC_NGRAM_MOD_N_MIN:=48}" -: "${SPEC_NGRAM_MOD_N_MAX:=64}" -: "${SPEC_NGRAM_MOD_N_MATCH:=24}" -: "${CACHE_TYPE_K:=q8_0}" -: "${CACHE_TYPE_V:=f16}" +: "${LISTEN_PORT:=52625}" +: "${CTX_SIZE:=262144}" +: "${PARALLEL:=2}" : "${BATCH_SIZE:=2048}" -: "${UBATCH_SIZE:=1024}" -: "${TEMPERATURE:=1.0}" -: "${TOP_K:=20}" +: "${UBATCH_SIZE:=2048}" +: "${CACHE_TYPE_K:=f16}" +: "${CACHE_TYPE_V:=f16}" +: "${CTX_CHECKPOINTS:=32}" +: "${CACHE_RAM:=8192}" +: "${SPEC_DRAFT_N_MAX:=4}" +: "${SPEC_DRAFT_N_MIN:=0}" +: "${SPEC_DRAFT_P_MIN:=0.0}" +: "${SPEC_DRAFT_P_SPLIT:=0.10}" +: "${TEMPERATURE:=0.7}" : "${TOP_P:=0.95}" -: "${MIN_P:=0.0}" -: "${THREADS:=8}" -: "${THREADS_HTTP:=1}" +: "${REASONING:=on}" +: "${REASONING_FORMAT:=deepseek}" : "${VERBOSITY:=4}" MODEL="$MODEL_DIR/$MODEL_FILE" -DRAFT="$MODEL_DIR/$DRAFT_FILE" +export PROMPTFORGE_SIDECAR="$MODEL_DIR/$FFN_PFS" +export PROMPTFORGE_GDN_SIDECAR="$MODEL_DIR/$GDN_PFS" +export PROMPTFORGE_MODE="m2048_fused_tail1476" +export GGML_CUDA_GRAPH_OPT=0 +export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1 if [[ ! -f "$MODEL" ]]; then echo "missing required file: $MODEL" >&2 exit 1 fi -if [[ ! -f "$DRAFT" ]]; then - echo "missing draft file: $DRAFT" >&2 +if [[ ! -f "$PROMPTFORGE_SIDECAR" ]]; then + echo "missing PromptForge FFN companion: $PROMPTFORGE_SIDECAR" >&2 + exit 1 +fi +if [[ ! -f "$PROMPTFORGE_GDN_SIDECAR" ]]; then + echo "missing PromptForge GDN companion: $PROMPTFORGE_GDN_SIDECAR" >&2 exit 1 fi if [[ ! -x "$LLAMA_BIN" ]]; then @@ -69,40 +79,41 @@ if [[ ! -x "$LLAMA_BIN" ]]; then exit 1 fi -# --- speculative decoding: FastMTP (MTP) first, then ngram-mod fallback ---- -SPEC_ARGS=() -SPEC_ARGS+=(--spec-type draft-mtp) -SPEC_ARGS+=(--spec-draft-model "$DRAFT") -SPEC_ARGS+=(--spec-draft-n-max "$SPEC_DRAFT_N_MAX") -SPEC_ARGS+=(--spec-draft-ngl "$SPEC_DRAFT_NGL") -SPEC_ARGS+=(--spec-draft-p-min "$SPEC_DRAFT_P_MIN") -if [[ "$SPEC_DEFAULT" == "1" ]]; then - SPEC_ARGS+=(--spec-default) - SPEC_ARGS+=(--spec-ngram-mod-n-min "$SPEC_NGRAM_MOD_N_MIN") - SPEC_ARGS+=(--spec-ngram-mod-n-max "$SPEC_NGRAM_MOD_N_MAX") - SPEC_ARGS+=(--spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH") -fi +# ROCm 7.15 SDK libraries must precede any system ROCm in LD_LIBRARY_PATH. +export ROCM_PATH="$ROCM_DEVEL" +export LD_LIBRARY_PATH="${ROCM_LIB}/lib:${ROCM_CORE}/lib:${ROCM_DEVEL}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" exec "$LLAMA_BIN" \ --model "$MODEL" \ - "${SPEC_ARGS[@]}" \ - --gpu-layers all \ - --no-mmap \ - --flash-attn on \ - --ctx-size "$CTX_SIZE" \ - --parallel "$PARALLEL" \ - --cache-type-k "$CACHE_TYPE_K" \ - --cache-type-v "$CACHE_TYPE_V" \ - --batch-size "$BATCH_SIZE" \ - --ubatch-size "$UBATCH_SIZE" \ - --temp "$TEMPERATURE" \ - --top-k "$TOP_K" \ - --top-p "$TOP_P" \ - --min-p "$MIN_P" \ - --threads "$THREADS" \ - --threads-http "$THREADS_HTTP" \ - --jinja \ --alias "$ALIAS" \ - -lv "$VERBOSITY" \ --host "$BIND_ADDR" \ - --port "$LISTEN_PORT" + --port "$LISTEN_PORT" \ + -dev ROCm0 \ + -ngl 999 \ + -c "$CTX_SIZE" \ + -b "$BATCH_SIZE" \ + -ub "$UBATCH_SIZE" \ + -fa on \ + -ctk "$CACHE_TYPE_K" \ + -ctv "$CACHE_TYPE_V" \ + -np "$PARALLEL" \ + -ctxcp "$CTX_CHECKPOINTS" \ + --cache-ram "$CACHE_RAM" \ + --cache-prompt \ + --metrics \ + --spec-type draft-mtp \ + --spec-draft-device ROCm0 \ + --spec-draft-ngl 999 \ + --spec-draft-type-k f16 \ + --spec-draft-type-v f16 \ + --spec-draft-n-max "$SPEC_DRAFT_N_MAX" \ + --spec-draft-n-min "$SPEC_DRAFT_N_MIN" \ + --spec-draft-p-min "$SPEC_DRAFT_P_MIN" \ + --spec-draft-p-split "$SPEC_DRAFT_P_SPLIT" \ + --spec-draft-backend-sampling \ + --temp "$TEMPERATURE" \ + --top-p "$TOP_P" \ + --reasoning "$REASONING" \ + --reasoning-format "$REASONING_FORMAT" \ + --reasoning-budget -1 \ + -lv "$VERBOSITY" diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index f56e1b2..6bb9f19 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -1,19 +1,13 @@ #!/usr/bin/env bash -# Serve jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge via the CIRU ROCmFPX -# llama.cpp runtime on Dankclaw (Strix Halo gfx1151). +# Serve HauhauCS Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q6_K_P with +# ngram-mod speculative decoding on Dankserver (RTX 3090, MoE experts on CPU). # -# Runtime: the PromptForge llama-server built from ciru-ai/ROCmFPX tag -# qwen3.8-activefpx-promptforge-v2.3 (commit 103af8b41) against the private -# TheRock ROCm 7.15 SDK (kept in the retired ling3-ciru venv). The model is -# a GGUF with optional PromptForge companion views (FFN/GDN .pfs) that enable -# CIRU DualView prompt acceleration. Native depth-4 MTP speculative decoding. +# Runtime: upstream llama.cpp llama-server (b10121). The model is a 35B-total / +# 3B-active MoE vision-language model; --n-cpu-moe keeps most expert weights on +# CPU so the non-expert tensors + KV cache fit the 24 GB 3090. # -# The model card requires ROCm 7.15 (not the system ROCm 7.2.4) for the -# advertised Strix Halo performance. LD_LIBRARY_PATH therefore points at the -# 7.15 SDK libraries first. An older system ROCm must not precede them. -# -# Tunables come from frank-default-small.env in this directory unless -# FRANK_SMALL_ENV_FILE overrides that path. +# Tunables come from frank-default-small.env in this directory unless the +# FRANK_SMALL_ENV_FILE environment variable overrides that path. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -25,95 +19,75 @@ if [[ -f "$ENV_FILE" ]]; then set +a fi -# --- defaults (used when the env file does not set them) ------------------- -: "${LLAMA_BIN:=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server}" -: "${MODEL_DIR:=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX}" -: "${MODEL_FILE:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v2.gguf}" -: "${FFN_PFS:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-FFN.pfs}" -: "${GDN_PFS:=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-GDN.pfs}" -: "${ROCM_DEVEL:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel}" -: "${ROCM_LIB:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries}" -: "${ROCM_CORE:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core}" +# --- defaults (used when the env file does not set them) ----------------- +: "${MODEL_DIR:=/home/dank/models}" +: "${MODEL_FILE:=Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-Q6_K_P.gguf}" +: "${MMPROJ_FILE:=mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf}" +: "${LLAMA_SERVER:=/home/dank/llama.cpp/build/bin/llama-server}" : "${ALIAS:=frank-default-small}" : "${BIND_ADDR:=0.0.0.0}" -: "${LISTEN_PORT:=52625}" -: "${CTX_SIZE:=131072}" -: "${PARALLEL:=1}" -: "${BATCH_SIZE:=2048}" -: "${UBATCH_SIZE:=2048}" +: "${LISTEN_PORT:=7770}" +: "${CTX_SIZE:=700000}" +: "${PARALLEL:=4}" +: "${SPEC_TYPE:=ngram-mod}" +: "${NGRAM_MOD_N_MIN:=48}" +: "${NGRAM_MOD_N_MAX:=64}" +: "${NGRAM_MOD_N_MATCH:=24}" : "${CACHE_TYPE_K:=f16}" : "${CACHE_TYPE_V:=f16}" -: "${CTX_CHECKPOINTS:=32}" -: "${CACHE_RAM:=8192}" -: "${SPEC_DRAFT_N_MAX:=4}" -: "${SPEC_DRAFT_N_MIN:=0}" -: "${SPEC_DRAFT_P_MIN:=0.0}" -: "${SPEC_DRAFT_P_SPLIT:=0.10}" -: "${TEMPERATURE:=0.7}" -: "${TOP_P:=0.95}" -: "${REASONING:=on}" -: "${REASONING_FORMAT:=deepseek}" : "${VERBOSITY:=4}" +: "${N_CPU_MOE:=36}" +: "${N_GPU_LAYERS:=all}" +: "${BATCH_SIZE:=8192}" +: "${UBATCH_SIZE:=3072}" +: "${CTX_CHECKPOINTS:=4}" +: "${NO_MMAP:=1}" +: "${TEMPERATURE:=1.0}" +: "${TOP_K:=20}" +: "${TOP_P:=0.95}" +: "${MIN_P:=0.0}" +: "${THREADS:=7}" +: "${THREADS_HTTP:=8}" MODEL="$MODEL_DIR/$MODEL_FILE" -export PROMPTFORGE_SIDECAR="$MODEL_DIR/$FFN_PFS" -export PROMPTFORGE_GDN_SIDECAR="$MODEL_DIR/$GDN_PFS" -export PROMPTFORGE_MODE="m2048_fused_tail1476" -export GGML_CUDA_GRAPH_OPT=0 -export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1 +MMPROJ="$MODEL_DIR/$MMPROJ_FILE" -if [[ ! -f "$MODEL" ]]; then - echo "missing required file: $MODEL" >&2 - exit 1 -fi -if [[ ! -f "$PROMPTFORGE_SIDECAR" ]]; then - echo "missing PromptForge FFN companion: $PROMPTFORGE_SIDECAR" >&2 - exit 1 -fi -if [[ ! -f "$PROMPTFORGE_GDN_SIDECAR" ]]; then - echo "missing PromptForge GDN companion: $PROMPTFORGE_GDN_SIDECAR" >&2 - exit 1 -fi -if [[ ! -x "$LLAMA_BIN" ]]; then - echo "missing llama-server binary: $LLAMA_BIN" >&2 +for f in "$MODEL" "$MMPROJ"; do + if [[ ! -f "$f" ]]; then + echo "missing required file: $f" >&2 + exit 1 + fi +done +if [[ ! -x "$LLAMA_SERVER" ]]; then + echo "missing llama-server binary: $LLAMA_SERVER" >&2 exit 1 fi -# ROCm 7.15 SDK libraries must precede any system ROCm in LD_LIBRARY_PATH. -export ROCM_PATH="$ROCM_DEVEL" -export LD_LIBRARY_PATH="${ROCM_LIB}/lib:${ROCM_CORE}/lib:${ROCM_DEVEL}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - -exec "$LLAMA_BIN" \ +exec "$LLAMA_SERVER" \ --model "$MODEL" \ + --mmproj "$MMPROJ" \ + --jinja \ + --spec-type "$SPEC_TYPE" \ + --spec-ngram-mod-n-min "$NGRAM_MOD_N_MIN" \ + --spec-ngram-mod-n-max "$NGRAM_MOD_N_MAX" \ + --spec-ngram-mod-n-match "$NGRAM_MOD_N_MATCH" \ + --cache-type-k "$CACHE_TYPE_K" \ + --cache-type-v "$CACHE_TYPE_V" \ + --verbosity "$VERBOSITY" \ + --ctx-size "$CTX_SIZE" \ + --parallel "$PARALLEL" \ + --n-cpu-moe "$N_CPU_MOE" \ + --gpu-layers "$N_GPU_LAYERS" \ + --batch-size "$BATCH_SIZE" \ + --ubatch-size "$UBATCH_SIZE" \ + --ctx-checkpoints "$CTX_CHECKPOINTS" \ + $([ "$NO_MMAP" = "1" ] && printf -- "--no-mmap") \ + --temp "$TEMPERATURE" \ + --top-k "$TOP_K" \ + --top-p "$TOP_P" \ + --min-p "$MIN_P" \ + --threads "$THREADS" \ + --threads-http "$THREADS_HTTP" \ --alias "$ALIAS" \ --host "$BIND_ADDR" \ - --port "$LISTEN_PORT" \ - -dev ROCm0 \ - -ngl 999 \ - -c "$CTX_SIZE" \ - -b "$BATCH_SIZE" \ - -ub "$UBATCH_SIZE" \ - -fa on \ - -ctk "$CACHE_TYPE_K" \ - -ctv "$CACHE_TYPE_V" \ - -np "$PARALLEL" \ - -ctxcp "$CTX_CHECKPOINTS" \ - --cache-ram "$CACHE_RAM" \ - --cache-prompt \ - --metrics \ - --spec-type draft-mtp \ - --spec-draft-device ROCm0 \ - --spec-draft-ngl 999 \ - --spec-draft-type-k f16 \ - --spec-draft-type-v f16 \ - --spec-draft-n-max "$SPEC_DRAFT_N_MAX" \ - --spec-draft-n-min "$SPEC_DRAFT_N_MIN" \ - --spec-draft-p-min "$SPEC_DRAFT_P_MIN" \ - --spec-draft-p-split "$SPEC_DRAFT_P_SPLIT" \ - --spec-draft-backend-sampling \ - --temp "$TEMPERATURE" \ - --top-p "$TOP_P" \ - --reasoning "$REASONING" \ - --reasoning-format "$REASONING_FORMAT" \ - --reasoning-budget -1 \ - -lv "$VERBOSITY" + --port "$LISTEN_PORT"