From d9af7f1d197b2bc13080b069da7b40a85575707f Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 17 Aug 2026 15:23:46 -0700 Subject: [PATCH] fleet redo 2026-08-17: retire PP+NPU, new topology - Move all previous scripts to old/scripts/ (retired PP/NPU-era) - frank-default-large: llama.cpp FastMTP on dankserver (HauhauCS Qwen3.8-27B IQ4_XS + FastMTP 32K draft + ngram-mod, K q8_0/V f16, 128K ctx, batch 2048/ubatch 1024, port 7770) - frank-default-small: vLLM ROCm on dankclaw (Avesed Qwen3.6-35B-A3B INT8 W8A8, full 262144 ctx, no KV quant, 4 seqs, batch 16384, ngram spec, port 52625) - retarget deploy.sh (dankserver/dankclaw, inline sudo, no helpers) - docs: deployment + index + AGENTS.md + README for new topology, superseded banners on retired docs, add frank-redo-2026-08-17 runbook --- AGENTS.md | 132 ++++++---- README.md | 40 ++- docs/FastFlowLM NPU Inference.md | 4 + .../Qwen3.8 PP Deployment.md | 4 + docs/dankserver-vram-growth.md | 4 + docs/deployment.md | 243 +++++------------- docs/frank-default-large-benchmark.md | 4 + docs/frank-redo-2026-08-17.md | 127 +++++++++ docs/index.md | 94 +++---- docs/qwen3.8-spec-matrix.md | 4 + .../scripts}/analyze-spec-matrix.py | 0 {scripts => old/scripts}/bench-frank-large.sh | 0 {scripts => old/scripts}/bench-high-ctx.sh | 0 .../scripts}/bench-pipeline-64k.sh | 0 old/scripts/deploy.sh | 196 ++++++++++++++ old/scripts/frank-default-large.env | 88 +++++++ .../scripts}/frank-default-large.ling.env | 0 old/scripts/frank-default-large.service | 16 ++ old/scripts/frank-default-small.env | 33 +++ old/scripts/frank-default-small.service | 17 ++ .../scripts}/frank-large-ling.service | 0 {scripts => old/scripts}/frank-pp-worker.env | 0 .../scripts}/frank-pp-worker.service | 0 {scripts => old/scripts}/llama-server.service | 0 {scripts => old/scripts}/pp-bench.py | 0 {scripts => old/scripts}/pp-test.sh | 0 {scripts => old/scripts}/prune-rpc-cache.sh | 0 {scripts => old/scripts}/serve-flm-small.sh | 0 {scripts => old/scripts}/serve-ling-flash.sh | 0 {scripts => old/scripts}/serve-ling3-vllm.sh | 0 .../scripts}/serve-llama-frank-large.sh | 0 .../scripts}/serve-llama-frank-small.sh | 0 {scripts => old/scripts}/serve-pp-head.sh | 0 {scripts => old/scripts}/serve-pp-worker.sh | 0 .../scripts}/serve-vllm-qwen36-dflash.sh | 0 {scripts => old/scripts}/spec-matrix-bench.sh | 0 .../scripts}/test_dspark_stream.py | 0 {scripts => old/scripts}/validate-vllm.sh | 0 scripts/deploy.sh | 88 ++----- scripts/frank-default-large.env | 92 +++---- scripts/frank-default-large.service | 6 +- scripts/frank-default-small.env | 69 ++--- scripts/frank-default-small.service | 8 +- scripts/serve-frank-large.sh | 108 ++++++++ scripts/serve-frank-small.sh | 53 ++++ 45 files changed, 961 insertions(+), 469 deletions(-) create mode 100644 docs/frank-redo-2026-08-17.md rename {scripts => old/scripts}/analyze-spec-matrix.py (100%) rename {scripts => old/scripts}/bench-frank-large.sh (100%) rename {scripts => old/scripts}/bench-high-ctx.sh (100%) rename {scripts => old/scripts}/bench-pipeline-64k.sh (100%) create mode 100755 old/scripts/deploy.sh create mode 100644 old/scripts/frank-default-large.env rename {scripts => old/scripts}/frank-default-large.ling.env (100%) create mode 100644 old/scripts/frank-default-large.service create mode 100644 old/scripts/frank-default-small.env create mode 100644 old/scripts/frank-default-small.service rename {scripts => old/scripts}/frank-large-ling.service (100%) rename {scripts => old/scripts}/frank-pp-worker.env (100%) rename {scripts => old/scripts}/frank-pp-worker.service (100%) rename {scripts => old/scripts}/llama-server.service (100%) rename {scripts => old/scripts}/pp-bench.py (100%) rename {scripts => old/scripts}/pp-test.sh (100%) rename {scripts => old/scripts}/prune-rpc-cache.sh (100%) rename {scripts => old/scripts}/serve-flm-small.sh (100%) rename {scripts => old/scripts}/serve-ling-flash.sh (100%) rename {scripts => old/scripts}/serve-ling3-vllm.sh (100%) rename {scripts => old/scripts}/serve-llama-frank-large.sh (100%) rename {scripts => old/scripts}/serve-llama-frank-small.sh (100%) rename {scripts => old/scripts}/serve-pp-head.sh (100%) rename {scripts => old/scripts}/serve-pp-worker.sh (100%) rename {scripts => old/scripts}/serve-vllm-qwen36-dflash.sh (100%) rename {scripts => old/scripts}/spec-matrix-bench.sh (100%) rename {scripts => old/scripts}/test_dspark_stream.py (100%) rename {scripts => old/scripts}/validate-vllm.sh (100%) mode change 100755 => 100644 scripts/deploy.sh create mode 100644 scripts/serve-frank-large.sh create mode 100644 scripts/serve-frank-small.sh diff --git a/AGENTS.md b/AGENTS.md index 25fef98..7fff800 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,48 +2,44 @@ ## Overview -This project manages deployment scripts for the LLM inference fleet. -Dankclaw runs a llama.cpp pipeline-parallel head serving -Qwen3.8-27B-Uncensored Q6_K (imatrix): MTP draft (standalone -`-draft-Q8_0` sidecar, `--spec-draft-n-max 4`) over the `--spec-default` -ngram-mod fallback, 1 slot at 262144 context (pool 262144), tensors -split 0.68/0.32 (45 of 65 blocks) to the Dankserver RTX 3090 `ggml-rpc` -worker, KV q8_0/f16 (mismatched, FA retained via the -GGML_CUDA_FA_ALL_QUANTS rebuild), head GPU side on ROCm0 (no `--device` -flag), port 7770. -The Dankclaw NPU endpoint (FastFlowLM, `frank-default-small`, port 52625) -serves Qwythos-9B-Claude-Mythos-5-1M (abliterated) as registry tag -`qwythos:9b` at 262144 ctx (swapped in 2026-08-17; previously -`qwen3.6-moe:35b-a3b`; the old llama.cpp 35B service on Dankserver is -stopped). **Boot policy (2026-08-17):** only `frank-default-large` and -`frank-pp-worker` are enabled for boot. The NPU unit `frank-default-small` -is disabled; the running service keeps serving until stopped or rebooted. -See [docs/deployment.md](docs/deployment.md) for the deployment -process, the boot policy, and the re-enable recipes. +This project manages deployment scripts for the LLM inference fleet on two +local machines. Topology (2026-08-17 redo): + +| Target | Host | Runtime | Service | Model | Port | +|--------|------|---------|---------|-------|------| +| Dankserver | dank@dankserver | llama.cpp (patched FastMTP build) | frank-default-large | HauhauCS Qwen3.8-27B-Uncensored-Aggressive IQ4_XS + FastMTP 32K draft + ngram-mod | 7770 | +| Dankclaw | dank@dankclaw | vLLM (ROCm, /home/dank/vllm-venv) | frank-default-small | Avesed Qwen3.6-35B-A3B INT8 W8A8, ngram spec | 52625 | + +The 2026-08-17 redo retired the pipeline-parallel split and the FastFlowLM +NPU endpoint. The RTX 3090 now runs the whole 27B model standalone +(no RPC worker). Dankclaw runs vLLM on the Strix Halo iGPU (gfx1151). +The old scripts live in `old/scripts/` for reference; they describe the +retired PP topology. **Sibling project:** [../serverAppConfigs](../serverAppConfigs) manages the Podman app stack on the same hosts (OpenWebUI, Gitea, observability). Its -OpenWebUI consumes the `:7770` endpoint; its Gitea instance hosts this -repository. Cross-links live in both knowledge-base indexes. +OpenWebUI consumes the `:7770` endpoint on Dankserver; its Gitea instance +hosts this repository. ## Benchmarking Policy (MANDATORY) **STOP the production service BEFORE running any benchmark on a target.** These servers serve real users (about 3 people). Running a bench while the service is up starves the GPU and corrupts results. Restart the service -when the bench finishes. The users will call if they need the model. +when the bench finishes. Rules: -1. Stop the service before any benchmark: `systemctl stop frank-default-large` - (Dankclaw) or `systemctl stop frank-default-small` (Dankclaw NPU). Use the - passwordless helper or the password `Assface.3241` with `sudo -S`. -2. Run the bench on a scratch port (never the service port 7770). -3. Restart the service when the bench completes: `systemctl start `. -4. Verify health before reporting: `curl http://127.0.0.1:7770/health`. - -Do not skip this policy. A stale GPU contention or a dead production box is -worse than a slow bench. +1. Stop the service before any benchmark: + `sudo systemctl stop frank-default-large` (Dankserver) or + `sudo systemctl stop frank-default-small` (Dankclaw). Use the password + `Assface.3241` with `sudo -S` when no tty is available. +2. Run the bench on a scratch port (never the service ports 7770/52625). +3. Restart the service when the bench completes: + `sudo systemctl start `. +4. Verify health before reporting: + `curl http://192.168.0.235:7770/health` or + `curl http://192.168.0.143:52625/health`. ## Git Workflow @@ -63,11 +59,15 @@ All script edits must flow through git. Do not edit files on the remote machine Use `scripts/deploy.sh` to deploy committed changes to a target. It pushes `main`, pulls on the target, restarts the service, and checks the health endpoint. ``` -./scripts/deploy.sh dankclaw # PP head (frank-default-large, port 7770) -./scripts/deploy.sh dankserver # PP worker (frank-pp-worker, RPC 50052) -./scripts/deploy.sh dankclaw --dry-run # preview without changes +FRANK_SUDO_PW=... ./scripts/deploy.sh dankserver # llama.cpp FastMTP, port 7770 +FRANK_SUDO_PW=... ./scripts/deploy.sh dankclaw # vLLM, port 52625 +./scripts/deploy.sh dankserver --dry-run # preview without changes ``` +Set `FRANK_SUDO_PW` for headless runs (remote sudo reads it from stdin). +The passwordless restart helpers installed for the retired topology are +obsolete; `deploy.sh` no longer uses them. + Deployment workflow: ```mermaid @@ -79,25 +79,45 @@ graph LR E --> F[health check] ``` -Target map: - -| Target | Host | Runtime | Service | Port | -|--------|------|---------|---------|------| -| Dankclaw | dank@dankclaw | llama.cpp PP head (Qwen3.8-27B-Uncensored Q6_K, split 0.68/0.32 to the Dankserver worker via RPC, standalone MTP draft + ngram-mod, q8_0/f16 KV) | frank-default-large | 7770 | -| Dankclaw | dank@dankclaw | FastFlowLM NPU (XDNA2 50 TOPS, qwythos:9b Q4NX 7.76 GB, 262144 ctx; see [[FastFlowLM NPU Inference]]) — disabled for boot since 2026-08-17 | frank-default-small | 52625 | -| Dankserver | dank@dankserver | ggml-rpc worker (RTX 3090, receives the head's tensor share) | frank-pp-worker | 50052 | - -> Warning: `deploy.sh dankserver` is currently broken. It targets the -> removed `frank-default-small` unit. Restart the worker with -> `sudo systemctl restart frank-pp-worker` until the script is fixed. - ## Runtime & Troubleshooting +### FastMTP patched llama.cpp (Dankserver) + +The HauhauCS FastMTP draft sidecar requires a llama.cpp build from commit +`4df29be4f4c3673f428170fda944a5b19f743bb8` plus the +`HauhauCS-FastMTP-llama.cpp.patch` (from the HF repo). The build lives at +`/home/dank/llama.cpp-fastmtp/build` on Dankserver. Loading the draft with +a stock binary fails with `expected 5120, 248320, got 5120, 32768`. Do not +rebuild from another commit without re-applying the patch. See the +HauhauCS HF repo README for the exact build recipe. + +### vLLM ROCm install (Dankclaw) + +vLLM runs from `/home/dank/vllm-venv` (Python 3.12). Install recipe: + +``` +uv venv --python /usr/bin/python3.12 /home/dank/vllm-venv +uv pip install --python /home/dank/vllm-venv vllm --torch-backend=auto +``` + +`--torch-backend=auto` resolves the AMD ROCm wheel set (torch + +amd-torch-device-gfx1151 + the vllm rocm variant) from the AMD index. +The model cache is the default HF cache +(`/home/dank/.cache/huggingface`); the HF token lives at +`~/.cache/huggingface/token`. The retired CIRU package +(`/home/dank/ling3-ciru`) documents a source-build fallback for gfx1151. + ### Env var naming rule — HIP runtime collision -The ROCm HIP runtime reads the bare environment variables `TEMP`, `HOST`, and `GPU_MAX_HEAP_SIZE`. The serve scripts export the env file with `set -a`. A tunable named `TEMP` leaks into HIP. On 2026-08-13 `TEMP=1.0` made HIP use the path `1.0` for its JIT kernel cache, then segfaulted every model load. Commit `227ce01` fixed it by renaming the sampling variable to `TEMPERATURE`. +The ROCm HIP runtime reads the bare environment variables `TEMP`, `HOST`, +and `GPU_MAX_HEAP_SIZE`. The serve scripts export the env file with +`set -a`. A tunable named `TEMP` leaks into HIP. On 2026-08-13 +`TEMP=1.0` made HIP use the path `1.0` for its JIT kernel cache, then +segfaulted every model load. Commit `227ce01` fixed it by renaming the +sampling variable to `TEMPERATURE`. -Never name a tunable `TEMP`, `HOST`, or `GPU_MAX_HEAP_SIZE`. Check for collisions with this command: +Never name a tunable `TEMP`, `HOST`, or `GPU_MAX_HEAP_SIZE`. Check for +collisions with this command: ``` strings /opt/rocm-*/lib/libamdhip64.so.* | grep -xE "TEMP|HOST|GPU_MAX_HEAP_SIZE" @@ -121,25 +141,27 @@ graph TD 2. Run the serve script directly. Then run the raw binary with the same flags. A different outcome isolates the env file sourcing. 3. Run gdb for the backtrace with this command: ``` - gdb -batch -ex run -ex "bt 25" --args bash scripts/serve-pp-head.sh + gdb -batch -ex run -ex "bt 25" --args bash scripts/serve-frank-large.sh ``` `llama-server` buffers its timed stdout. On SIGSEGV the log tail is lost. Trust gdb, not the log. 4. Kernel spam about `amdgpu: SVM mapping failed` is a symptom, not the root cause. Investigate other causes before assuming memory pressure. ### Operational quirks -- `sudo` on both boxes needs a tty. The remote sudo steps in `deploy.sh` fail non-interactively. Use `echo '' | sudo -S `. -- Passwordless restarts: each target has a root-owned helper and a sudoers NOPASSWD drop-in (`/etc/sudoers.d/frank-*`). The Dankclaw helper `/usr/local/sbin/frank-restart-large.sh` is correct for the PP head. The Dankserver helper `frank-restart-small.sh` is **obsolete** — it restarts the removed `frank-default-small` unit. Restart the worker with `sudo systemctl restart frank-pp-worker`. The sudoers rules match commands exactly: no extra arguments (for example, `--no-pager` breaks the `systemctl status` rule). See [docs/deployment.md](docs/deployment.md). +- `sudo` on both boxes needs a tty. Remote sudo steps fail non-interactively. Use `echo '' | sudo -S `. - Never run `pkill -f` with a pattern that appears in your own command line. It kills your ssh session. Use `pkill -x `. -- Model load is slow: about 3 minutes warm, 8-17 minutes cold (measured 17 minutes on 2026-08-13 after a cache drop). Health returns `503 Loading model` the whole time. That is normal. Do not kill the process. `deploy.sh` waits up to 20 minutes on Dankclaw (see `HEALTH_WAIT`). -- Page-cache state confounds crash tests. Cold-cache loads are slow and survive. Warm-cache loads crash fast. Control for it when testing. `deploy.sh` drops caches on Dankclaw before restart (see `DROP_CACHES`); manual restarts must run `sync && echo 3 | sudo tee /proc/sys/vm/drop_caches` first. +- Model load is slow: Dankserver ~5 minutes cold (15.7 GB IQ4_XS through RAM, `--no-mmap`); Dankclaw vLLM ~5-10 minutes (38 GB checkpoint). Health returns 503 while loading. Do not kill the process. `deploy.sh` waits up to 10 minutes on Dankserver and 20 minutes on Dankclaw (see `HEALTH_WAIT`). - A client abort and a server release look different in the journal. A client abort logs `W srv stop: cancel task, id_task = N`. A natural stop (EOS or limit) logs only `slot release` with `truncated = 0`. The server never preempts a running generation; when all slots are busy, new tasks defer into a queue. See the 2026-08-13 incident in [docs/deployment.md](docs/deployment.md). -- Dankserver crash signature (historical): the retired 35B service aborted under parallel load with `CUDA error: an unsupported value or parameter was passed to the function` in `cublasSgemm_v2` at `ggml-cuda.cu:1541`, then `status=6/ABRT`, then `503 Loading model` for ~5 minutes (llama.cpp b10121, driver 595.84). Root cause was never determined; the service is retired. Do not confuse that crash with speculative decoding. See [docs/dankserver-vram-growth.md](docs/dankserver-vram-growth.md). -- DFlash premature EOS under parallel load (historical): on Dankclaw, `draft-dflash` ended a generating slot ~1s after a new task landed with high prompt-cache reuse (LCP ~0.99), with `finish_reason "stop"` and no `cancel task` log — a DFlash shared-draft-context bug. DFlash and DSpark are retired from Dankclaw. Production uses the standalone MTP draft over the ngram-mod fallback at PARALLEL=1 on ROCm0. See [docs/qwen3.8-spec-matrix.md](docs/qwen3.8-spec-matrix.md). +- Retired hardware history: the Dankserver 35B cuBLAS crash and the DFlash premature-EOS bug are historical; see [docs/dankserver-vram-growth.md](docs/dankserver-vram-growth.md) and [docs/qwen3.8-spec-matrix.md](docs/qwen3.8-spec-matrix.md). The FastFlowLM NPU endpoint is retired; see [docs/FastFlowLM NPU Inference.md](docs/FastFlowLM%20NPU%20Inference.md). ### Dankclaw memory model -The GPU budget on Dankclaw is the rocm-smi figure: 96 GiB addressable VRAM (SVM). The GPU node reports `local_mem_size 0`, so all allocations are GTT/SVM-backed. Trust rocm-smi and the runtime. Do not use `free` or `dmidecode` for the GPU budget; they show the CPU-side carve-out. This is intentional. Do not change it. +The GPU budget on Dankclaw is the rocm-smi figure: 64 GiB VRAM reported +by rocm-smi (SVM-backed; the docs' older 96 GiB figure was the addressable +SVM pool). The GPU node reports `local_mem_size 0`, so all allocations are +GTT/SVM-backed. Trust rocm-smi and the runtime. Do not use `free` or +`dmidecode` for the GPU budget; they show the CPU-side carve-out. This is +intentional. Do not change it. ## Why diff --git a/README.md b/README.md index 517038e..698e6a1 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,37 @@ # vLLM Scripts -Deployment scripts for the LLM inference fleet: a llama.cpp -pipeline-parallel Qwen3.8-27B split across two machines, plus the -FastFlowLM NPU endpoint on Dankclaw. +Deployment scripts for the LLM inference fleet (2026-08-17 topology): +llama.cpp FastMTP on Dankserver and vLLM ROCm on Dankclaw. ## Targets -| Host | User | Codename | Architecture | -|------|------|----------|--------------| -| dankserver | dank | Dankserver | NVIDIA RTX 3090 + 62 GB RAM | -| dankclaw | dank | Dankclaw | AMD Strix Halo APU (shared memory) | +| Host | User | Codename | GPU | +|------|------|----------|-----| +| dankserver | dank | Dankserver | NVIDIA RTX 3090 24 GB + 62 GB RAM | +| dankclaw | dank | Dankclaw | AMD Strix Halo iGPU gfx1151 (unified memory) | ## Active Services | Service | Host | Runtime | Model | Port | |---------|------|---------|-------|------| -| frank-default-large | dankclaw | llama.cpp PP head (RPC to Dankserver) | Qwen3.8-27B-Uncensored Q6_K, standalone MTP draft | 7770 | -| frank-pp-worker | dankserver | ggml-rpc worker (RTX 3090) | receives the head's tensor share | 50052 | +| frank-default-large | dankserver | llama.cpp (patched FastMTP build) | HauhauCS Qwen3.8-27B-Uncensored-Aggressive IQ4_XS, FastMTP 32K draft + ngram-mod, K q8_0 / V f16 KV, 128K ctx | 7770 | +| frank-default-small | dankclaw | vLLM ROCm (vllm-venv) | Avesed Qwen3.6-35B-A3B INT8 W8A8, full 262144 ctx, no KV quantization, 4 seqs, ngram spec | 52625 | -Boot policy (2026-08-17): only these two units are enabled. The -FastFlowLM NPU endpoint (`frank-default-small`, qwythos:9b, port 52625) -is disabled for boot; re-enable it from `scripts/frank-default-small.service` -if needed. Retired units: the Dankserver llama.cpp 35B service, -`llamaserver`, `ollama`, `vllm-awq-qwen36`, `vllm-qwen36`, -`frank-large-ling` (Ling trial), and the vLLM-ROCm CIRU path. Their unit -files remain in `scripts/` for reference. +The 2026-08-17 redo retired the pipeline-parallel split (PP head + RPC +worker) and the FastFlowLM NPU endpoint. Their scripts are preserved in +`old/scripts/` for reference. ## Structure -- `docs/` — knowledge base: deployment runbook, PP runbook, spec matrix, - NPU notes, research archive -- `scripts/` — serve scripts, env files, systemd units, deploy and test - tools +- `docs/` — knowledge base: deployment runbook, model notes, research archive +- `scripts/` — serve scripts, env files, systemd units, deploy and test tools +- `old/scripts/` — retired PP/NPU-era scripts (reference only) ## Deployment See [docs/deployment.md](docs/deployment.md). Run -`./scripts/deploy.sh dankclaw` for the PP head. The `dankserver` target -is broken (it targets the removed `frank-default-small` unit); restart -the worker with `sudo systemctl restart frank-pp-worker` until the -script is fixed. +`FRANK_SUDO_PW=... ./scripts/deploy.sh dankserver` for the llama.cpp box or +`FRANK_SUDO_PW=... ./scripts/deploy.sh dankclaw` for the vLLM box. ## Knowledge Base diff --git a/docs/FastFlowLM NPU Inference.md b/docs/FastFlowLM NPU Inference.md index e4535cd..45d73aa 100644 --- a/docs/FastFlowLM NPU Inference.md +++ b/docs/FastFlowLM NPU Inference.md @@ -1,3 +1,7 @@ +> **SUPERSEDED 2026-08-17.** The Frank redo retired this topology. +> See [frank-redo-2026-08-17.md](frank-redo-2026-08-17.md) and +> [deployment.md](deployment.md) for the active configuration. + --- type: research title: FastFlowLM NPU Inference diff --git a/docs/Pipeline Parallelism/Qwen3.8 PP Deployment.md b/docs/Pipeline Parallelism/Qwen3.8 PP Deployment.md index d5290c7..42b460f 100644 --- a/docs/Pipeline Parallelism/Qwen3.8 PP Deployment.md +++ b/docs/Pipeline Parallelism/Qwen3.8 PP Deployment.md @@ -1,3 +1,7 @@ +> **SUPERSEDED 2026-08-17.** The Frank redo retired this topology. +> See [frank-redo-2026-08-17.md](frank-redo-2026-08-17.md) and +> [deployment.md](deployment.md) for the active configuration. + # Qwen3.8-27B Pipeline-Parallel Deployment (Dankclaw + Dankserver) Split Qwen3.8-27B across the Strix Halo iGPU (head) and the RTX diff --git a/docs/dankserver-vram-growth.md b/docs/dankserver-vram-growth.md index 462bd94..289da13 100644 --- a/docs/dankserver-vram-growth.md +++ b/docs/dankserver-vram-growth.md @@ -1,3 +1,7 @@ +> **SUPERSEDED 2026-08-17.** The Frank redo retired this topology. +> See [frank-redo-2026-08-17.md](frank-redo-2026-08-17.md) and +> [deployment.md](deployment.md) for the active configuration. + --- type: reference title: Dankserver VRAM and RAM Growth Under Load diff --git a/docs/deployment.md b/docs/deployment.md index 57d0799..a4fdebd 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -8,43 +8,8 @@ Two servers run LLM inference services. Each server pulls its scripts from this | Target | Host | Runtime | Service | Port | |--------|------|---------|---------|------| -| Dankclaw | dank@dankclaw | llama.cpp PP head (Qwen3.8-27B-Uncensored Q6_K, split 0.68 to the Dankserver worker) | frank-default-large | 7770 | -| Dankclaw | dank@dankclaw | FastFlowLM NPU (qwythos:9b, 262144 ctx) | frank-default-small | 52625 | -| Dankserver | dank@dankserver | ggml-rpc worker (RTX 3090) | frank-pp-worker | 50052 | - -The `frank-default-small` (NPU) unit is not in `deploy.sh` and is disabled for boot (see Boot Policy below). If the unit is still loaded (pre-reboot), restart it by hand with `sudo systemctl restart frank-default-small`. After a reboot, re-enable the unit first (see Boot Policy). - -## Boot Policy (2026-08-17) - -Only the pipeline-parallel pair starts at boot: `frank-default-large` (Dankclaw head) and `frank-pp-worker` (Dankserver worker). Every other inference unit is disabled. This was set on 2026-08-17. - -Disabled units: - -| Machine | Unit | Fate | -|---------|------|------| -| Dankclaw | `frank-default-small` (FastFlowLM NPU, port 52625) | Disabled; unit file removed from `/etc/systemd/system`. The service keeps running until stopped or rebooted. | -| Dankclaw | `frank-large-ling` (Ling trial) | Disabled. | -| Dankclaw | `vllm-thinkingcap` | Disabled (already). | -| Dankserver | `frank-default-small` (old llama.cpp 35B) | Disabled; unit file removed. | -| Dankserver | `llamaserver`, `ollama`, `vllm-awq-qwen36`, `vllm-qwen36` | Disabled. | - -`systemctl disable` removes the symlinked unit file from `/etc/systemd/system` when the file is a symlink. The repo copies stay intact. To restore a disabled service, re-symlink the unit and enable it: - -``` -sudo ln -sf /home/dank//scripts/.service /etc/systemd/system/.service -sudo systemctl daemon-reload -sudo systemctl enable -``` - -The NPU re-enable recipe uses the Dankclaw repo path `/home/dank/vllmScripts/scripts/frank-default-small.service`. - -The Dankserver helper `frank-restart-small.sh` targets the removed `frank-default-small` unit. Do not use it. Restart the PP worker with an explicit command: - -``` -sudo systemctl restart frank-pp-worker -``` - -`scripts/deploy.sh dankserver` also targets the removed unit. Do not run it until the script is retargeted to `frank-pp-worker`. The `dankclaw` target works: it restarts the PP head through `frank-restart-large.sh`. +| Dankserver | dank@dankserver | llama.cpp FastMTP (Qwen3.8-27B IQ4_XS + FastMTP 32K draft + ngram-mod) | frank-default-large | 7770 | +| Dankclaw | dank@dankclaw | vLLM ROCm (Avesed Qwen3.6-35B-A3B INT8 W8A8, ngram spec) | frank-default-small | 52625 | ## Workflow @@ -70,22 +35,22 @@ graph LR The `scripts/deploy.sh` script executes steps 4 through 6. Run it from the repository root. -Run this command to deploy to Dankclaw: - -``` -./scripts/deploy.sh dankclaw -``` - Run this command to deploy to Dankserver: ``` -./scripts/deploy.sh dankserver +FRANK_SUDO_PW=... ./scripts/deploy.sh dankserver +``` + +Run this command to deploy to Dankclaw: + +``` +FRANK_SUDO_PW=... ./scripts/deploy.sh dankclaw ``` Add the `--dry-run` flag to preview without making changes: ``` -./scripts/deploy.sh dankclaw --dry-run +./scripts/deploy.sh dankserver --dry-run ``` The script does these actions: @@ -93,153 +58,81 @@ The script does these actions: 1. Verifies that the local working tree has no uncommitted changes. 2. Pushes the `main` branch to origin. 3. Pulls the repository on the target machine. -4. Restarts the service. When the passwordless helper is installed (see below), it runs the helper with plain `sudo`. Otherwise it falls back to the inline sudo commands (unit symlink, cache drop, daemon-reload, restart). +4. Symlinks the systemd unit, reloads systemd, enables, and restarts the service. 5. Waits for the health endpoint to respond (over ssh). -Set `FRANK_SUDO_PW` when running headless on a target that does not yet have the passwordless helper, so that remote sudo reads the password from stdin: +Set `FRANK_SUDO_PW` when running headless, so that remote sudo reads the +password from stdin (`sudo -S`). The passwordless restart helpers from the +retired topology are obsolete and are not used. + +Health endpoints: + +- Dankserver: `http://127.0.0.1:7770/health` +- Dankclaw: `http://127.0.0.1:52625/health` + +## Manual Update + +### Update Dankserver (frank-default-large) + +Run these commands on Dankserver: ``` -FRANK_SUDO_PW=... ./scripts/deploy.sh dankclaw +cd /home/dank/vLLM +git pull origin main +sudo ln -sf /home/dank/vLLM/scripts/frank-default-large.service /etc/systemd/system/frank-default-large.service +sudo systemctl daemon-reload +sudo systemctl restart frank-default-large +curl -sf http://127.0.0.1:7770/health ``` -## Passwordless Restart - -Each target has a root-owned helper script that performs the full restart. The user `dank` runs it with `sudo` and no password via a sudoers drop-in. - -| Target | Helper | Sudoers drop-in | -|--------|--------|-----------------| -| Dankclaw | `/usr/local/sbin/frank-restart-large.sh` | `/etc/sudoers.d/frank-large` | -| Dankserver | `/usr/local/sbin/frank-restart-small.sh` (obsolete, see below) | `/etc/sudoers.d/frank-small` | - -The Dankclaw helper does these steps: - -1. Runs `sync`. -2. Drops the page caches with `echo 3 > /proc/sys/vm/drop_caches`. -3. Symlinks the systemd unit. -4. Reloads systemd. -5. Enables and restarts the service. - -The Dankserver helper `frank-restart-small.sh` is obsolete: it restarts the removed `frank-default-small` unit. Do not use it. The sudoers drop-in does not cover `frank-pp-worker`, so the worker restart needs the password (or a new helper). Restart the worker with `sudo systemctl restart frank-pp-worker`. - -The sudoers drop-in contains exactly two rules: - -``` -dank ALL=(root) NOPASSWD: /usr/local/sbin/frank-restart-large.sh -dank ALL=(root) NOPASSWD: /usr/bin/systemctl status frank-default-large -``` - -Rules match the command and its arguments exactly. Do not append extra arguments. For example, `sudo systemctl status frank-default-large --no-pager` fails; use the exact form instead. Install or replace the helper and drop-in only through `sudo` on the target, and validate with `sudo visudo -c`. - -To test that the setup works, run this command: - -``` -sudo -n /usr/local/sbin/frank-restart-large.sh -``` - -### Update Dankclaw - -1. Pull the repository. -2. Symlink the unit file, if it changed. -3. Drop the page caches. -4. Reload and restart the service. +### Update Dankclaw (frank-default-small) Run these commands on Dankclaw: ``` cd /home/dank/vllmScripts git pull origin main -sudo ln -sf /home/dank/vllmScripts/scripts/frank-default-large.service /etc/systemd/system/frank-default-large.service -sync && echo 3 | sudo tee /proc/sys/vm/drop_caches +sudo ln -sf /home/dank/vllmScripts/scripts/frank-default-small.service /etc/systemd/system/frank-default-small.service sudo systemctl daemon-reload -sudo systemctl restart frank-default-large -curl -sf http://127.0.0.1:7770/health +sudo systemctl restart frank-default-small +curl -sf http://127.0.0.1:52625/health ``` -### Update Dankserver - -Run these commands on Dankserver to update and restart the PP worker: - -``` -cd /home/dank/vLLM -git pull origin main -sudo ln -sf /home/dank/vLLM/scripts/frank-pp-worker.service /etc/systemd/system/frank-pp-worker.service -sudo systemctl daemon-reload -sudo systemctl restart frank-pp-worker -``` - -The worker has no HTTP health endpoint. Verify the listener instead: - -``` -ss -tln | grep :50052 -``` - -The old passwordless helper `frank-restart-small.sh` targets the removed `frank-default-small` unit. Do not use it. See Boot Policy. - ## Service Units -### Dankclaw — frank-default-large +### Dankserver — frank-default-large -The active service on Dankclaw is the llama.cpp **pipeline-parallel -head**. It serves `Qwen3.8-27B-Uncensored-Q6_K.gguf` (JonathanColetti, -imatrix) under the alias `frank-default-large` on port 7770. The head -splits the tensors `0.68,0.32` to the `ggml-rpc` worker on Dankserver -(RTX 3090, port 50052): 45 of 65 blocks (0-44) on the 3090 — the -maximum for this config (measured 2026-08-17; see the PP doc). -Speculative decoding runs **MTP** with a standalone draft head -(`--spec-draft-model`, `Qwen3.8-27B-Uncensored-draft-Q8_0.gguf`, n_max -4) plus the `--spec-default` ngram-mod fallback (48/64/24). The head GPU -side runs **ROCm0** (no `--device` flag; default device selection puts -RPC first and picks ROCm for the iGPU). **1 slot** at 262144 context -(pool 262144, `n_ctx_slot = 262144`), KV cache **K q8_0 + V f16** -(mismatched, 784 MiB per FA layer; RPC0 8624 + ROCm0 3920 MiB) with -Flash Attention retained via the GGML_CUDA_FA_ALL_QUANTS rebuild, full -layer offload (`--gpu-layers all`, `--fit off`), batch 8192 / ubatch -256, `--direct-io`. The unit file is +The active service on Dankserver is llama.cpp serving +`Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-IQ4_XS.gguf` under the alias +`frank-default-large` on port 7770. It runs the patched FastMTP build +(`/home/dank/llama.cpp-fastmtp/build/bin/llama-server`, commit 4df29be + +the HauhauCS patch) with the FastMTP 32K draft sidecar (`--spec-type +draft-mtp`, `--spec-draft-n-max 2`, `--spec-draft-ngl all`) and the +ngram-mod fallback. Context 131072 (128K, single slot), KV K q8_0 + V f16, +batch 2048 / ubatch 1024, `--no-mmap`, full GPU offload. The unit file is `scripts/frank-default-large.service`. It launches through -`scripts/serve-pp-head.sh`, which reads tunables from +`scripts/serve-frank-large.sh`, which reads tunables from `scripts/frank-default-large.env`. -> Note: 2026-08-17 — the Q6_K swap: model + standalone draft head, -> mismatched q8_0/f16 KV with FA (both binaries rebuilt with -> GGML_CUDA_FA_ALL_QUANTS at 9b05354ec), split 0.68 (was 0.53 with the -> Q8_K_XL). Worker at 23854/24576 MiB, ~273 MiB free. MTP acceptance -> 0.75-0.83 (mean len 4.0-4.3), stronger than the old bundled head. +### Dankclaw — frank-default-small -> Note: 2026-08-17 — single-slot config: `PARALLEL=1`, pool 262144 -> (was 2 slots x 200700 at 401400), split 0.53 (was 0.40). The 3090 -> landed at 23460/24576 MiB, ~667 MiB free. Same-day incident: a full -> Dankserver disk corrupted the worker RPC tensor cache -> degenerate -> output; fixed by clearing the cache + freeing the disk. Guard: -> `scripts/prune-rpc-cache.sh` (cron, 40 GiB cap). - -> Note: 2026-08-16 — the pipeline-parallel split was promoted to -> production as `frank-default-large` (renamed from `frank-pp-head`); -> the standalone single-node service retired. MTP was re-enabled at -> `n_max 4` over the ngram-mod fallback. See -> [[Pipeline Parallelism/Qwen3.8 PP Deployment]] for the runbook. - -> Note: 2026-08-15 — the spec went DSpark (crashed under two concurrent 16k -> prompts on ROCm, `ROCm error: an illegal memory access`, status 6/ABRT) → -> `--spec-default` (ngram-mod, parallel-safe, PARALLEL=2). The backend went -> ROCm → Vulkan0 after a 262k-ctx bench showed Vulkan decodes 48% faster -> (5.68 vs 3.83 t/s). That Vulkan0 verdict applied to the standalone -> configuration; the PP head runs ROCm0 by default (HIP registers before -> Vulkan). See [[qwen3.8-spec-matrix]] for both decisions and the bench -> table. Also retired earlier: the vLLM-ROCm CIRU INT4 Ling-3.0-flash -> runtime, and the llama-serve/vLLM units on Dankclaw. - -### Dankserver — old llama.cpp 35B service (retired 2026-08-16) - -Dankserver previously ran `frank-default-small`: llama.cpp serving the HauhauCS Qwen3.6-35B-A3B uncensored `Q6_K_P` (ngram-mod spec, 4 slots x 175104 ctx, 36 of 40 MoE expert layers on CPU, mmproj vision). The service was verified `inactive` on 2026-08-16, and its unit file was removed on 2026-08-17 under the boot policy. The name `frank-default-small` now belongs to the FastFlowLM NPU endpoint on Dankclaw (see [[FastFlowLM NPU Inference]]). The RTX 3090 now runs the PP worker. See git history for the full retired configuration. +The active service on Dankclaw is vLLM (ROCm) serving +`Avesed/Qwen3.6-35B-A3B-INT8-W8A8` under the served model name +`frank-default-small` on port 52625. Full 262144 context, unquantized KV +cache, `--max-num-seqs 4`, `--max-num-batched-tokens 16384`, ngram +speculative decoding, `--reasoning-parser qwen3`. The unit file is +`scripts/frank-default-small.service`. It launches through +`scripts/serve-frank-small.sh`, which reads tunables from +`scripts/frank-default-small.env`. ## Validation -The deploy script checks the health endpoint after a restart. For deeper checks, run the validation script on the target. - -Run this command on Dankserver to test vLLM throughput: +The deploy script checks the health endpoint after a restart. For deeper checks, run a completion request against the endpoint: ``` -./scripts/validate-vllm.sh +curl -s http://127.0.0.1:7770/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model":"frank-default-large","messages":[{"role":"user","content":"Say OK"}]}' ``` ## Troubleshooting @@ -248,32 +141,24 @@ If the health check fails after a deploy: 1. Read the service logs with `journalctl -u -n 200`. 2. Confirm that the model files exist in the cache. -3. Confirm that the GPU has free memory. +3. Confirm that the GPU has free memory (`nvidia-smi` on Dankserver, `rocm-smi` on Dankclaw). 4. Check the service status with `systemctl status `. ## Incident: HIP Runtime Crash From the `TEMP` Env Variable (2026-08-13) -On 2026-08-13 the `frank-default-large` service crash-looped for hours. Every restart died with SIGSEGV about 20 seconds into model load, at a fixed offset in `libamdhip64` (NULL refcount unref during `ggml_backend_cuda_buffer_set_tensor`). The kernel log showed bursts of `amdgpu: SVM mapping failed, exceeds resident system memory limit`, which pointed at memory but was a red herring. +On 2026-08-13 the old `frank-default-large` service crash-looped for hours on Dankclaw. Root cause: the env files exported `TEMP=1.0` (a sampling default). The ROCm HIP runtime reads `TEMP` as its JIT kernel-cache directory, used the path `1.0`, failed, and segfaulted on a NULL object. Fix: rename the sampling variable to `TEMPERATURE` (commit `227ce01`). -Root cause: the env files exported `TEMP=1.0` (a sampling default added in commit `db3eb01`). The ROCm HIP runtime reads the `TEMP` environment variable as its JIT kernel-cache directory. HIP then used the path `1.0`, failed, and segfaulted on a NULL object. The crash was deterministic whenever the serve script sourced the env file, and absent when it did not. - -Fix: rename the sampling variable to `TEMPERATURE` in the env files and serve scripts (commit `227ce01`). The serve script still passes `--temp "$TEMPERATURE"` to `llama-server`, so sampling defaults are unchanged. - -If a service segfaults at load with no obvious cause, compare the exported env against the HIP runtime's env reads. Check with `strings /opt/rocm-*/lib/libamdhip64.so.* | grep -xE "TEMP|HOST|..."`. Do not name any tunable `TEMP`, `HOST`, or `GPU_MAX_HEAP_SIZE`. +If a service segfaults at load with no obvious cause, compare the exported env against the HIP runtime's env reads. Do not name any tunable `TEMP`, `HOST`, or `GPU_MAX_HEAP_SIZE`. ## Incident: Jobs "Released Prematurely" Under Parallel Load (2026-08-13) -A user reported that a streaming job on `frank-default-large` was released early when another job started. The suspicion was speculative decoding. The investigation found three facts: - -1. **The server does not preempt.** `llama-server` defers a new task into a queue when all slots are busy. It never releases a running generation to make room. Verified empirically with concurrent streaming jobs. -2. **The releases were client-side aborts.** The pi coding agent aborts a stream when the user interrupts it. The client records `stopReason: "aborted"`; the server logs `W srv stop: cancel task, id_task = N`. A natural stop (EOS or limit) logs only `slot release` with `truncated = 0`. -3. **Dankserver crashed under the parallel researcher load.** Five parallel research subagents hammered `frank-default-small` with back-to-back requests. The service aborted twice with `CUDA error: an unsupported value or parameter was passed to the function` in `cublasSgemm_v2` at `ggml-cuda.cu:1541`, then `status=6/ABRT`, then `503 Loading model` for ~5 minutes. That crash, not speculative decoding, killed the parallel jobs. The affected service is retired; see [[dankserver-vram-growth]] for the VRAM/RAM growth findings behind the crash class. +Historical: on the retired topology, parallel jobs on the Dankserver 35B service died with a cuBLAS crash under researcher load. The affected service is retired. A client abort logs `W srv stop: cancel task, id_task = N`; a natural stop logs only `slot release` with `truncated = 0`. The server never preempts a running generation. ## See Also - [AGENTS.md](../AGENTS.md) — the git workflow rules - [README.md](../README.md) — repository overview - [index.md](index.md) — knowledge base index +- [frank-redo-2026-08-17.md](frank-redo-2026-08-17.md) — the full redo runbook - [serverAppConfigs knowledge base](../serverAppConfigs/docs/index.md) — - the sibling Podman app stack on the same hosts; its OpenWebUI consumes - the `:7770` endpoint, and its Gitea instance hosts this repository + the sibling Podman app stack on the same hosts \ No newline at end of file diff --git a/docs/frank-default-large-benchmark.md b/docs/frank-default-large-benchmark.md index 97c75a0..f711565 100644 --- a/docs/frank-default-large-benchmark.md +++ b/docs/frank-default-large-benchmark.md @@ -1,3 +1,7 @@ +> **SUPERSEDED 2026-08-17.** The Frank redo retired this topology. +> See [frank-redo-2026-08-17.md](frank-redo-2026-08-17.md) and +> [deployment.md](deployment.md) for the active configuration. + # frank-default-large — Backend and Spec Benchmark This page records the benchmarks that chose the runtime for diff --git a/docs/frank-redo-2026-08-17.md b/docs/frank-redo-2026-08-17.md new file mode 100644 index 0000000..ba4e48f --- /dev/null +++ b/docs/frank-redo-2026-08-17.md @@ -0,0 +1,127 @@ +# Frank Program Redo — 2026-08-17 + +This page records the full redo of the inference fleet on 2026-08-17. +The pipeline-parallel split and the FastFlowLM NPU endpoint were retired. +The old scripts moved to `old/scripts/` in this repository. + +## New Topology + +| Service | Host | Runtime | Model | Port | +|---------|------|---------|-------|------| +| frank-default-large | Dankserver | llama.cpp (patched FastMTP) | HauhauCS Qwen3.8-27B-Uncensored-Aggressive IQ4_XS | 7770 | +| frank-default-small | Dankclaw | vLLM ROCm | Avesed Qwen3.6-35B-A3B INT8 W8A8 | 52625 | + +The RTX 3090 runs the whole 27B model standalone. No RPC worker, no +tensor split, no NPU. + +## Models + +### frank-default-large — HauhauCS/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-MTP-GGUF:IQ4_XS + +- Base: Qwen3.8-27B (dense 27B, 64 layers: 48 GatedDeltaNet + 16 + gated-attention, hidden 5120, 24 Q heads / 4 KV heads, head dim 256, + native 262144 ctx). +- Quant: IQ4_XS (4.60 BPW, 15.71 GB). Every GGUF in the repo preserves the + native embedded MTP/NextN head. +- FastMTP draft sidecar: `...FastMTP-32K.gguf` (903 MB). Requires the + patched llama.cpp build (commit `4df29be4f4c3673f428170fda944a5b19f743bb8` + + `HauhauCS-FastMTP-llama.cpp.patch`). Without the patch, the draft load + fails with `expected 5120, 248320, got 5120, 32768`. +- Spec chain: FastMTP (`--spec-type draft-mtp`, `--spec-draft-n-max 2`, + `--spec-draft-ngl all`, `--spec-draft-p-min 0`) then ngram-mod fallback + (48/64/24). + +### frank-default-small — Avesed/Qwen3.6-35B-A3B-INT8-W8A8 + +- Base: Qwen3.6-35B-A3B (hybrid MoE, 256 experts top-8, ~3B active, 40 + layers: 10 blocks of 3 DeltaNet + 1 Attention, native 262144 ctx). +- Quant: compressed-tensors INT8 W8A8 (symmetric per-channel weights, + dynamic per-token activations). Expert FFNs quantized; attention, router, + DeltaNet, lm_head kept bf16. Weights ~38 GB. +- Served with vLLM ROCm, ngram speculative decoding. + +## Context Math (frank-default-large, RTX 3090) + +KV per token: 16 attention layers x 4 KV heads x 256 dim. +K q8_0 = 1088 B/token, V f16 = 2048 B/token, total 50176 B/token. + +Budget (MiB): 24576 total - 14982 weights - 861 draft - 1024 fixed +(CUDA context, compute graph, scratch, draft KV) - headroom. + +| Context | KV (MiB) | Free after load | +|---------|----------|-----------------| +| 114688 (112K) | 5488 | 2221 (2.17 GiB) | +| 122880 (120K) | 5880 | 1829 (1.79 GiB) | +| 131072 (128K) | 6272 | 1437 (1.40 GiB) | +| 147456 (144K) | 7056 | 653 (0.64 GiB) | + +Production choice: **131072 (128K)** — the maximum that keeps about +1.4 GiB headroom for checkpoints. Lower `CTX_SIZE` in +`scripts/frank-default-large.env` to 122880 for ~1.8 GiB headroom. + +## Install Runbook + +### Dankserver (frank-default-large) + +1. Stop the old worker: `sudo systemctl stop frank-pp-worker`. +2. Download the model and draft to `/home/dank/models/hauhaucs-qwen38/`. +3. Build the patched llama.cpp: + ``` + git clone https://github.com/ggerganov/llama.cpp /home/dank/llama.cpp-fastmtp + cd /home/dank/llama.cpp-fastmtp + git checkout 4df29be4f4c3673f428170fda944a5b19f743bb8 + curl -L -o HauhauCS-FastMTP-llama.cpp.patch + git apply HauhauCS-FastMTP-llama.cpp.patch + cmake -S . -B build -DGGML_CUDA=ON -DCMAKE_BUILD_TYPE=Release + cmake --build build --config Release -j$(nproc) + ``` +4. Symlink the unit, reload, enable, restart: + ``` + sudo ln -sf /home/dank/vLLM/scripts/frank-default-large.service /etc/systemd/system/ + sudo systemctl daemon-reload && sudo systemctl enable frank-default-large + sudo systemctl restart frank-default-large + ``` +5. Verify: `curl -sf http://127.0.0.1:7770/health` and a chat completion. +6. Confirm VRAM headroom with `nvidia-smi`. + +### Dankclaw (frank-default-small) + +1. Stop the old services: `sudo systemctl stop frank-default-large + frank-default-small` (retired PP head and NPU endpoint). +2. Install uv and create the venv: + ``` + curl -LsSf https://astral.sh/uv/install.sh | sh + uv venv --python /usr/bin/python3.12 /home/dank/vllm-venv + uv pip install --python /home/dank/vllm-venv vllm --torch-backend=auto + ``` + The resolver pulls the AMD ROCm wheel set (torch + + amd-torch-device-gfx1151 + the vllm rocm variant) from the AMD index. +3. Cache the model (default HF cache): + ``` + python3 -c "from huggingface_hub import snapshot_download; snapshot_download('Avesed/Qwen3.6-35B-A3B-INT8-W8A8')" + ``` +4. Symlink the unit, reload, enable, restart: + ``` + sudo ln -sf /home/dank/vllmScripts/scripts/frank-default-small.service /etc/systemd/system/ + sudo systemctl daemon-reload && sudo systemctl enable frank-default-small + sudo systemctl restart frank-default-small + ``` +5. Verify: `curl -sf http://127.0.0.1:52625/health` and a chat completion. + +## Fallback: vLLM source build for gfx1151 + +If the prebuilt ROCm vLLM wheel cannot run on gfx1151, build from source. +The retired CIRU package documents the proven recipe: +`/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/scripts/build-vllm-gfx1151.sh` +(branch `ciru/ling3-gfx1151-rocm715`, index `https://rocm.nightlies.amd.com/whl-multi-arch/`). +The upstream starting point needed a five-line ROCm sampler fix to launch +on gfx1151; newer vLLM releases may already include it. + +## Configuration + +- `models.json` (pi): two providers — `llamacpp (dankserver)` at + http://192.168.0.235:7770/v1 and `vllm (dankclaw)` at + http://192.168.0.143:52625/v1. +- HF token: `~/.cache/huggingface/token` on both machines (mode 600). +- OpenWebUI (sibling repo): retarget `OPENAI_API_BASE_URL` to the + Dankserver endpoint. diff --git a/docs/index.md b/docs/index.md index 8b19407..dc1544c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,8 +1,8 @@ # vLLM Scripts — Knowledge Base -Documentation for the LLM inference fleet on two local machines: -llama.cpp production (PP head + RPC worker), the FastFlowLM NPU endpoint, -plus a vLLM research archive. +Documentation for the LLM inference fleet on two local machines +(2026-08-17 redo): llama.cpp FastMTP on Dankserver and vLLM ROCm on +Dankclaw. ## Deployment Topology @@ -10,44 +10,47 @@ plus a vLLM research archive. graph LR subgraph Dankserver[NVIDIA Dankserver - Ubuntu 24.04] A[RTX 3090 24GB + 62GB RAM] - B[ggml-rpc worker -frank-pp-worker, port 50052] - A --> B + B[llama.cpp FastMTP +frank-default-large, port 7770] + C[Qwen3.8-27B-Aggressive IQ4_XS +FastMTP 32K draft + ngram-mod, 128K ctx] + A --> B --> C end subgraph Dankclaw[AMD Strix Halo - Ubuntu 26.04] - D[gfx1151 APU 30GiB phys / 96GiB GPU-visible] - E[llama.cpp PP head -frank-default-large, port 7770] - F[Qwen3.8-27B-Uncensored Q6_K -split 0.68/0.32 to worker, standalone MTP draft + ngram-mod] + D[gfx1151 iGPU unified memory] + E[vLLM ROCm +frank-default-small, port 52625] + F[Avesed Qwen3.6-35B-A3B INT8 W8A8 +full 262144 ctx, ngram spec, 4 seqs] D --> E --> F - G[FastFlowLM NPU XDNA2 50 TOPS -frank-default-small, port 52625 - DISABLED for boot 2026-08-17] - H[qwythos:9b -Q4NX 7.76 GB, 262k ctx] - D --> G --> H end - I[LLM Clients / API] -->|HTTP :7770| E - I -->|HTTP :52625| G - E -->|RPC layers 0.68| B + I[LLM Clients / API] -->|HTTP :7770| B + I -->|HTTP :52625| E ``` -Boot policy (2026-08-17): only `frank-default-large` and `frank-pp-worker` are enabled. The NPU endpoint and all retired units are disabled; see [[deployment]] for the table and re-enable recipes. - ## Servers -| Server | GPU | VRAM / Memory | OS | -| --------------------------------------- | ----------------------------- | ----------------------------------- | ------------ | -| [[Dankserver System Specs\|Dankserver]] | NVIDIA RTX 3090 | 24 GB VRAM, 62 GB RAM | Ubuntu 24.04 | -| [[Dankclaw System Specs\|Dankclaw]] | AMD Radeon 8060S (Strix Halo) | 30 GiB physical / 96 GiB GPU-visible | Ubuntu 26.04 | +| Server | GPU | VRAM / Memory | OS | +| --------------------------------------- | ----------------------------- | -------------------- | ------------ | +| [[Dankserver System Specs\|Dankserver]] | NVIDIA RTX 3090 | 24 GB VRAM, 62 GB RAM | Ubuntu 24.04 | +| [[Dankclaw System Specs\|Dankclaw]] | AMD Radeon 8060S (Strix Halo) | gfx1151 unified memory | Ubuntu 26.04 | -## vLLM Reference +## Active Deployment + +- [[deployment]] — deploy and update process for both servers +- [[frank-redo-2026-08-17]] — the full redo: topology change, model + choices, context math, and the two install runbooks +- [[Dankserver]] — the llama.cpp FastMTP box (frank-default-large) +- [[Dankclaw]] — the vLLM ROCm box (frank-default-small) + +## vLLM Reference (Dankclaw) - [[VLLM Overview]] — installation, CLI, API, architecture, quantization, parallelism - [[VLLM Index]] — quick reference with commands, endpoints, and parameter table -- [[deployment]] — deploy and update process for both servers +- [[Running Qwen3.6 35B A3B FP8 on Dankserver]] — Qwen3.6-35B-A3B hardware + notes and sizing (the model family served by frank-default-small) ## Memory Management @@ -55,36 +58,23 @@ Boot policy (2026-08-17): only `frank-default-large` and `frank-pp-worker` are e - [[Hybrid CPU GPU Processing Dankserver]] — NVIDIA GPU compute with system RAM offload - [[Shared Memory Architecture Dankclaw]] — AMD unified memory and ROCm configuration -## Model Deployment +## Retired Topology (2026-08-17 redo) -- [[Running Qwen3.6 35B A3B FP8 on Dankserver]] — MoE model (35B total, 3B active) on RTX 3090 -- [[Running Qwen3.6 27B FP8 on Dankclaw]] — Dense model (27B) on Strix Halo APU -- [[frank-default-large-benchmark]] — backend/spec benchmark; chose MTP sidecar (2026-08-14) over DFlash (parallel premature-EOS bug) -- [[qwen3.8-spec-matrix]] — Qwen3.8-27B spec history + current production state: Uncensored Q6_K + standalone MTP draft, q8_0/f16 KV with FA (FA_ALL_QUANTS rebuild), 1 slot @ 262144, split 0.68 (2026-08-17) -- [[qwen3.8-strix-optimizations]] — Qwen3.8-27B Strix speed research: ranked levers that keep UD-Q8_K_XL (PP to 3090, MTP tuning, same-bit Q8_0 A/B, prefill path, OS), quant landscape, 2026-08-16 test matrix -- [[frank-default-small-benchmark]] — ncmoe and batch/ubatch sweep that drove the Dankserver state -- [[dankserver-vram-growth]] — VRAM/RAM growth investigation behind the recurring cuBLAS crash on frank-default-small -- [[Dankclaw Improvements]] — researched enhancements: self-hosted agentic tools, runtime tuning, server/OS settings -- [[FastFlowLM NPU Inference]] — FastFlowLM (FLM), the AMD NPU-first runtime. The 50 TOPS Strix Halo NPU on Dankclaw: model fit, vendor benchmarks, verified install state, test runbook (2026-08-16) - -## Speculative Decoding & Quantization - -- [[DFlash DSpark Speculative Decoding]] — block diffusion (DFlash) and semi-autoregressive (DSpark) drafting on vLLM and llama.cpp +- [[Qwen3.8 PP Deployment]] — the retired pipeline-parallel head + worker +- [[qwen3.8-spec-matrix]] — Qwen3.8-27B spec history (superseded by FastMTP) +- [[FastFlowLM NPU Inference]] — the retired FastFlowLM NPU endpoint +- [[frank-default-large-benchmark]] — backend/spec benchmark behind the old head +- [[dankserver-vram-growth]] — VRAM/RAM growth behind the retired cuBLAS crash +- [[DFlash DSpark Speculative Decoding]] — retired DFlash/DSpark drafting - [[Quantization by Hardware Architecture]] — which quant format fits which GPU/CPU -## Pipeline Parallelism +## Pipeline Parallelism (archive) -- [[Pipeline Parallelism]] — index of multi-node inference research and sample scripts - - [[Pipeline Parallelism/llama.cpp RPC]] — ggml-rpc-server + `--rpc` distributed inference - - [[Pipeline Parallelism/vLLM Pipeline Parallelism]] — vLLM PP across machines - - [[Pipeline Parallelism/Ethernet Bandwidth Math]] — bytes per token and link feasibility - - [[Pipeline Parallelism/Mixed-Vendor Experiments]] — NVIDIA + Strix Halo real-world runs - - [[Pipeline Parallelism/Expected Performance]] — estimated tok/s over 1/2.5/10 GbE - - [[Pipeline Parallelism/Sample Script Set]] — educational two-node Qwen3.6-27B Q8 split - - [[Pipeline Parallelism/Qwen3.8 PP Deployment]] — the production deployment + runbook: Qwen3.8-27B head (Strix) / worker (RTX 3090) over RPC, b10433, scripts in scripts/; split limits, FA-all-quants rebuild, cache incident (2026-08-17) +- [[Pipeline Parallelism]] — index of multi-node inference research. The PP + pair is retired; these pages are research history. ## Sibling projects - [serverAppConfigs](../serverAppConfigs/docs/index.md) — the Podman app - stack on the same two hosts. Its OpenWebUI consumes the `:7770` - endpoint; its Gitea instance hosts this repository. + stack on the same two hosts. Its OpenWebUI consumes the `:7770` endpoint + on Dankserver; its Gitea instance hosts this repository. \ No newline at end of file diff --git a/docs/qwen3.8-spec-matrix.md b/docs/qwen3.8-spec-matrix.md index 45ae763..654842d 100644 --- a/docs/qwen3.8-spec-matrix.md +++ b/docs/qwen3.8-spec-matrix.md @@ -1,3 +1,7 @@ +> **SUPERSEDED 2026-08-17.** The Frank redo retired this topology. +> See [frank-redo-2026-08-17.md](frank-redo-2026-08-17.md) and +> [deployment.md](deployment.md) for the active configuration. + # Qwen3.8-27B Speculative-Decoding Matrix This page records the speculative-decoding benchmark history and the diff --git a/scripts/analyze-spec-matrix.py b/old/scripts/analyze-spec-matrix.py similarity index 100% rename from scripts/analyze-spec-matrix.py rename to old/scripts/analyze-spec-matrix.py diff --git a/scripts/bench-frank-large.sh b/old/scripts/bench-frank-large.sh similarity index 100% rename from scripts/bench-frank-large.sh rename to old/scripts/bench-frank-large.sh diff --git a/scripts/bench-high-ctx.sh b/old/scripts/bench-high-ctx.sh similarity index 100% rename from scripts/bench-high-ctx.sh rename to old/scripts/bench-high-ctx.sh diff --git a/scripts/bench-pipeline-64k.sh b/old/scripts/bench-pipeline-64k.sh similarity index 100% rename from scripts/bench-pipeline-64k.sh rename to old/scripts/bench-pipeline-64k.sh diff --git a/old/scripts/deploy.sh b/old/scripts/deploy.sh new file mode 100755 index 0000000..2dc6019 --- /dev/null +++ b/old/scripts/deploy.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +# deploy.sh — Deploy the vllmScripts repo to a target and restart its service. +# +# This script implements the second half of the git deployment workflow from +# AGENTS.md. It assumes the local changes are committed and pushed, then: +# 1. Pulls the repo on the target +# 2. Installs / enables the systemd unit +# 3. Drops page caches (dankclaw) and restarts the service +# 4. Waits for the health endpoint and reports status +# +# Sudo: when FRANK_SUDO_PW is set, remote sudo pipes it via `sudo -S` for +# headless runs. Otherwise plain `sudo` is used (tty or passwordless config). +# Prefer the passwordless restart helpers installed on each target +# (/usr/local/sbin/frank-restart-.sh); they cover the unit +# symlink, cache drop (dankclaw), daemon-reload, and restart in one root +# script. See docs/deployment.md -> Passwordless restart. +# +# Usage: +# ./deploy.sh [--dry-run] +# FRANK_SUDO_PW=... ./deploy.sh +# +# Targets: +# dankclaw - llama.cpp server (frank-default-large.service, port 7770) +# dankserver - llama.cpp server (frank-default-small.service, port 7770) +# +# Example: +# ./deploy.sh dankclaw +# ./deploy.sh dankserver --dry-run +set -euo pipefail + +# --------------------------------------------------------------------------- +# Target configuration +# --------------------------------------------------------------------------- +declare -A REMOTE_USER REMOTE_HOST REMOTE_DIR SERVICE SERVICE_HEALTH DROP_CACHES HELPER HEALTH_WAIT + +REMOTE_USER[dankclaw]=dank +REMOTE_HOST[dankclaw]=dankclaw +REMOTE_DIR[dankclaw]="/home/dank/vllmScripts" +SERVICE[dankclaw]="frank-default-large" +SERVICE_HEALTH[dankclaw]="http://127.0.0.1:7770/health" +# Drop page caches before restart: with the model's pages fully cached, the +# KV/graph allocation stalls at load (VRAM stuck at 28 GiB). See AGENTS.md. +DROP_CACHES[dankclaw]=1 +# Passwordless restart helper (root-owned, installed via /etc/sudoers.d/frank-large). +# When present, deploy.sh uses it instead of inline sudo commands. +HELPER[dankclaw]="/usr/local/sbin/frank-restart-large.sh" +# Health wait in iterations of 2s. Cold loads after the cache drop measured +# at up to 17 minutes on 2026-08-13; 600 iterations = 20 minutes. +HEALTH_WAIT[dankclaw]=600 + +REMOTE_USER[dankserver]=dank +REMOTE_HOST[dankserver]=dankserver +REMOTE_DIR[dankserver]="/home/dank/vLLM" +SERVICE[dankserver]="frank-default-small" +SERVICE_HEALTH[dankserver]="http://127.0.0.1:7770/health" +DROP_CACHES[dankserver]=0 +HELPER[dankserver]="/usr/local/sbin/frank-restart-small.sh" +HEALTH_WAIT[dankserver]=300 + +TARGET="${1:-}" +DRY_RUN=0 +if [ "${2:-}" = "--dry-run" ]; then + DRY_RUN=1 +fi + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- +if [ -z "${TARGET:-}" ]; then + echo "Usage: $0 [--dry-run]" + exit 1 +fi +if [ -z "${REMOTE_HOST[$TARGET]:-}" ]; then + echo "Unknown target: $TARGET" + exit 1 +fi + +# --------------------------------------------------------------------------- +# Sudo wrapper (headless-friendly) +# --------------------------------------------------------------------------- +# When FRANK_SUDO_PW is set, remote sudo reads the password from stdin via +# `sudo -S`. Otherwise plain `sudo` is used (tty or passwordless config). +SUDO_PREFIX="sudo" +if [ -n "${FRANK_SUDO_PW:-}" ]; then + SUDO_PREFIX="printf '%s\n' '$FRANK_SUDO_PW' | sudo -S" +fi + +# --------------------------------------------------------------------------- +# Local checks (pre-flight) +# --------------------------------------------------------------------------- +cd "$(dirname "$0")/.." +if ! git diff --quiet HEAD; then + echo "FAIL: Local working tree has uncommitted changes." + echo "Commit or stash them before deploying." + exit 1 +fi +if [ "$(git rev-parse --abbrev-ref HEAD)" != "main" ]; then + echo "WARN: Not on 'main'; deploying a non-main branch." +fi + +echo "=== Deploying to ${TARGET} ===" +echo " host: ${REMOTE_HOST[$TARGET]}" +echo " dir: ${REMOTE_DIR[$TARGET]}" +echo " service: ${SERVICE[$TARGET]}" +echo " health: ${SERVICE_HEALTH[$TARGET]}" +echo "" + +# --------------------------------------------------------------------------- +# Push (unless dry-run) +# --------------------------------------------------------------------------- +if [ "$DRY_RUN" = "1" ]; then + echo "[dry-run] Would push local main to origin" + echo "[dry-run] Would SSH to ${REMOTE_HOST[$TARGET]} and git pull" +else + echo "[1/4] Pushing local main to origin..." + git push origin main +fi + +REMOTE="$( + printf '%s@%s' "${REMOTE_USER[$TARGET]}" "${REMOTE_HOST[$TARGET]}" +)" + +# --------------------------------------------------------------------------- +# Remote: pull + install + restart +# --------------------------------------------------------------------------- +DROP_CACHES_CMD="" +if [ "${DROP_CACHES[$TARGET]}" = "1" ]; then + DROP_CACHES_CMD=" +echo 'Dropping page caches for reliable model load...' +${SUDO_PREFIX} sh -c 'sync; echo 3 > /proc/sys/vm/drop_caches' +" +fi + +HELPER_PATH="${HELPER[$TARGET]:-}" + +REMOTE_SCRIPT=" +set -euo pipefail +cd '${REMOTE_DIR[$TARGET]}' + +echo '[2/4] Pulling repo on ${TARGET}...' +git pull origin main + +if [ -x '${HELPER_PATH}' ]; then + # Passwordless restart helper (sudoers NOPASSWD) - preferred path. + echo '[3/4] Restarting via passwordless helper ${HELPER_PATH}...' + ${SUDO_PREFIX} '${HELPER_PATH}' +else + echo '[3/4] Helper not installed; using inline sudo commands...' + # Install / symlink the systemd unit + case '${TARGET}' in + dankclaw) + ${SUDO_PREFIX} ln -sf '${REMOTE_DIR[$TARGET]}/scripts/frank-default-large.service' /etc/systemd/system/frank-default-large.service + ;; + dankserver) + ${SUDO_PREFIX} ln -sf '${REMOTE_DIR[$TARGET]}/scripts/frank-default-small.service' /etc/systemd/system/frank-default-small.service + ;; + esac + ${DROP_CACHES_CMD} + echo '[3/4] Reload and restart ${SERVICE[$TARGET]}...' + ${SUDO_PREFIX} systemctl daemon-reload + ${SUDO_PREFIX} systemctl enable '${SERVICE[$TARGET]}' + ${SUDO_PREFIX} systemctl restart '${SERVICE[$TARGET]}' +fi +" + +if [ "$DRY_RUN" = "1" ]; then + echo "[dry-run] Remote commands would run:" + echo "$REMOTE_SCRIPT" +else + # shellcheck disable=SC2087 + ssh "$REMOTE" bash -s < /dev/null 2>&1; then + echo " Service healthy after approx $((i * 2))s" + echo "" + echo "=== Deploy complete: ${TARGET} ===" + exit 0 + fi + sleep 2 + done + echo "FAIL: Service did not become healthy in $((HEALTH_ITERS * 2))s" + exit 1 +fi diff --git a/old/scripts/frank-default-large.env b/old/scripts/frank-default-large.env new file mode 100644 index 0000000..875e62c --- /dev/null +++ b/old/scripts/frank-default-large.env @@ -0,0 +1,88 @@ +# Head tunables — Qwen3.8-27B pipeline-parallel head (Dankclaw). +# +# The head is the LOCAL side of the split. It runs llama-server (b10433, +# HIP+Vulkan+RPC build in /home/dank/llama.cpp/build) with --rpc pointing at +# the worker on Dankserver. The model file lives on this machine only; the +# worker receives its tensor share over the network at load time. +# +# CRITICAL: this script must NOT pass --device. When --device is set, +# llama.cpp uses exactly that device list and silently drops the RPC device +# (src/llama.cpp llama_prepare_model_devices). Default device selection +# inserts RPC servers at the FRONT of the device list, so the split becomes +# [RPC0, iGPU] and --tensor-split index 0 is the worker. +# +# WARNING: do not name any variable here TEMP, HOST, or GPU_MAX_HEAP_SIZE +# (HIP runtime collision, see the 2026-08-13 TEMP incident). + +MODEL_DIR=/home/dank/models +MODEL_FILE=Qwen3.8-27B-Uncensored-Q6_K.gguf +HEAD_BIN=/home/dank/llama.cpp/build/bin/llama-server +ALIAS=frank-default-large +BIND_ADDR=0.0.0.0 +LISTEN_PORT=7770 + +# Comma-separated RPC endpoints (host:port). One worker here. +RPC_SERVERS=192.168.0.235:50052 + +# Model split fractions, one per device in order [RPC0, iGPU]. +# 0.68,0.32 = 45 of 65 blocks on the 3090 (blocks 0-44) - the maximum +# for the Q6_K + q8_0/f16 KV config, measured 2026-08-17 phase 2: +# - KV per FA layer = 784 MiB (K 1088 B/token + V 2048 B/token @262144) +# - weights(f) = 20334 MB layer pool x f (GGUF scan, exact); embd on +# the head side (RPC0 buffer observed 10429.88 MiB at f=0.53) +# - KV-per-device count = round(f x 66) layers -> 45 layers = 11 FA +# (3..43) -> 8624 MiB. 46 layers (f>=0.7045) still 11 FA but margin +# drops to ~0.4 GiB; 48 layers (f>=0.7197) adds FA block 47 -> KV +# 9408 -> OOM by ~0.85 GiB. +# Predicted: 13430 (weights) + 8624 (KV) + ~530 (RS) + 189 (compute) = +# ~22773 MiB -> ~0.7 GiB free. Draft model runs on ROCm0 only (measured +# phase 1: 1718 MiB, no worker impact). +TENSOR_SPLIT=0.68,0.32 + +# Context pool. 262144 total = 1 parallel slot x 262144 per slot +# (n_ctx_seq = n_ctx / n_seq_max, llama-context.cpp:293). +CTX_SIZE=262144 +PARALLEL=1 + +# Speculative decoding. MTP with the STANDALONE draft head (the Q6_K +# model has no bundled MTP head; the repo ships the draft sidecar) + the +# ngram-mod fallback. Set SPEC_NONE=1 for a raw baseline. +SPEC_NONE=0 +SPEC_DEFAULT=1 +MTP=1 +SPEC_DRAFT_N_MAX=4 +# Standalone MTP draft head (Q8_0 MTP-only sidecar from the same repo). +SPEC_DRAFT_MODEL=/home/dank/models/Qwen3.8-27B-Uncensored-draft-Q8_0.gguf +SPEC_NGRAM_MOD_N_MIN=48 +SPEC_NGRAM_MOD_N_MAX=64 +SPEC_NGRAM_MOD_N_MATCH=24 + +# KV on GPU: K q8_0 + V f16 (mismatched, 16 full-attention layers x 4 +# heads x 256 dim: K = 1088 B/token, V = 2048 B/token -> 3136 B/token -> +# 784 MiB per FA layer at 262144, 12.25 GiB total). Mismatched K/V with +# Flash Attention requires the GGML_CUDA_FA_ALL_QUANTS rebuild (2026-08-17: +# head HIP + worker CUDA at 9b05354ec). Do not revert the binaries without +# also reverting this. +CACHE_TYPE_K=q8_0 +CACHE_TYPE_V=f16 + +# Batch sizing: production-verified values (2026-08-14). +BATCH_SIZE=8192 +UBATCH_SIZE=256 + +# Sampling — Qwen3.8 thinking-mode defaults. +TEMPERATURE=1.0 +TOP_K=20 +TOP_P=0.95 +MIN_P=0.0 + +THREADS=12 +THREADS_HTTP=1 + +# 4 (info): clean journal for interactive use. Raise to 10 only when +# debugging the layer split (the per-layer assignment lines are debug-level). +VERBOSITY=4 + +# 1: --direct-io model load. Required in practice for large models loaded +# over RPC (load hangs otherwise; visorcraft Strix RPC finding). +DIRECT_IO=1 diff --git a/scripts/frank-default-large.ling.env b/old/scripts/frank-default-large.ling.env similarity index 100% rename from scripts/frank-default-large.ling.env rename to old/scripts/frank-default-large.ling.env diff --git a/old/scripts/frank-default-large.service b/old/scripts/frank-default-large.service new file mode 100644 index 0000000..53fb69a --- /dev/null +++ b/old/scripts/frank-default-large.service @@ -0,0 +1,16 @@ +[Unit] +Description=llama.cpp Qwen3.8-27B pipeline-parallel head (Strix + RPC to Dankserver) (frank-default-large) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=dank +Group=dank +Environment="FRANK_PP_HEAD_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-large.env" +ExecStart=/home/dank/vllmScripts/scripts/serve-pp-head.sh +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/old/scripts/frank-default-small.env b/old/scripts/frank-default-small.env new file mode 100644 index 0000000..b3f7588 --- /dev/null +++ b/old/scripts/frank-default-small.env @@ -0,0 +1,33 @@ +# Tunables for frank-default-small (FastFlowLM NPU endpoint, dankclaw) +# Runtime: FLM 1.0.1 on the XDNA2 NPU (50 TOPS). Model choice, measured +# results, and the NPU memory budget are in docs/FastFlowLM NPU Inference.md. +# The NPU allocates from the unified pool shared with frank-default-large +# (the PP head). Keep the resident footprint small enough that the head +# keeps its headroom. +# +# 2026-08-17: qwythos:9b (Huihui-Qwythos-9B-Claude-Mythos-5-1M-abliterated, +# Qwen3.5-9B hybrid linear-attention base, Q4NX, 7.76 GB) chosen after the +# user's model swap (was qwen3.6-moe:35b-a3b, which stays installed for +# rollback). FLM upgraded 0.9.45 -> 1.0.1: the pack declares flm_version +# 1.0.0 and 0.9.45 refused to serve it. + +FLM_MODEL=qwythos:9b +FLM_HOST=0.0.0.0 +FLM_PORT=52625 +# Power mode: turbo. Smoke decode ~5-7 t/s on the 9B dense at 262k ctx +# (not benchmarked; see docs/FastFlowLM NPU Inference.md). +FLM_PMODE=turbo +# Load the ASR model (Whisper-V3-Turbo-NPU2) alongside the LLM. Exposes +# /v1/audio/transcriptions. Adds ~1 GiB to the resident footprint. +FLM_ASR=1 +# Maximum context length. Qwythos-9B native max is 262144 (Qwen3.5-9B +# hybrid: 8 of 32 layers are full attention, so the KV cache stays small). +# Verified 2026-08-17: loads at -c 262144 with turbo+asr on FLM 1.0.1; +# KV occupancy 0.007% at 19 tokens. +FLM_CTX_LEN=262144 +# Custom-model resolution: the qwythos:9b tag and its xclbin symlinks live +# in the user config, not the system registry. Without these, flm treats +# the tag as unknown and silently serves llama3.2:1b (see the unknown-tag +# fallback in docs/FastFlowLM NPU Inference.md). +FLM_CONFIG_PATH=/home/dank/.config/flm/model_list.json +FLM_XCLBIN_PATH=/home/dank/.config/flm diff --git a/old/scripts/frank-default-small.service b/old/scripts/frank-default-small.service new file mode 100644 index 0000000..cf8b12e --- /dev/null +++ b/old/scripts/frank-default-small.service @@ -0,0 +1,17 @@ +[Unit] +Description=FastFlowLM NPU inference (frank-default-small) on Dankclaw +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=dank +Group=dank +Environment="FLM_SMALL_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-small.env" +ExecStart=/home/dank/vllmScripts/scripts/serve-flm-small.sh +LimitMEMLOCK=infinity +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/scripts/frank-large-ling.service b/old/scripts/frank-large-ling.service similarity index 100% rename from scripts/frank-large-ling.service rename to old/scripts/frank-large-ling.service diff --git a/scripts/frank-pp-worker.env b/old/scripts/frank-pp-worker.env similarity index 100% rename from scripts/frank-pp-worker.env rename to old/scripts/frank-pp-worker.env diff --git a/scripts/frank-pp-worker.service b/old/scripts/frank-pp-worker.service similarity index 100% rename from scripts/frank-pp-worker.service rename to old/scripts/frank-pp-worker.service diff --git a/scripts/llama-server.service b/old/scripts/llama-server.service similarity index 100% rename from scripts/llama-server.service rename to old/scripts/llama-server.service diff --git a/scripts/pp-bench.py b/old/scripts/pp-bench.py similarity index 100% rename from scripts/pp-bench.py rename to old/scripts/pp-bench.py diff --git a/scripts/pp-test.sh b/old/scripts/pp-test.sh similarity index 100% rename from scripts/pp-test.sh rename to old/scripts/pp-test.sh diff --git a/scripts/prune-rpc-cache.sh b/old/scripts/prune-rpc-cache.sh similarity index 100% rename from scripts/prune-rpc-cache.sh rename to old/scripts/prune-rpc-cache.sh diff --git a/scripts/serve-flm-small.sh b/old/scripts/serve-flm-small.sh similarity index 100% rename from scripts/serve-flm-small.sh rename to old/scripts/serve-flm-small.sh diff --git a/scripts/serve-ling-flash.sh b/old/scripts/serve-ling-flash.sh similarity index 100% rename from scripts/serve-ling-flash.sh rename to old/scripts/serve-ling-flash.sh diff --git a/scripts/serve-ling3-vllm.sh b/old/scripts/serve-ling3-vllm.sh similarity index 100% rename from scripts/serve-ling3-vllm.sh rename to old/scripts/serve-ling3-vllm.sh diff --git a/scripts/serve-llama-frank-large.sh b/old/scripts/serve-llama-frank-large.sh similarity index 100% rename from scripts/serve-llama-frank-large.sh rename to old/scripts/serve-llama-frank-large.sh diff --git a/scripts/serve-llama-frank-small.sh b/old/scripts/serve-llama-frank-small.sh similarity index 100% rename from scripts/serve-llama-frank-small.sh rename to old/scripts/serve-llama-frank-small.sh diff --git a/scripts/serve-pp-head.sh b/old/scripts/serve-pp-head.sh similarity index 100% rename from scripts/serve-pp-head.sh rename to old/scripts/serve-pp-head.sh diff --git a/scripts/serve-pp-worker.sh b/old/scripts/serve-pp-worker.sh similarity index 100% rename from scripts/serve-pp-worker.sh rename to old/scripts/serve-pp-worker.sh diff --git a/scripts/serve-vllm-qwen36-dflash.sh b/old/scripts/serve-vllm-qwen36-dflash.sh similarity index 100% rename from scripts/serve-vllm-qwen36-dflash.sh rename to old/scripts/serve-vllm-qwen36-dflash.sh diff --git a/scripts/spec-matrix-bench.sh b/old/scripts/spec-matrix-bench.sh similarity index 100% rename from scripts/spec-matrix-bench.sh rename to old/scripts/spec-matrix-bench.sh diff --git a/scripts/test_dspark_stream.py b/old/scripts/test_dspark_stream.py similarity index 100% rename from scripts/test_dspark_stream.py rename to old/scripts/test_dspark_stream.py diff --git a/scripts/validate-vllm.sh b/old/scripts/validate-vllm.sh similarity index 100% rename from scripts/validate-vllm.sh rename to old/scripts/validate-vllm.sh diff --git a/scripts/deploy.sh b/scripts/deploy.sh old mode 100755 new mode 100644 index 2dc6019..9b2aba9 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -1,62 +1,49 @@ #!/usr/bin/env bash -# deploy.sh — Deploy the vllmScripts repo to a target and restart its service. +# deploy.sh - Deploy the vllmScripts repo to a target and restart its service. # # This script implements the second half of the git deployment workflow from # AGENTS.md. It assumes the local changes are committed and pushed, then: # 1. Pulls the repo on the target -# 2. Installs / enables the systemd unit -# 3. Drops page caches (dankclaw) and restarts the service +# 2. Symlinks the systemd unit +# 3. Reloads systemd and restarts the service # 4. Waits for the health endpoint and reports status # -# Sudo: when FRANK_SUDO_PW is set, remote sudo pipes it via `sudo -S` for -# headless runs. Otherwise plain `sudo` is used (tty or passwordless config). -# Prefer the passwordless restart helpers installed on each target -# (/usr/local/sbin/frank-restart-.sh); they cover the unit -# symlink, cache drop (dankclaw), daemon-reload, and restart in one root -# script. See docs/deployment.md -> Passwordless restart. +# Sudo: set FRANK_SUDO_PW so remote sudo reads the password from stdin +# (`sudo -S`). The passwordless restart helpers installed for the retired +# topology are obsolete and must not be used. # # Usage: # ./deploy.sh [--dry-run] # FRANK_SUDO_PW=... ./deploy.sh # -# Targets: -# dankclaw - llama.cpp server (frank-default-large.service, port 7770) -# dankserver - llama.cpp server (frank-default-small.service, port 7770) -# -# Example: -# ./deploy.sh dankclaw -# ./deploy.sh dankserver --dry-run +# Targets (2026-08-17 topology): +# dankserver - llama.cpp FastMTP (frank-default-large.service, port 7770) +# dankclaw - vLLM ROCm (frank-default-small.service, port 52625) set -euo pipefail # --------------------------------------------------------------------------- # Target configuration # --------------------------------------------------------------------------- -declare -A REMOTE_USER REMOTE_HOST REMOTE_DIR SERVICE SERVICE_HEALTH DROP_CACHES HELPER HEALTH_WAIT - -REMOTE_USER[dankclaw]=dank -REMOTE_HOST[dankclaw]=dankclaw -REMOTE_DIR[dankclaw]="/home/dank/vllmScripts" -SERVICE[dankclaw]="frank-default-large" -SERVICE_HEALTH[dankclaw]="http://127.0.0.1:7770/health" -# Drop page caches before restart: with the model's pages fully cached, the -# KV/graph allocation stalls at load (VRAM stuck at 28 GiB). See AGENTS.md. -DROP_CACHES[dankclaw]=1 -# Passwordless restart helper (root-owned, installed via /etc/sudoers.d/frank-large). -# When present, deploy.sh uses it instead of inline sudo commands. -HELPER[dankclaw]="/usr/local/sbin/frank-restart-large.sh" -# Health wait in iterations of 2s. Cold loads after the cache drop measured -# at up to 17 minutes on 2026-08-13; 600 iterations = 20 minutes. -HEALTH_WAIT[dankclaw]=600 +declare -A REMOTE_USER REMOTE_HOST REMOTE_DIR SERVICE SERVICE_HEALTH DROP_CACHES HEALTH_WAIT REMOTE_USER[dankserver]=dank REMOTE_HOST[dankserver]=dankserver REMOTE_DIR[dankserver]="/home/dank/vLLM" -SERVICE[dankserver]="frank-default-small" +SERVICE[dankserver]="frank-default-large" SERVICE_HEALTH[dankserver]="http://127.0.0.1:7770/health" DROP_CACHES[dankserver]=0 -HELPER[dankserver]="/usr/local/sbin/frank-restart-small.sh" +# Cold load: --no-mmap reads 15.7 GB through RAM; allow 10 minutes. HEALTH_WAIT[dankserver]=300 +REMOTE_USER[dankclaw]=dank +REMOTE_HOST[dankclaw]=dankclaw +REMOTE_DIR[dankclaw]="/home/dank/vllmScripts" +SERVICE[dankclaw]="frank-default-small" +SERVICE_HEALTH[dankclaw]="http://127.0.0.1:52625/health" +DROP_CACHES[dankclaw]=0 +# vLLM loads a 38 GB checkpoint; allow 20 minutes. +HEALTH_WAIT[dankclaw]=600 + TARGET="${1:-}" DRY_RUN=0 if [ "${2:-}" = "--dry-run" ]; then @@ -67,7 +54,7 @@ fi # Validation # --------------------------------------------------------------------------- if [ -z "${TARGET:-}" ]; then - echo "Usage: $0 [--dry-run]" + echo "Usage: $0 [--dry-run]" exit 1 fi if [ -z "${REMOTE_HOST[$TARGET]:-}" ]; then @@ -78,8 +65,6 @@ fi # --------------------------------------------------------------------------- # Sudo wrapper (headless-friendly) # --------------------------------------------------------------------------- -# When FRANK_SUDO_PW is set, remote sudo reads the password from stdin via -# `sudo -S`. Otherwise plain `sudo` is used (tty or passwordless config). SUDO_PREFIX="sudo" if [ -n "${FRANK_SUDO_PW:-}" ]; then SUDO_PREFIX="printf '%s\n' '$FRANK_SUDO_PW' | sudo -S" @@ -131,8 +116,6 @@ ${SUDO_PREFIX} sh -c 'sync; echo 3 > /proc/sys/vm/drop_caches' " fi -HELPER_PATH="${HELPER[$TARGET]:-}" - REMOTE_SCRIPT=" set -euo pipefail cd '${REMOTE_DIR[$TARGET]}' @@ -140,27 +123,12 @@ cd '${REMOTE_DIR[$TARGET]}' echo '[2/4] Pulling repo on ${TARGET}...' git pull origin main -if [ -x '${HELPER_PATH}' ]; then - # Passwordless restart helper (sudoers NOPASSWD) - preferred path. - echo '[3/4] Restarting via passwordless helper ${HELPER_PATH}...' - ${SUDO_PREFIX} '${HELPER_PATH}' -else - echo '[3/4] Helper not installed; using inline sudo commands...' - # Install / symlink the systemd unit - case '${TARGET}' in - dankclaw) - ${SUDO_PREFIX} ln -sf '${REMOTE_DIR[$TARGET]}/scripts/frank-default-large.service' /etc/systemd/system/frank-default-large.service - ;; - dankserver) - ${SUDO_PREFIX} ln -sf '${REMOTE_DIR[$TARGET]}/scripts/frank-default-small.service' /etc/systemd/system/frank-default-small.service - ;; - esac - ${DROP_CACHES_CMD} - echo '[3/4] Reload and restart ${SERVICE[$TARGET]}...' - ${SUDO_PREFIX} systemctl daemon-reload - ${SUDO_PREFIX} systemctl enable '${SERVICE[$TARGET]}' - ${SUDO_PREFIX} systemctl restart '${SERVICE[$TARGET]}' -fi +echo '[3/4] Symlink unit and restart ${SERVICE[$TARGET]}...' +${SUDO_PREFIX} ln -sf '${REMOTE_DIR[$TARGET]}/scripts/${SERVICE[$TARGET]}.service' /etc/systemd/system/${SERVICE[$TARGET]}.service +${DROP_CACHES_CMD} +${SUDO_PREFIX} systemctl daemon-reload +${SUDO_PREFIX} systemctl enable '${SERVICE[$TARGET]}' +${SUDO_PREFIX} systemctl restart '${SERVICE[$TARGET]}' " if [ "$DRY_RUN" = "1" ]; then diff --git a/scripts/frank-default-large.env b/scripts/frank-default-large.env index 875e62c..6b5a402 100644 --- a/scripts/frank-default-large.env +++ b/scripts/frank-default-large.env @@ -1,88 +1,58 @@ -# Head tunables — Qwen3.8-27B pipeline-parallel head (Dankclaw). +# Tunables for frank-default-large (llama.cpp, dankserver, RTX 3090). # -# The head is the LOCAL side of the split. It runs llama-server (b10433, -# HIP+Vulkan+RPC build in /home/dank/llama.cpp/build) with --rpc pointing at -# the worker on Dankserver. The model file lives on this machine only; the -# worker receives its tensor share over the network at load time. +# Runtime: patched llama-server (HauhauCS FastMTP build) serving +# HauhauCS/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-MTP-GGUF IQ4_XS with +# the FastMTP 32K draft sidecar and the ngram-mod fallback. Single machine, +# no pipeline split. Alias frank-default-large, port 7770. # -# CRITICAL: this script must NOT pass --device. When --device is set, -# llama.cpp uses exactly that device list and silently drops the RPC device -# (src/llama.cpp llama_prepare_model_devices). Default device selection -# inserts RPC servers at the FRONT of the device list, so the split becomes -# [RPC0, iGPU] and --tensor-split index 0 is the worker. -# -# WARNING: do not name any variable here TEMP, HOST, or GPU_MAX_HEAP_SIZE -# (HIP runtime collision, see the 2026-08-13 TEMP incident). +# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The ROCm +# HIP runtime reads those bare env vars (see the 2026-08-13 TEMP incident). +# This box is NVIDIA/CUDA, but the rule still applies to keep the env files +# portable. + +# Model files (downloaded to this directory from the HF repo). +MODEL_DIR=/home/dank/models/hauhaucs-qwen38 +MODEL_FILE=Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-IQ4_XS.gguf +DRAFT_FILE=Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-FastMTP-32K.gguf +# Patched llama.cpp binary. The FastMTP sidecar needs the patch; see AGENTS.md. +LLAMA_BIN=/home/dank/llama.cpp-fastmtp/build/bin/llama-server -MODEL_DIR=/home/dank/models -MODEL_FILE=Qwen3.8-27B-Uncensored-Q6_K.gguf -HEAD_BIN=/home/dank/llama.cpp/build/bin/llama-server ALIAS=frank-default-large BIND_ADDR=0.0.0.0 LISTEN_PORT=7770 -# Comma-separated RPC endpoints (host:port). One worker here. -RPC_SERVERS=192.168.0.235:50052 - -# Model split fractions, one per device in order [RPC0, iGPU]. -# 0.68,0.32 = 45 of 65 blocks on the 3090 (blocks 0-44) - the maximum -# for the Q6_K + q8_0/f16 KV config, measured 2026-08-17 phase 2: -# - KV per FA layer = 784 MiB (K 1088 B/token + V 2048 B/token @262144) -# - weights(f) = 20334 MB layer pool x f (GGUF scan, exact); embd on -# the head side (RPC0 buffer observed 10429.88 MiB at f=0.53) -# - KV-per-device count = round(f x 66) layers -> 45 layers = 11 FA -# (3..43) -> 8624 MiB. 46 layers (f>=0.7045) still 11 FA but margin -# drops to ~0.4 GiB; 48 layers (f>=0.7197) adds FA block 47 -> KV -# 9408 -> OOM by ~0.85 GiB. -# Predicted: 13430 (weights) + 8624 (KV) + ~530 (RS) + 189 (compute) = -# ~22773 MiB -> ~0.7 GiB free. Draft model runs on ROCm0 only (measured -# phase 1: 1718 MiB, no worker impact). -TENSOR_SPLIT=0.68,0.32 - -# Context pool. 262144 total = 1 parallel slot x 262144 per slot -# (n_ctx_seq = n_ctx / n_seq_max, llama-context.cpp:293). -CTX_SIZE=262144 +# Context, single slot. 131072 (128K) leaves ~1.4 GiB VRAM headroom on the +# 3090 with IQ4_XS + FastMTP + K q8_0 / V f16 KV. Lower to 122880 (120K) for +# ~1.8 GiB headroom. KV per token = 16 attention layers x 4 KV heads x 256 +# dim: K q8_0 (1088 B/token) + V f16 (2048 B/token) = 3136 B/token/layer. +CTX_SIZE=131072 PARALLEL=1 -# Speculative decoding. MTP with the STANDALONE draft head (the Q6_K -# model has no bundled MTP head; the repo ships the draft sidecar) + the -# ngram-mod fallback. Set SPEC_NONE=1 for a raw baseline. -SPEC_NONE=0 +# Speculative decoding. FastMTP (MTP depth 2) with the ngram-mod fallback. SPEC_DEFAULT=1 MTP=1 -SPEC_DRAFT_N_MAX=4 -# Standalone MTP draft head (Q8_0 MTP-only sidecar from the same repo). -SPEC_DRAFT_MODEL=/home/dank/models/Qwen3.8-27B-Uncensored-draft-Q8_0.gguf +SPEC_DRAFT_N_MAX=2 +SPEC_DRAFT_NGL=all +SPEC_DRAFT_P_MIN=0 SPEC_NGRAM_MOD_N_MIN=48 SPEC_NGRAM_MOD_N_MAX=64 SPEC_NGRAM_MOD_N_MATCH=24 -# KV on GPU: K q8_0 + V f16 (mismatched, 16 full-attention layers x 4 -# heads x 256 dim: K = 1088 B/token, V = 2048 B/token -> 3136 B/token -> -# 784 MiB per FA layer at 262144, 12.25 GiB total). Mismatched K/V with -# Flash Attention requires the GGML_CUDA_FA_ALL_QUANTS rebuild (2026-08-17: -# head HIP + worker CUDA at 9b05354ec). Do not revert the binaries without -# also reverting this. +# KV quantization on GPU: K q8_0 + V f16 (user request: K=8, V=16). CACHE_TYPE_K=q8_0 CACHE_TYPE_V=f16 -# Batch sizing: production-verified values (2026-08-14). -BATCH_SIZE=8192 -UBATCH_SIZE=256 +# Batch sizing (user request). +BATCH_SIZE=2048 +UBATCH_SIZE=1024 -# Sampling — Qwen3.8 thinking-mode defaults. +# Sampling - Qwen3.8 thinking-mode defaults. TEMPERATURE=1.0 TOP_K=20 TOP_P=0.95 MIN_P=0.0 -THREADS=12 +THREADS=8 THREADS_HTTP=1 -# 4 (info): clean journal for interactive use. Raise to 10 only when -# debugging the layer split (the per-layer assignment lines are debug-level). VERBOSITY=4 - -# 1: --direct-io model load. Required in practice for large models loaded -# over RPC (load hangs otherwise; visorcraft Strix RPC finding). -DIRECT_IO=1 diff --git a/scripts/frank-default-large.service b/scripts/frank-default-large.service index 53fb69a..78ac03f 100644 --- a/scripts/frank-default-large.service +++ b/scripts/frank-default-large.service @@ -1,5 +1,5 @@ [Unit] -Description=llama.cpp Qwen3.8-27B pipeline-parallel head (Strix + RPC to Dankserver) (frank-default-large) +Description=llama.cpp Qwen3.8-27B Aggressive IQ4_XS + FastMTP (dankserver, frank-default-large) After=network-online.target Wants=network-online.target @@ -7,8 +7,8 @@ Wants=network-online.target Type=simple User=dank Group=dank -Environment="FRANK_PP_HEAD_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-large.env" -ExecStart=/home/dank/vllmScripts/scripts/serve-pp-head.sh +Environment="FRANK_LARGE_ENV_FILE=/home/dank/vLLM/scripts/frank-default-large.env" +ExecStart=/home/dank/vLLM/scripts/serve-frank-large.sh Restart=on-failure RestartSec=10 diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index b3f7588..8efd650 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -1,33 +1,40 @@ -# Tunables for frank-default-small (FastFlowLM NPU endpoint, dankclaw) -# Runtime: FLM 1.0.1 on the XDNA2 NPU (50 TOPS). Model choice, measured -# results, and the NPU memory budget are in docs/FastFlowLM NPU Inference.md. -# The NPU allocates from the unified pool shared with frank-default-large -# (the PP head). Keep the resident footprint small enough that the head -# keeps its headroom. +# Tunables for frank-default-small (vLLM, dankclaw, Strix Halo gfx1151). # -# 2026-08-17: qwythos:9b (Huihui-Qwythos-9B-Claude-Mythos-5-1M-abliterated, -# Qwen3.5-9B hybrid linear-attention base, Q4NX, 7.76 GB) chosen after the -# user's model swap (was qwen3.6-moe:35b-a3b, which stays installed for -# rollback). FLM upgraded 0.9.45 -> 1.0.1: the pack declares flm_version -# 1.0.0 and 0.9.45 refused to serve it. +# Runtime: vLLM (ROCm) in /home/dank/vllm-venv serving +# Avesed/Qwen3.6-35B-A3B-INT8-W8A8 (compressed-tensors INT8 W8A8, MoE 35B +# total / 3B active, Qwen3.5-MoE hybrid: 30 of 40 layers are GatedDeltaNet +# linear attention). Alias frank-default-small, port 52625. +# +# The ROCm vLLM install was created with: +# uv venv --python /usr/bin/python3.12 /home/dank/vllm-venv +# uv pip install --python /home/dank/vllm-venv vllm --torch-backend=auto +# The --torch-backend=auto resolver pulls the AMD ROCm wheels (torch + +# amd-torch-device-gfx1151 + vllm rocm variant) from the AMD index. -FLM_MODEL=qwythos:9b -FLM_HOST=0.0.0.0 -FLM_PORT=52625 -# Power mode: turbo. Smoke decode ~5-7 t/s on the 9B dense at 262k ctx -# (not benchmarked; see docs/FastFlowLM NPU Inference.md). -FLM_PMODE=turbo -# Load the ASR model (Whisper-V3-Turbo-NPU2) alongside the LLM. Exposes -# /v1/audio/transcriptions. Adds ~1 GiB to the resident footprint. -FLM_ASR=1 -# Maximum context length. Qwythos-9B native max is 262144 (Qwen3.5-9B -# hybrid: 8 of 32 layers are full attention, so the KV cache stays small). -# Verified 2026-08-17: loads at -c 262144 with turbo+asr on FLM 1.0.1; -# KV occupancy 0.007% at 19 tokens. -FLM_CTX_LEN=262144 -# Custom-model resolution: the qwythos:9b tag and its xclbin symlinks live -# in the user config, not the system registry. Without these, flm treats -# the tag as unknown and silently serves llama3.2:1b (see the unknown-tag -# fallback in docs/FastFlowLM NPU Inference.md). -FLM_CONFIG_PATH=/home/dank/.config/flm/model_list.json -FLM_XCLBIN_PATH=/home/dank/.config/flm +VLLM_BIN=/home/dank/vllm-venv/bin/vllm +MODEL=Avesed/Qwen3.6-35B-A3B-INT8-W8A8 +ALIAS=frank-default-small +BIND_ADDR=0.0.0.0 +LISTEN_PORT=52625 + +# Full native context. The KV cache stays unquantized (auto = model dtype, +# bf16). Worst-case KV at 262144 x 4 sequences is ~10.7 GiB; weights are +# ~35 GiB; total fits the Strix Halo unified pool with room to spare. +MAX_MODEL_LEN=262144 +KV_CACHE_DTYPE=auto + +# Concurrency and batching (user request). +MAX_NUM_SEQS=4 +MAX_NUM_BATCHED_TOKENS=16384 + +# vLLM has no separate ubatch knob. The scheduler issues one batch per step +# with up to max-num-batched-tokens tokens; the KV cache is preallocated at +# max-model-len. Set gpu-memory-utilization below 1.0 so the unified pool +# keeps headroom for the host. +GPU_MEMORY_UTILIZATION=0.92 + +# ngram speculative decoding (vLLM ngram proposer). +# prompt_lookup_max = ngram window; num_speculative_tokens = draft depth. +NGRAM_LOOKUP_MAX=5 +NGRAM_LOOKUP_MIN=1 +NUM_SPEC_TOKENS=5 diff --git a/scripts/frank-default-small.service b/scripts/frank-default-small.service index cf8b12e..779715c 100644 --- a/scripts/frank-default-small.service +++ b/scripts/frank-default-small.service @@ -1,5 +1,5 @@ [Unit] -Description=FastFlowLM NPU inference (frank-default-small) on Dankclaw +Description=vLLM Qwen3.6-35B-A3B INT8 W8A8 ngram spec (dankclaw, frank-default-small) After=network-online.target Wants=network-online.target @@ -7,9 +7,9 @@ Wants=network-online.target Type=simple User=dank Group=dank -Environment="FLM_SMALL_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-small.env" -ExecStart=/home/dank/vllmScripts/scripts/serve-flm-small.sh -LimitMEMLOCK=infinity +Environment="FRANK_SMALL_ENV_FILE=/home/dank/vllmScripts/scripts/frank-default-small.env" +Environment="HF_HOME=/home/dank/.cache/huggingface" +ExecStart=/home/dank/vllmScripts/scripts/serve-frank-small.sh Restart=on-failure RestartSec=10 diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh new file mode 100644 index 0000000..38eb5d2 --- /dev/null +++ b/scripts/serve-frank-large.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Serve the Qwen3.8-27B HauhauCS Aggressive IQ4_XS on Dankserver (RTX 3090). +# +# Runs a patched llama-server (HauhauCS FastMTP build, commit 4df29be + +# HauhauCS-FastMTP-llama.cpp.patch) with the FastMTP draft sidecar and the +# ngram-mod fallback. There is no pipeline split: the whole model lives on +# the 3090. +# +# The FastMTP draft sidecar requires the patched executable. Loading the +# draft with the stock binary fails with "expected 5120, 248320, got 5120, +# 32768" (see the HauhauCS repo README). +# +# Tunables come from frank-default-large.env in this directory unless +# FRANK_LARGE_ENV_FILE overrides that path. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}" +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +# --- defaults (used when the env file does not set them) ------------------- +: "${MODEL_DIR:=/home/dank/models/hauhaucs-qwen38}" +: "${MODEL_FILE:=Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-IQ4_XS.gguf}" +: "${DRAFT_FILE:=Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-FastMTP-32K.gguf}" +: "${LLAMA_BIN:=/home/dank/llama.cpp-fastmtp/build/bin/llama-server}" +: "${ALIAS:=frank-default-large}" +: "${BIND_ADDR:=0.0.0.0}" +: "${LISTEN_PORT:=7770}" +: "${CTX_SIZE:=131072}" +: "${PARALLEL:=1}" +: "${SPEC_DEFAULT:=1}" +: "${MTP:=1}" +: "${SPEC_DRAFT_N_MAX:=2}" +: "${SPEC_DRAFT_NGL:=all}" +: "${SPEC_DRAFT_P_MIN:=0}" +: "${SPEC_NGRAM_MOD_N_MIN:=48}" +: "${SPEC_NGRAM_MOD_N_MAX:=64}" +: "${SPEC_NGRAM_MOD_N_MATCH:=24}" +: "${CACHE_TYPE_K:=q8_0}" +: "${CACHE_TYPE_V:=f16}" +: "${BATCH_SIZE:=2048}" +: "${UBATCH_SIZE:=1024}" +: "${TEMPERATURE:=1.0}" +: "${TOP_K:=20}" +: "${TOP_P:=0.95}" +: "${MIN_P:=0.0}" +: "${THREADS:=8}" +: "${THREADS_HTTP:=1}" +: "${VERBOSITY:=4}" + +MODEL="$MODEL_DIR/$MODEL_FILE" +DRAFT="$MODEL_DIR/$DRAFT_FILE" + +if [[ ! -f "$MODEL" ]]; then + echo "missing required file: $MODEL" >&2 + exit 1 +fi +if [[ ! -f "$DRAFT" ]]; then + echo "missing draft file: $DRAFT" >&2 + exit 1 +fi +if [[ ! -x "$LLAMA_BIN" ]]; then + echo "missing llama-server binary: $LLAMA_BIN" >&2 + exit 1 +fi + +# --- speculative decoding: FastMTP (MTP) first, then ngram-mod fallback ---- +SPEC_ARGS=() +SPEC_ARGS+=(--spec-type draft-mtp) +SPEC_ARGS+=(--spec-draft-model "$DRAFT") +SPEC_ARGS+=(--spec-draft-n-max "$SPEC_DRAFT_N_MAX") +SPEC_ARGS+=(--spec-draft-ngl "$SPEC_DRAFT_NGL") +SPEC_ARGS+=(--spec-draft-p-min "$SPEC_DRAFT_P_MIN") +if [[ "$SPEC_DEFAULT" == "1" ]]; then + SPEC_ARGS+=(--spec-default) + SPEC_ARGS+=(--spec-ngram-mod-n-min "$SPEC_NGRAM_MOD_N_MIN") + SPEC_ARGS+=(--spec-ngram-mod-n-max "$SPEC_NGRAM_MOD_N_MAX") + SPEC_ARGS+=(--spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH") +fi + +exec "$LLAMA_BIN" \ + --model "$MODEL" \ + "${SPEC_ARGS[@]}" \ + --gpu-layers all \ + --no-mmap \ + --flash-attn on \ + --ctx-size "$CTX_SIZE" \ + --parallel "$PARALLEL" \ + --cache-type-k "$CACHE_TYPE_K" \ + --cache-type-v "$CACHE_TYPE_V" \ + --batch-size "$BATCH_SIZE" \ + --ubatch-size "$UBATCH_SIZE" \ + --temp "$TEMPERATURE" \ + --top-k "$TOP_K" \ + --top-p "$TOP_P" \ + --min-p "$MIN_P" \ + --threads "$THREADS" \ + --threads-http "$THREADS_HTTP" \ + --jinja \ + --alias "$ALIAS" \ + -lv "$VERBOSITY" \ + --host "$BIND_ADDR" \ + --port "$LISTEN_PORT" diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh new file mode 100644 index 0000000..48472a7 --- /dev/null +++ b/scripts/serve-frank-small.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Serve Avesed/Qwen3.6-35B-A3B-INT8-W8A8 via vLLM on Dankclaw (Strix Halo). +# +# Runs the ROCm vLLM build in /home/dank/vllm-venv. The model is a +# compressed-tensors INT8 W8A8 quant (MoE, 35B total / 3B active). Full +# 262144 context, unquantized KV cache, 4 concurrent sequences, ngram +# speculative decoding. +# +# Tunables come from frank-default-small.env in this directory unless +# FRANK_SMALL_ENV_FILE overrides that path. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-default-small.env}" +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +# --- defaults (used when the env file does not set them) ------------------- +: "${VLLM_BIN:=/home/dank/vllm-venv/bin/vllm}" +: "${MODEL:=Avesed/Qwen3.6-35B-A3B-INT8-W8A8}" +: "${ALIAS:=frank-default-small}" +: "${BIND_ADDR:=0.0.0.0}" +: "${LISTEN_PORT:=52625}" +: "${MAX_MODEL_LEN:=262144}" +: "${KV_CACHE_DTYPE:=auto}" +: "${MAX_NUM_SEQS:=4}" +: "${MAX_NUM_BATCHED_TOKENS:=16384}" +: "${GPU_MEMORY_UTILIZATION:=0.92}" +: "${NGRAM_LOOKUP_MAX:=5}" +: "${NGRAM_LOOKUP_MIN:=1}" +: "${NUM_SPEC_TOKENS:=5}" + +if [[ ! -x "$VLLM_BIN" ]]; then + echo "missing vllm binary: $VLLM_BIN" >&2 + exit 1 +fi + +exec "$VLLM_BIN" serve "$MODEL" \ + --host "$BIND_ADDR" \ + --port "$LISTEN_PORT" \ + --served-model-name "$ALIAS" \ + --max-model-len "$MAX_MODEL_LEN" \ + --kv-cache-dtype "$KV_CACHE_DTYPE" \ + --max-num-seqs "$MAX_NUM_SEQS" \ + --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" \ + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \ + --speculative-config "{\"method\":\"ngram\",\"prompt_lookup_max\":$NGRAM_LOOKUP_MAX,\"prompt_lookup_min\":$NGRAM_LOOKUP_MIN,\"num_speculative_tokens\":$NUM_SPEC_TOKENS}" \ + --reasoning-parser qwen3 \ + --trust-remote-code