Frank
1949f0651b
- Abandon vLLM/FP8 (Triton compilation hangs, ~0.5 tok/s on Strix Halo) - llama.cpp v10133 with ROCm0 offload, DFlash draft model - Main: unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 (~22GB) - Draft: Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 (~1.8GB) - 3 parallel streams, 131072 context/stream - systemd service with minimal sandboxing (ROCm requires /dev/kfd access) - Baseline: 7.62 tok/s (threshold 5.0 tok/s)
15 lines
596 B
Desktop File
15 lines
596 B
Desktop File
[Unit]
|
|
Description=llama.cpp DFlash+MTP Inference Server (Qwen3.6-27B on ROCm)
|
|
After=network.target
|
|
|
|
[Service]
|
|
Type=simple
|
|
User=dank
|
|
Group=dank
|
|
ExecStart=/home/dank/llama.cpp/build/bin/llama-server --hf-repo unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0 --spec-draft-hf Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0 --spec-draft-n-max 16 --device ROCm0 --spec-draft-device ROCm0 --gpu-layers all --fit on --fit-target 512 --parallel 3 --ctx-size 131072 --batch-size 4096 --port 8080 --host 0.0.0.0 --threads 16 --load-mode mlock
|
|
Restart=on-failure
|
|
RestartSec=10
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|