feat: initial vLLM Qwen3.6-35B-A3B-FP8 serve script with DFlash drafting on dankserver

- Podman container with CUDA GPU access
- DFlash speculative decoding via --spec-method dflash
- FP8 KV cache, 4 parallel streams, 262K context
- CPU expert offloading via --cpu-offload-gb
- systemd service for persistent runtime
This commit is contained in:
Frank committed 2026-08-07 18:28:38 -07:00
1 parent 5452bb63f1
commit 9971bdef0f
3 files changed
+224 -16

No files matched your search

+2 -10
View File
@@ -7,6 +7,7 @@ Wants=network-online.target
Type=simple
User=dank
Group=dank
WorkingDirectory=/home/dank/Projects/vllmScripts
Environment="PORT=8000"
Environment="GPU_MEM_UTIL=0.95"
Environment="MAX_NUM_SEQS=4"
@@ -17,16 +18,7 @@ ExecStart=/home/dank/Projects/vllmScripts/scripts/serve-vllm-qwen36-dflash.sh
ExecStop=/usr/bin/podman stop vllm-qwen36-dflash
ExecStopPost=/usr/bin/podman rm -f vllm-qwen36-dflash
Restart=on-failure
RestartSec=10
TimeoutStopSec=60
# Ensure GPU devices are available
DevicePolicy=closed
DeviceAllow=/dev/nvidia0 rwm
DeviceAllow=/dev/nvidiactl rwm
DeviceAllow=/dev/nvidia-uvm rwm
DeviceAllow=/dev/nvidia-uvm-tools rwm
DeviceAllow=/dev/nvidia-modeset rwm
RestartSec=5
[Install]
WantedBy=multi-user.target