feat: initial vLLM Qwen3.6-35B-A3B-FP8 serve script with DFlash drafting on dankserver
- Podman container with CUDA GPU access - DFlash speculative decoding via --spec-method dflash - FP8 KV cache, 4 parallel streams, 262K context - CPU expert offloading via --cpu-offload-gb - systemd service for persistent runtime
This commit is contained in:
1 parent
5452bb63f1
commit
9971bdef0f
3 files changed
+224
-16
No files matched your search
@@ -7,6 +7,7 @@ Wants=network-online.target
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
WorkingDirectory=/home/dank/Projects/vllmScripts
|
||||
Environment="PORT=8000"
|
||||
Environment="GPU_MEM_UTIL=0.95"
|
||||
Environment="MAX_NUM_SEQS=4"
|
||||
@@ -17,16 +18,7 @@ ExecStart=/home/dank/Projects/vllmScripts/scripts/serve-vllm-qwen36-dflash.sh
|
||||
ExecStop=/usr/bin/podman stop vllm-qwen36-dflash
|
||||
ExecStopPost=/usr/bin/podman rm -f vllm-qwen36-dflash
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
TimeoutStopSec=60
|
||||
|
||||
# Ensure GPU devices are available
|
||||
DevicePolicy=closed
|
||||
DeviceAllow=/dev/nvidia0 rwm
|
||||
DeviceAllow=/dev/nvidiactl rwm
|
||||
DeviceAllow=/dev/nvidia-uvm rwm
|
||||
DeviceAllow=/dev/nvidia-uvm-tools rwm
|
||||
DeviceAllow=/dev/nvidia-modeset rwm
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Reference in new issue
Block a user