Frank
9971bdef0f
- Podman container with CUDA GPU access - DFlash speculative decoding via --spec-method dflash - FP8 KV cache, 4 parallel streams, 262K context - CPU expert offloading via --cpu-offload-gb - systemd service for persistent runtime
25 lines
659 B
Desktop File
25 lines
659 B
Desktop File
[Unit]
|
|
Description=vLLM Qwen3.6-35B-A3B-FP8 with DFlash Drafting (Podman)
|
|
After=network-online.target
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
Type=simple
|
|
User=dank
|
|
Group=dank
|
|
WorkingDirectory=/home/dank/Projects/vllmScripts
|
|
Environment="PORT=8000"
|
|
Environment="GPU_MEM_UTIL=0.95"
|
|
Environment="MAX_NUM_SEQS=4"
|
|
Environment="MAX_MODEL_LEN=262144"
|
|
Environment="KV_CACHE_DTYPE=fp8"
|
|
Environment="NUM_SPEC_TOKENS=8"
|
|
ExecStart=/home/dank/Projects/vllmScripts/scripts/serve-vllm-qwen36-dflash.sh
|
|
ExecStop=/usr/bin/podman stop vllm-qwen36-dflash
|
|
ExecStopPost=/usr/bin/podman rm -f vllm-qwen36-dflash
|
|
Restart=on-failure
|
|
RestartSec=5
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|