#!/usr/bin/env bash
# Serve Qwen3.6-27B with DFlash + MTP speculative decoding via llama.cpp on AMD Strix Halo (gfx1151)
# Uses llama.cpp built with ROCm support
# 3 parallel streams, 131072 context each
set -euo pipefail

SERVER_BIN="/home/dank/llama.cpp/build/bin/llama-server"
PORT="${LLAMA_PORT:-8080}"
N_PARALLEL="${LLAMA_N_PARALLEL:-3}"
CTX_SIZE="${LLAMA_CTX_SIZE:-600000}"
BATCH_SIZE="${LLAMA_BATCH_SIZE:-4096}"
DRAFT_N_MAX="${LLAMA_DRAFT_N_MAX:-16}"

# Models via Hugging Face
TARGET_REPO="unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0"
DRAFT_REPO="Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q8_0"

echo "Starting llama.cpp server..."
echo "  Target model: ${TARGET_REPO}"
echo "  Draft model:  ${DRAFT_REPO}"
echo "  Parallel streams: ${N_PARALLEL}"
echo "  Context/stream: ${CTX_SIZE}"
echo "  Draft tokens: ${DRAFT_N_MAX}"
echo ""

exec "${SERVER_BIN}" \
    --hf-repo "${TARGET_REPO}" \
    --spec-draft-hf "${DRAFT_REPO}" \
    --spec-type draft-dflash \
    --spec-draft-n-max "${DRAFT_N_MAX}" \
    --device ROCm0 \
    --spec-draft-device ROCm0 \
    --gpu-layers all \
    --fit on \
    --fit-target 512 \
    --parallel "${N_PARALLEL}" \
    --ctx-size "${CTX_SIZE}" \
    --batch-size "${BATCH_SIZE}" \
    --port "${PORT}" \
    --host "0.0.0.0" \
    --threads 16 \
    --alias frank-default-large \
    --load-mode none