frank-small: SGLang MiMo-V2.6-Distill-Qwen-9B INT8 (qwen3_5 hybrid, RTX 3090)

Replaces llama.cpp Gemma4-12B-QAT (payload + binary retained as rollback).
SGLang in a self-contained python3.12 venv with an extracted CUDA 13.0
nvcc toolchain for the sgl-kernel/flashinfer JIT.

Shape: fp8 KV, pinned 606,208-token pool / 4 x 151,552 slots, bf16 mamba
states pinned at 32 slots (the engine autofills the ssm pool to a fixed
~5.1 GB regardless of dtype), eager prefill (no headroom for graph
capture), no spec decode (the checkpoint ships no mtp.* tensors despite
mtp_num_hidden_layers=1 — NEXTN drafts with an uninitialised 4.24 GiB
ghost at 0% acceptance).

Verified: 83.9 tok/s decode, 170k single prompt, 4 concurrent 54k
requests, vision (in-checkpoint tower), reasoning_content field.

scripts/deploy-remote.sh

@@ -1177,23 +1177,39 @@ deploy_llm_large() {
     wait_for_health frank-large http://127.0.0.1:7770/health 1200
 }
 
-# deploy_llm_small — frank-small is mainline llama.cpp on the RTX 3090, as a
-# ROOT unit: the runtime is a native CUDA binary, so there is no container
-# image to install and no generated unit to avoid. relink_unit points
-# /etc/systemd/system/frank-small.service at the repo file and boot persistence
-# is that unit's [Install] section.
+# deploy_llm_small — frank-small is SGLang (python3.12 venv) on the RTX 3090,
+# as a ROOT unit: the runtime is a self-contained venv, so there is no
+# container image to install and no generated unit to avoid. relink_unit
+# points /etc/systemd/system/frank-small.service at the repo file and boot
+# persistence is that unit's [Install] section.
 #
 # It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
 # not shared: frank-large's engine runs on the gfx1151 iGPU.
-LLM_SMALL_MODEL="${LLM_SMALL_MODEL:-/home/dank/models/Gemma4-12B-QAT/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf}"
+LLM_SMALL_MODEL_DIR="${LLM_SMALL_MODEL_DIR:-/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound}"
+LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/sglang-cu128/bin/sglang}"
 
 deploy_llm_small() {
-    if [[ ! -f "$LLM_SMALL_MODEL" ]]; then
-        echo "ERROR: model absent at $LLM_SMALL_MODEL" >&2
-        echo "frank-small: fetch HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced (Q4_K_M, the mmproj and the MTP sidecar) first;" >&2
+    # Refuse rather than restart-loop: a unit whose runtime or payload is
+    # missing never becomes healthy, and a green deploy would leave the
+    # endpoint dark with nothing in the log to say why.
+    if [[ ! -x "$LLM_SMALL_RUNTIME" ]]; then
+        echo "ERROR: SGLang runtime absent at $LLM_SMALL_RUNTIME" >&2
+        echo "frank-small: build the venv (python3.12 -m venv /home/dank/GitPackages/sglang-cu128" >&2
+        echo "frank-small: and pip install 'sglang[all]' into it) first." >&2
+        return 1
+    fi
+    if [[ ! -f "$LLM_SMALL_MODEL_DIR/model.safetensors.index.json" ]]; then
+        echo "ERROR: payload index absent at $LLM_SMALL_MODEL_DIR" >&2
+        echo "frank-small: run scripts/download-frank-small.sh first;" >&2
         echo "frank-small: refusing to start a unit that cannot load." >&2
         return 1
     fi
+    local count
+    count="$(find "$LLM_SMALL_MODEL_DIR" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l)"
+    if [[ "$count" -ne 8 ]]; then
+        echo "ERROR: payload shard count wrong: $LLM_SMALL_MODEL_DIR (found $count, want 8)" >&2
+        return 1
+    fi
     relink_unit frank-small
     run_privileged systemctl daemon-reload
     run_privileged systemctl enable frank-small.service