Frank
a2b39b179a
Runtime swap on the RTX 3090 (port 7771): - vLLM gemma-x86_64-cu129 dev build, digest-pinned, audio extra pip-installed into localhost/vllm/vllm-openai:frank-small (Dockerfile, no COPY lines) - google/gemma-4-12B-it-qat-w4a16-ct (10.26 GiB int4) + Google's bf16 MTP assistant (845 MB) — the w4a16-ct assistant cannot draft in vLLM (drafter path skips Marlin; acceptance 1.00), bf16 measures 2.80-3.56 - root systemd unit -> rootless Quadlet (dankclaw/frank-small/quadlet/); manual CUDA device+lib mounts (no nvidia-container-toolkit on host) - bf16 KV (fp8 hardware-dead on sm_86), 289,647-token pool, 2 concurrent full-context sessions, CUDA graphs on - native text+image+audio: the fleet's multimodal endpoint - measured: pp@67k 1293 t/s, pp@213k 608 t/s, tg@213k ~13.6 t/s, tg@85k ~17.6 t/s - host driver 610.57.04 (Ubuntu 26.04); sm_86 is correct, no arch override - planner/scoping: frank-small now selected by dankclaw/frank-small/ + env/serve/download trio; old unit retired to scripts/.AGENTTRASH/ - docs: new runbook frank-small-vllm-gemma4.md; AGENTS.md, LLM index, top docs index updated; old runbook moved to retired
81 lines
3.3 KiB
Bash
Executable File
81 lines
3.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Fetch the frank-small vLLM payloads on dankclaw (RTX 3090, since 2026-09-18):
|
|
#
|
|
# google/gemma-4-12B-it-qat-w4a16-ct
|
|
# compressed-tensors w4a16 (int4, group 32) QAT checkpoint — the model
|
|
# vLLM's gemma4_unified implementation loads natively. One 9.56 GiB
|
|
# safetensors plus tokenizer/config.
|
|
#
|
|
# google/gemma-4-12B-it-assistant
|
|
# the MTP draft/assistant (gemma4_unified_assistant, 4 layers) in
|
|
# bf16 — Google's original. 806 MiB. The w4a16-ct re-quant
|
|
# (kunhunjon) that was used first drafts at acceptance 1.00: vLLM's
|
|
# MTP drafter path does not apply the Marlin w4a16 kernel to the
|
|
# assistant's linears (no "Using MarlinLinearKernel" after the
|
|
# drafter load), so its packed int4 weights are misread. The bf16
|
|
# assistant accepts at 2.80 (measured 2026-09-19).
|
|
#
|
|
# Both repos are public and ungated; no token is needed. Every fetch is -f
|
|
# (an HTTP error is a failure, not a 153-byte "artifact"), resumable, and the
|
|
# two LFS payloads are sha256-verified against the Hub's declared oid — the
|
|
# discipline from scripts/download-frank-medium-gguf.sh.
|
|
#
|
|
# The serve script refuses to start when either payload is absent, so a
|
|
# half-finished run of this script reads as a missing file, not a runtime
|
|
# fault.
|
|
set -euo pipefail
|
|
|
|
MODELS_DIR="${MODELS_DIR:-/home/dank/models}"
|
|
LOG_DIR="$MODELS_DIR/logs"
|
|
mkdir -p "$LOG_DIR"
|
|
LOG="$LOG_DIR/$(basename "$0" .sh).log"
|
|
|
|
log() { echo "$(date -Is) $*" | tee -a "$LOG"; }
|
|
|
|
# fetch <repo> <dir> <file> [sha256] [size]
|
|
fetch() {
|
|
local repo="$1" dir="$2" file="$3" sha="${4:-}" size="${5:-}"
|
|
mkdir -p "$dir"
|
|
if [[ -n "$size" && -f "$dir/$file" ]] && [[ "$(stat -c%s "$dir/$file")" == "$size" ]]; then
|
|
log "skip $dir/$file (size $size matches)"
|
|
else
|
|
log "fetch $repo/$file"
|
|
curl -fsSL -C - "https://huggingface.co/$repo/resolve/main/$file" -o "$dir/$file"
|
|
log "fetched $dir/$file ($(stat -c%s "$dir/$file") bytes)"
|
|
fi
|
|
if [[ -n "$sha" ]]; then
|
|
local actual
|
|
actual="$(sha256sum "$dir/$file" | cut -d' ' -f1)"
|
|
if [[ "$actual" != "$sha" ]]; then
|
|
log "SHA256 MISMATCH $dir/$file: $actual != $sha"
|
|
return 1
|
|
fi
|
|
log "sha256 ok $dir/$file"
|
|
fi
|
|
}
|
|
|
|
MAIN_REPO=google/gemma-4-12B-it-qat-w4a16-ct
|
|
ASSISTANT_REPO=google/gemma-4-12B-it-assistant
|
|
MAIN_DIR="$MODELS_DIR/gemma4-12b-qat-w4a16-ct"
|
|
ASSISTANT_DIR="$MODELS_DIR/gemma4-12b-assistant-bf16"
|
|
|
|
# Hub-declared LFS oids (sha256), 2026-09-18.
|
|
fetch "$MAIN_REPO" "$MAIN_DIR" model.safetensors \
|
|
60b6e3989502969d8ae04185d72ecbbc7db63978d5af747a493d53895aa6bfa3 10264229896
|
|
fetch "$MAIN_REPO" "$MAIN_DIR" tokenizer.json \
|
|
cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f 32169626
|
|
for f in config.json generation_config.json chat_template.jinja \
|
|
processor_config.json tokenizer_config.json recipe.yaml; do
|
|
fetch "$MAIN_REPO" "$MAIN_DIR" "$f"
|
|
done
|
|
|
|
fetch "$ASSISTANT_REPO" "$ASSISTANT_DIR" model.safetensors \
|
|
3279c173daddd7186e79d652ad94022415736d3a1370625696c898429b06d6df 845719296
|
|
fetch "$ASSISTANT_REPO" "$ASSISTANT_DIR" tokenizer.json \
|
|
c001d9ada50af662c94f5ab17ec7e09f6438d1bec8246c47fee6510693d8de35 32169884
|
|
for f in config.json generation_config.json tokenizer_config.json; do
|
|
fetch "$ASSISTANT_REPO" "$ASSISTANT_DIR" "$f"
|
|
done
|
|
|
|
log "ALL FETCHED OK"
|