frank-primary: cutover state to Qwen3.8-Flash-Next Q8_0 (6 shards, PLE pinned, MTP4, sharp template)
- env: STATE_FILE -> r9v-state/q8xl; add dev-overlay adapter mount (quantized inject shard dequant, fork 363d606), FULL_MUTABLE_CACHE=0 (80-slot tiered vs the overlay's uncensored 160-slot injection), and the qwen-sharp-v22.4.0.jinja chat template via the launcher's new R9V_CHAT_TEMPLATE_FILE/KWARGS passthrough (fork 30d4a32) - serve/deploy: shard gates extended to SHARD4/5/6 (unset-safe, the uncensored rollback state carries none) - seams: STATE_FILE/LLM_PRIMARY_STATE -> q8xl
This commit is contained in:
Frank
@@ -1185,10 +1185,12 @@ deploy_llm_small() {
|
||||
|
||||
# deploy_llm_primary — frank-primary is the R9V radiance-vLLM fork (docker-
|
||||
# pinned prebuilt, gfx1201, driven through the docker->podman shim) serving
|
||||
# Qwen3.8-Flash-Next-Uncensored IQ4_XS with the MTP4 drafter on the two
|
||||
# R9700s as a ROOT unit whose serve script execs the fork's launcher through
|
||||
# rootless podman as dank. Same port and alias as the llama.cpp endpoint it
|
||||
# replaced (2026-09-27 cutover); the harness models.yml id is unchanged.
|
||||
# the unsloth Qwen3.8-Flash-Next Q8_0 payload (6 GGUF shards) with the MTP4
|
||||
# drafter on the two R9700s as a ROOT unit whose serve script execs the
|
||||
# fork's launcher through rootless podman as dank. Same port and alias as
|
||||
# the endpoints it replaced (the 2026-09-27 cutover from uncensored
|
||||
# IQ4_XS); the harness models.yml id is unchanged. The uncensored state dir
|
||||
# (/home/dank/r9v-state/uncensored) stays on disk as the rollback target.
|
||||
#
|
||||
# Gates: the on-host R9V setup state must exist and be ready (it carries the
|
||||
# pinned image digest and every payload path), the pinned runtime image must
|
||||
@@ -1196,7 +1198,7 @@ deploy_llm_small() {
|
||||
# PLE payload must match its pinned byte count, and every GGUF shard + the
|
||||
# mmproj must exist (per-file sha256 verification was setup's job at fetch
|
||||
# time, never re-hashed here).
|
||||
LLM_PRIMARY_STATE=/home/dank/r9v-state/uncensored/setup.json
|
||||
LLM_PRIMARY_STATE=/home/dank/r9v-state/q8xl/setup.json
|
||||
LLM_PRIMARY_CONTAINER=frank-primary-r9v
|
||||
|
||||
deploy_llm_primary() {
|
||||
@@ -1216,7 +1218,9 @@ config = state.get("config") or {}
|
||||
if state.get("ready") is not True:
|
||||
sys.exit("R9V setup state is not ready")
|
||||
for key in ("R9V_IMAGE", "R9V_MODEL_DIR", "R9V_PLE_PATH", "R9V_PLE_EXPECTED_BYTES",
|
||||
"R9V_TARGET_REL", "R9V_TARGET_SHARD2_REL", "R9V_TARGET_SHARD3_REL", "R9V_MMPROJ_REL"):
|
||||
"R9V_TARGET_REL", "R9V_TARGET_SHARD2_REL", "R9V_TARGET_SHARD3_REL",
|
||||
"R9V_TARGET_SHARD4_REL", "R9V_TARGET_SHARD5_REL", "R9V_TARGET_SHARD6_REL",
|
||||
"R9V_MMPROJ_REL"):
|
||||
print(f"{key}={shlex.quote(config.get(key, ''))}")
|
||||
PY
|
||||
)" || return 1
|
||||
@@ -1239,7 +1243,9 @@ PY
|
||||
fi
|
||||
fi
|
||||
local rel
|
||||
for rel in "$R9V_TARGET_REL" "$R9V_TARGET_SHARD2_REL" "$R9V_TARGET_SHARD3_REL" "$R9V_MMPROJ_REL"; do
|
||||
for rel in "$R9V_TARGET_REL" "${R9V_TARGET_SHARD2_REL:-}" "${R9V_TARGET_SHARD3_REL:-}" \
|
||||
"${R9V_TARGET_SHARD4_REL:-}" "${R9V_TARGET_SHARD5_REL:-}" "${R9V_TARGET_SHARD6_REL:-}" \
|
||||
"$R9V_MMPROJ_REL"; do
|
||||
[[ -z "$rel" ]] && continue
|
||||
if [[ ! -f "$R9V_MODEL_DIR/$rel" ]]; then
|
||||
echo "ERROR: payload file missing: $R9V_MODEL_DIR/$rel" >&2
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
# frank-primary.env — R9V radiance-vLLM production overrides (dankserver :7770).
|
||||
#
|
||||
# The full tuned engine shape (104 R9V_* keys: tiered-expert placement,
|
||||
# CPU offload budgets 146.25/78.75 GiB per rank, CED on, MTP4, TP=2,
|
||||
# qwen3_coder tool parser, 196608 model len) lives on-host in the R9V setup
|
||||
# state — /home/dank/r9v-state/uncensored/setup.json — written by the
|
||||
# profile's setup run (PASS=28/FAIL=0, all 20 artifacts sha256-verified).
|
||||
# serve-frank-primary.sh layers that state first and THIS FILE second, so the
|
||||
# keys below win. Never copy tuned values from the state into this file —
|
||||
# the state is the single source of truth for engine shape.
|
||||
# The full tuned engine shape (tiered-expert placement, CPU offload budgets
|
||||
# per rank, MTP4 drafter, TP=2, qwen3_coder tool parser, 131072 model len,
|
||||
# 80-slot tiered expert cache, PLE pinned residency 54.4 GB) lives on-host
|
||||
# in the R9V setup state — /home/dank/r9v-state/q8xl/setup.json — written by
|
||||
# the q8 profile's setup run (ready: true; doctor PASS, artifacts
|
||||
# sha256-verified at fetch time). serve-frank-primary.sh layers that state
|
||||
# first and THIS FILE second, so the keys below win. Never copy tuned values
|
||||
# from the state into this file — the state is the single source of truth
|
||||
# for engine shape.
|
||||
R9V_HOME=/home/dank/GitPackages/radiance-vllm-qwen4exp
|
||||
R9V_STATE_FILE=/home/dank/r9v-state/uncensored/setup.json
|
||||
R9V_STATE_FILE=/home/dank/r9v-state/q8xl/setup.json
|
||||
# Fleet identity: LAN bind, the single frank-primary port, the harness's
|
||||
# model id, our own container name (the launcher default r9v-qwen38-flash-next
|
||||
# is the scratch identity).
|
||||
@@ -18,3 +19,23 @@ R9V_HOST_PORT=7770
|
||||
R9V_SERVED_MODEL_NAME=frank-primary
|
||||
R9V_CONTAINER_NAME=frank-primary-r9v
|
||||
R9V_EXPECTED_GPU_BDFS=0000:03:00.0,0000:07:00.0
|
||||
# The Q8_0 payload quantizes BOTH fused hyper-connection shards (the IQ4_XS
|
||||
# package ships inject as F32), so the shard-uniformity check rejects the
|
||||
# fused projection at load. This dev-overlay adapter dequantizes a quantized
|
||||
# inject shard in-stream (fork commit 363d606): opt-in mount, uncensored
|
||||
# payloads pass through unchanged.
|
||||
R9V_DEV_QWEN4_EXP_ADAPTER_PY=/home/dank/GitPackages/radiance-vllm-qwen4exp/runtimes/dev-overrides/qwen4_exp.py
|
||||
# The runtime overlay's env.always injects R9V_FULL_MUTABLE_CACHE=1 — the
|
||||
# qualified 160-slot uncensored IQ4_XS cache geometry — which the q8
|
||||
# profile's 80-slot tiered cache fails with a baseline mismatch. This flips
|
||||
# that injection off for this payload (launch.sh passes it after the overlay
|
||||
# envs; docker last-wins).
|
||||
R9V_FULL_MUTABLE_CACHE=0
|
||||
# The Qwen-Sharp template (the frank-large fleet law): folds reasoning
|
||||
# efforts (high/xhigh/max -> xhigh, minimal/low -> low) and defaults
|
||||
# preserve_thinking true. --chat-template wins over the tokenizer dir's
|
||||
# metadata/chat_template.jinja; request-level kwargs still win per request.
|
||||
# Braces in values get the same single-quote law as parentheses (source
|
||||
# parses assignment values).
|
||||
R9V_CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/qwen-sharp-v22.4.0.jinja
|
||||
R9V_CHAT_TEMPLATE_KWARGS='{"preserve_thinking":true}'
|
||||
@@ -1,7 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
# serve-frank-primary.sh — frank-primary endpoint (R9V radiance-vLLM fork,
|
||||
# Qwen3.8-Flash-Next-Uncensored IQ4_XS + MTP4 drafter, 2x R9700, dankserver
|
||||
# :7770).
|
||||
# unsloth Qwen3.8-Flash-Next Q8_0 + MTP4 drafter, 2x R9700, dankserver
|
||||
# :7770; 2026-09-27 cutover from the uncensored IQ4_XS payload, whose state
|
||||
# dir stays on disk as the rollback target).
|
||||
#
|
||||
# The engine is the upstream R9V launcher (scripts/launch.sh in the fork
|
||||
# checkout), NOT a hand-frozen podman argv: launch.sh owns the resolved
|
||||
@@ -20,10 +21,10 @@
|
||||
#
|
||||
# The root unit (scripts/frank-primary.service, User=dank,
|
||||
# LimitMEMLOCK=infinity) grants the pin budget for the tiered-expert UVA
|
||||
# host copy (~43 GiB pinned) and the 16 GiB PLE pinned reserve. A plain
|
||||
# shell session also works (proven 2026-09-27: podman rootless inherits
|
||||
# usable limits here), but the unit is the fleet-parity boot-persistent
|
||||
# shape.
|
||||
# host copy and the PLE pinned residency (54.4 GB on the q8 payload). A
|
||||
# plain shell session also works (proven 2026-09-27: podman rootless
|
||||
# inherits usable limits here), but the unit is the fleet-parity
|
||||
# boot-persistent shape.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -66,7 +67,7 @@ export PATH="$SHIM_DIR:$PATH"
|
||||
STATE_FILE="$R9V_STATE_FILE"
|
||||
if [ ! -f "$STATE_FILE" ]; then
|
||||
echo "serve-frank-primary: R9V setup state missing: $STATE_FILE" >&2
|
||||
echo " run the profile setup first (r9v setup qwen38-mtp4-uncensored ...)" >&2
|
||||
echo " run the profile setup first (tools/setup_profile.py setup --state-dir ...)" >&2
|
||||
exit 1
|
||||
fi
|
||||
STATE_EXPORTS="$(python3 - "$STATE_FILE" <<'PY'
|
||||
@@ -92,12 +93,13 @@ eval "$STATE_EXPORTS"
|
||||
# digest but preserves the manifest digest in RepoDigests — the shim's ID law).
|
||||
if ! podman images --format '{{.RepoDigests}}' | grep -qF "${R9V_IMAGE#sha256:}"; then
|
||||
echo "serve-frank-primary: pinned R9V image absent: $R9V_IMAGE" >&2
|
||||
echo " load the bundle: ./r9v setup qwen38-mtp4-uncensored --state-dir ..." >&2
|
||||
echo " load the pinned image bundle into podman rootless storage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# PLE payload at its pinned byte count (stat -c %s, never wc -c), plus the
|
||||
# three target GGUF shards and the mmproj the launcher mounts.
|
||||
# six target GGUF shards (mradermacher-style split parts) and the mmproj
|
||||
# the launcher mounts.
|
||||
PLE_BYTES="${R9V_PLE_EXPECTED_BYTES:-}"
|
||||
if [ -n "$PLE_BYTES" ]; then
|
||||
ACTUAL="$(stat -c %s "$R9V_PLE_PATH" 2>/dev/null || echo 0)"
|
||||
@@ -112,7 +114,9 @@ else
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
for rel in "$R9V_TARGET_REL" "$R9V_TARGET_SHARD2_REL" "$R9V_TARGET_SHARD3_REL" "$R9V_MMPROJ_REL"; do
|
||||
for rel in "$R9V_TARGET_REL" "${R9V_TARGET_SHARD2_REL:-}" "${R9V_TARGET_SHARD3_REL:-}" \
|
||||
"${R9V_TARGET_SHARD4_REL:-}" "${R9V_TARGET_SHARD5_REL:-}" "${R9V_TARGET_SHARD6_REL:-}" \
|
||||
"$R9V_MMPROJ_REL"; do
|
||||
if [ -z "$rel" ]; then continue; fi
|
||||
if [ ! -f "$R9V_MODEL_DIR/$rel" ]; then
|
||||
echo "serve-frank-primary: payload file missing: $R9V_MODEL_DIR/$rel" >&2
|
||||
|
||||
@@ -65,11 +65,11 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
}
|
||||
|
||||
@test "the tuned engine shape comes from the on-host setup state, not the repo" {
|
||||
# The profile setup qualified a 104-key shape (tiered-expert placement,
|
||||
# per-rank offload budgets, CED, MTP4, parser wiring). Copying it into
|
||||
# The profile setup qualified the tuned shape (tiered-expert placement,
|
||||
# per-rank offload budgets, MTP4, parser wiring). Copying it into
|
||||
# the repo would drift; the serve script layers it and skips keys the
|
||||
# env file defines, so production overrides always win.
|
||||
grep -q 'R9V_STATE_FILE=/home/dank/r9v-state/uncensored/setup.json' "$ENV_FILE"
|
||||
grep -q 'R9V_STATE_FILE=/home/dank/r9v-state/q8xl/setup.json' "$ENV_FILE"
|
||||
grep -q 'state must carry R9V_MODEL_DIR' "$SERVE"
|
||||
grep -q 'is not ready' "$SERVE"
|
||||
grep -q 'key not in env_keys' "$SERVE"
|
||||
@@ -122,7 +122,7 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
# the PLE payload is off its pinned byte count, or when any GGUF shard
|
||||
# or the mmproj is missing — then clears stale containers, waits out
|
||||
# the port, relinks/enables/restarts the root unit and health-gates it.
|
||||
grep -q 'LLM_PRIMARY_STATE=/home/dank/r9v-state/uncensored/setup.json' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_PRIMARY_STATE=/home/dank/r9v-state/q8xl/setup.json' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_PRIMARY_CONTAINER=frank-primary-r9v' "$REMOTE_DEPLOY"
|
||||
grep -q 'is not ready' "$REMOTE_DEPLOY"
|
||||
grep -q "podman images --format '{{.RepoDigests}}'" "$REMOTE_DEPLOY"
|
||||
|
||||
Reference in New Issue
Block a user