diff --git a/scripts/frank-default-large.service b/scripts/frank-default-large.service index 61d20fd..38034b1 100644 --- a/scripts/frank-default-large.service +++ b/scripts/frank-default-large.service @@ -1,5 +1,5 @@ [Unit] -Description=ROCmFPX llama.cpp Qwen3.8-Flash-Next ROCmFP4 STRIX_LEAN 56B MoE (dankclaw, frank-default-large) +Description=ROCmFPX (kingjones30 fork) llama.cpp Qwen3.8-Flash-Next ROCmFP4 STRIX 177B MoE (dankclaw, frank-default-large) After=network-online.target Wants=network-online.target diff --git a/scripts/frank-default-small.env b/scripts/frank-default-small.env index 46b96de..e0dc179 100644 --- a/scripts/frank-default-small.env +++ b/scripts/frank-default-small.env @@ -1,5 +1,5 @@ -# Tunables for frank-default-small (Qwen3.6-35B-A3B-FP8, -# qwen3_5_moe hybrid MoE, Dankserver, RTX 3090). +# Tunables for frank-default-small (batsclamp/Huihui-Qwen3.6-35B-A3B- +# abliterated-FP8, qwen3_5_moe hybrid MoE, Dankserver, RTX 3090). # # Runtime: FreeToken v0.1.2 (edge-native MoE serving engine). # Installed via uv into /home/dank/freetoken/.venv (CPython 3.12). @@ -8,13 +8,18 @@ # layers page KV (~20 KiB/token); the 30 GDN layers use constant-size # per-sequence state (mamba slot pool). # -# Model: Qwen/Qwen3.6-35B-A3B-FP8 (HuggingFace safetensors, modelopt -# block-fp8 quant, 128x128 weight_block_size, 35 GB on disk). -# Architecture: qwen3_5_moe — 40 layers (10 attention + 30 GDN), -# 256 experts / 8 used, 2 KV heads × 256 dim, full_attention_interval=4. -# -# Replaced the llama.cpp Tiel-Coder-35B-A3B (UD-Q6_K_XL) deployment on -# 2026-09-02. The old serve script and env are in .AGENTTRASH/. +# Model: batsclamp/Huihui-Qwen3.6-35B-A3B-abliterated-FP8 (HuggingFace +# safetensors). Abliterated (refusal-removed) Huihui finetune of +# Qwen3.6-35B-A3B, quantized to the SAME native format the previous +# official checkpoint used: modelopt block-fp8 e4m3, dynamic +# activations, 128x128 weight_block_size, bf16 weight_scale_inv; +# lm_head/embeds/norms/router gates/visual/MTP kept BF16. Un-fused +# per-expert layout matches the official Qwen FP8, so the FreeToken +# loader path is identical; ~35 GB on disk. Ships the fixed Qwen3.6 +# chat template (froggeric/Qwen-Fixed-Chat-Templates) — FreeToken +# reads it from the checkpoint; no template override needed. +# Replaced Qwen/Qwen3.6-35B-A3B-FP8 on 2026-08-30; pool numbers +# unchanged — same arch, same weight mass. # # Pool sizing: GPU budget (~17.6 GiB after weights + fixed caches) is # split MoE-first. MOE_CACHE_SIZE pins resident expert slots; NUM_TOKENS @@ -27,8 +32,8 @@ # The ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP # incident). This box is NVIDIA/CUDA, but the rule keeps the env portable. -# Model directory (safetensors checkpoint, 35 GB). -MODEL_DIR=/home/dank/models/qwen36-35b-a3b-fp8 +# Model directory (safetensors checkpoint, ~35 GB, block-fp8). +MODEL_DIR=/home/dank/models/huihui-qwen36-35b-a3b-abliterated-fp8 # FreeToken venv (uv-managed CPython 3.12, ft CLI, flashinfer, torch+cu130). FT_VENV=/home/dank/freetoken/.venv diff --git a/scripts/frank-default-small.service b/scripts/frank-default-small.service index 0a06079..fe4f791 100644 --- a/scripts/frank-default-small.service +++ b/scripts/frank-default-small.service @@ -1,5 +1,5 @@ [Unit] -Description=FreeToken Qwen3.6-35B-A3B-FP8 qwen3_5_moe hybrid MoE (dankserver, frank-default-small) +Description=FreeToken Huihui-Qwen3.6-35B-A3B-abliterated-FP8 qwen3_5_moe hybrid MoE (dankserver, frank-default-small) After=network-online.target Wants=network-online.target diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 745066c..fd1b396 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Serve Qwen/Qwen3.6-35B-A3B-FP8 as frank-default-small via FreeToken -# on Dankserver (RTX 3090, 60 GB RAM). +# Serve batsclamp/Huihui-Qwen3.6-35B-A3B-abliterated-FP8 as +# frank-default-small via FreeToken on Dankserver (RTX 3090, 60 GB RAM). # # Runtime: FreeToken v0.1.2 (edge-native MoE serving engine). # - Experts live in host RAM; an LRU cache of expert slots is resident @@ -10,10 +10,14 @@ # hold constant-size per-sequence state (mamba slots). # - OpenAI-compatible API (/v1/chat/completions, /v1/models). # -# Model: Qwen/Qwen3.6-35B-A3B-FP8 (safetensors, modelopt block-fp8, -# 128x128 weight_block_size). Architecture: qwen3_5_moe — 40 layers, -# full_attention_interval=4 (10 attention + 30 GDN), 256 experts / 8 used, -# 2 KV heads × 256 dim. 35 GB on disk. +# Model: batsclamp/Huihui-Qwen3.6-35B-A3B-abliterated-FP8 — Huihui +# abliterated finetune of Qwen3.6-35B-A3B in the same native modelopt +# block-fp8 format (e4m3, 128x128 weight_block_size, un-fused per-expert +# layout) as the official FP8 checkpoint this service ran before; the +# FreeToken loader path is unchanged. 40 layers (10 attention + 30 +# GDN), 256 experts / 8 used, 2 KV heads × 256 dim. ~35 GB on disk. +# The checkpoint ships the fixed Qwen3.6 chat template; FreeToken reads +# it from the model directory. # # Pool sizing: the GPU budget is split MoE-first between expert residency # and KV pages. --moe-cache-size pins the number of resident expert @@ -35,7 +39,7 @@ if [[ -f "$ENV_FILE" ]]; then fi # --- defaults (used when the env file does not set them) ----------------- -: "${MODEL_DIR:=/home/dank/models/qwen36-35b-a3b-fp8}" +: "${MODEL_DIR:=/home/dank/models/huihui-qwen36-35b-a3b-abliterated-fp8}" : "${ALIAS:=frank-default-small}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7770}"