diff --git a/scripts/frank-primary-expert-activity.py b/scripts/frank-primary-expert-activity.py new file mode 100644 index 0000000..0c8c873 --- /dev/null +++ b/scripts/frank-primary-expert-activity.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Measure per-block / per-expert expert-routing activity on live frank-primary. + +Method: AMD IBS op sampling (data addresses, `perf record -e ibs_op// --data`) +on the llama-server process while concurrent generations of mixed content run. +Sampled load addresses are mapped through /proc//maps (mmap VMAs of the +6 GGUF parts) back to individual tensors, then aggregated per expert block +and per expert WITHIN each block (512 experts, row-major expert-major layout). + +Tensor extents are DERIVED from consecutive GGUF header offsets — no quant +type-size table (those are easy to get subtly wrong; a wrong per-element size +silently under-covers each tensor and dumps tail samples into a junk bucket). + +Fleet law this instrument established (2026-09-26, 75 s window, 4 mixed +streams, 2.7M expert-tensor samples): per-BLOCK read volume is FLAT +(0.93x-1.08x of mean across all 40 CPU-resident blocks) because MoE top-k +routing fires every block every token — there are no "most active layers" at +`-ot` granularity. The real skew is per-EXPERT (top 10% of (block,expert) +cells carry ~40% of samples), which tensor-level overrides cannot reach. +Any 8 GPU-resident blocks are therefore equivalent; COUNT is the only lever. + +Usage (needs passwordless sudo for perf only; model files must be readable): + fp-expert-activity.py [duration_s] # dump + sample + analyze + fp-expert-activity.py --analyze-only # re-analyze saved samples +Artifacts: /tmp/fp-tensors.tsv, /tmp/fp-ibs.perf, /tmp/fp-addrs.txt +""" +import bisect, json, os, re, struct, subprocess, sys, threading, time +import urllib.request + +MODEL_DIR = "/home/dank/models/Qwen3.8-Flash-Next-GGUF/Q8_0" +N_PARTS, N_BLOCKS, N_EXPERTS = 6, 48, 512 +TSV, PERF_OUT, ADDRS = "/tmp/fp-tensors.tsv", "/tmp/fp-ibs.perf", "/tmp/fp-addrs.txt" +PORT = 7770 + +# ---------------------------------------------------------------- GGUF header +class P: + def __init__(self, f): self.f, self.ver = f, 2 + def rd(self, n): + b = self.f.read(n) + assert len(b) == n, "eof" + return b + def u8(self): return self.rd(1)[0] + def u32(self): return struct.unpack("= 3 else self.u32() + return self.rd(n).decode("utf-8", "replace") + def val(self, t): + if t == 8: return self.s() + if t == 9: + et, n = self.u32(), self.u64() + for _ in range(n): self.val(et) + return None + if t in (0, 7): return self.u8() + if t == 1: return struct.unpack(" 0xffff000000000000: continue + vi = bisect.bisect_right(vmas, (a, 1 << 62, 0, 0)) - 1 + if vi < 0: n_out += 1; continue + start, end, pgoff, pi = vmas[vi] + if not start <= a < end: n_out += 1; continue + foff = pgoff + (a - start) + st, en, nm = tensors[pi] + ti = bisect.bisect_right(st, foff) - 1 + if ti < 0 or foff >= en[ti]: n_gap += 1; continue + name, lo, span = nm[ti], foff - st[ti], en[ti] - st[ti] + if "exps" in name: + n_exps += 1 + b = int(re.match(r"blk\.(\d+)\.", name).group(1)) + e = int(lo / (span / N_EXPERTS)) + block[b] = block.get(b, 0) + 1 + expert[(b, e)] = expert.get((b, e), 0) + 1 + elif "per_layer" in name: n_ple += 1 + else: n_named += 1 + print(f"\n[rig] attribution: exps={n_exps} ple={n_ple} named={n_named} " + f"gap={n_gap} outside={n_out}") + tot = sum(block.values()) + print(f"== BLOCK histogram ({tot} samples, {len(block)} CPU-resident blocks):") + for b, c in sorted(block.items(), key=lambda kv: -kv[1]): + print(f" blk{b:2d}: {c:7d} {100*c/tot:5.2f}%") + mean = tot / max(len(block), 1) + print(f"block spread: max={max(block.values())/mean:.3f}x mean, min={min(block.values())/mean:.3f}x mean") + ev = sorted(expert.values(), reverse=True); et = sum(ev) + print(f"== EXPERT skew: {len(ev)} distinct (block,expert) cells hit of {N_BLOCKS*N_EXPERTS}") + print(f" top-half of cells carry {100*sum(ev[:len(ev)//2])/et:.1f}%; " + f"top-10% {100*sum(ev[:len(ev)//10])/et:.1f}%; hottest cell {100*ev[0]/et:.3f}%") + +if __name__ == "__main__": + main() diff --git a/serverAppConfigs-docs/LLM/frank-primary-llamacpp-qwen38-flash-next-q8.md b/serverAppConfigs-docs/LLM/frank-primary-llamacpp-qwen38-flash-next-q8.md index 50d757a..730aab7 100644 --- a/serverAppConfigs-docs/LLM/frank-primary-llamacpp-qwen38-flash-next-q8.md +++ b/serverAppConfigs-docs/LLM/frank-primary-llamacpp-qwen38-flash-next-q8.md @@ -163,7 +163,8 @@ miss costs ~0.5 GiB on ROCm1). At 4×262,144 the pool is ~23 GiB across both cards: main 9,408 MiB + indexer 1,536 MiB per card, draft 2,048 MiB on ROCm1. -Hot expert blocks stay GPU-resident. `-ot` granularity is the whole +Expert blocks stay GPU-resident — and the measurement below shows it +cannot matter *which* ones. `-ot` granularity is the whole per-block expert tensor (512 experts share one tensor), so "critical experts" = blocks co-located on the card that holds their layers: blocks 0-4 on ROCm0, 45-47 on ROCm1 (measured 2.42/2.49 GiB per @@ -189,6 +190,28 @@ delivered, prompt cache reusing 42/64 tokens across requests. All three Prometheus fleet targets up; the 5/3 deploy re-verified chat, vision and the target on the same battery class. +**Which blocks, measured (same day):** asked whether the `-ot` regex +could target the *most active* blocks, I instrumented the live server +instead of guessing — `scripts/frank-primary-expert-activity.py` +samples llama-server with AMD IBS (`perf record -e ibs_op// --data`) +while 4 mixed concurrent generations run, maps each sampled load +address through `/proc//maps` back to its backing tensor (extents +derived from consecutive GGUF header offsets — never from a quant +type-size table, whose errors silently mis-bucket sample tails), and +aggregates per block and per expert. Verdict, 75 s window / 2.7M +expert-tensor samples: **per-block read volume is FLAT, 0.93×–1.08× of +mean across all 40 CPU-resident blocks** (0-4 and 45-47 absent — +GPU-resident). MoE top-k routing fires every block on every token, so +"most active layers" do not exist at `-ot` granularity: the regex can +express any 8-block set, and any set is equivalent to any other — the +lever is COUNT, already at the 5/3 VRAM ceiling. The real skew is +per-EXPERT (top 10% of the 48×512 (block,expert) cells carry 40.5% of +samples; hottest cell 0.1%), below the reach of a tensor-level +override. Not a benchmark: it instruments production serving in place +at negligible IBS overhead; re-measure any future checkpoint or +traffic mix with `frank-primary-expert-activity.py ` (or +`--analyze-only` against the saved sample file in `/tmp`). + ## Rollback The lucebox DS4.1 material is fully retained on the host: image