diff --git a/scripts/bench-probe.py b/scripts/bench-probe.py index 55b069b..234ed59 100755 --- a/scripts/bench-probe.py +++ b/scripts/bench-probe.py @@ -7,7 +7,7 @@ import json, sys, urllib.request TARGET = 43600 # tokens BYTES_PER_TOK = 3.8 -URL = "http://localhost:7770/completion" +URL = "http://localhost:7770/v1/completions" def build_prompt(): want = int(TARGET * BYTES_PER_TOK) @@ -26,8 +26,8 @@ def build_prompt(): def probe(salt): prompt = f"[probe {salt}]\n" + build_prompt() body = json.dumps({ + "model": "frank-primary", "prompt": prompt, "temperature": 0.0, "max_tokens": 512, - "stream": False, }).encode() req = urllib.request.Request(URL, data=body, headers={"Content-Type": "application/json"}) diff --git a/scripts/deep-probe.py b/scripts/deep-probe.py new file mode 100755 index 0000000..2653e8a --- /dev/null +++ b/scripts/deep-probe.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""deep-probe.py — one chat completion at ~43.6k prompt tokens, temp 0, 512 +generated, streamed; decode = completion_tokens / (total - ttft).""" +import json, time, urllib.request + +TARGET_BYTES = int(43600 * 3.8) + +def build(): + parts, n = [], 0 + with open("/home/dank/corpora/out/calib.jsonl") as f: + for line in f: + p = json.loads(line)["prompt"] + parts.append(p) + n += len(p.encode()) + if n >= TARGET_BYTES: + break + return "".join(parts) + +def main(salt): + body = json.dumps({ + "model": "frank-primary", + "messages": [{"role": "user", "content": f"[probe {salt}] Continue and summarize this material in English, 400 words.\n\n" + build()}], + "temperature": 0.0, "max_tokens": 512, + "stream": True, "stream_options": {"include_usage": True}, + }).encode() + req = urllib.request.Request("http://localhost:7770/v1/chat/completions", + data=body, headers={"Content-Type": "application/json"}) + t0 = time.time(); ttft = None; ctok = 0; usage = {} + with urllib.request.urlopen(req, timeout=2400) as r: + for raw in r: + line = raw.decode().strip() + if not line.startswith("data:"): + continue + d = line[5:].strip() + if d == "[DONE]": + break + j = json.loads(d) + if j.get("usage"): + usage = j["usage"] + for ch in j.get("choices", []): + delta = (ch.get("delta") or {}) + piece = delta.get("content") or delta.get("reasoning_content") or "" + if piece and ttft is None: + ttft = time.time() - t0 + ctok += 1 + total = time.time() - t0 + gen = usage.get("completion_tokens") or ctok + dec = gen / (total - (ttft or 0)) if total > (ttft or 0) else 0 + print(json.dumps({"salt": salt, "prompt_tokens": usage.get("prompt_tokens"), + "completion_tokens": gen, "ttft_s": round(ttft or 0, 2), + "total_s": round(total, 1), "decode_tps": round(dec, 2)})) + +if __name__ == "__main__": + import sys + main(sys.argv[1] if len(sys.argv) > 1 else "x") diff --git a/scripts/frank-primary.env b/scripts/frank-primary.env index 4b3d311..8088eef 100644 --- a/scripts/frank-primary.env +++ b/scripts/frank-primary.env @@ -73,7 +73,7 @@ FRANK_PRIMARY_IMAGE=stilldeadcode/radiance@sha256:4c3f484564bb2872784e46b0a59cbd # .rad files; the raw HF download and its staging dir live beside it and # are never mounted. FRANK_PRIMARY_MODEL_DIR=/home/dank/models/rad -FRANK_PRIMARY_MODEL_FILE=qwen4exp-fp8-native.rad +FRANK_PRIMARY_MODEL_FILE=qwen4exp-swift15-w4dn8-i8-hc8m.rad FRANK_PRIMARY_STATE_DIR=/home/dank/radiance-state # The state dir's tune/ subdirectory holds rad-tune's kernel-selection # cache (the engine's selector reads $RADIANCE_HOME/tune/.tune @@ -161,4 +161,4 @@ FRANK_PRIMARY_EXTRA_ARGS= # container measured 177,109 MiB (1,857,127,997,440 B = 173.0 GiB; the # plan-only estimate 172.76 GiB covers the data planes but not the metadata # tail) - floor sits just under the payload. -FRANK_PRIMARY_RAD_MIN_MIB=177000 +FRANK_PRIMARY_RAD_MIN_MIB=110000