diff --git a/scripts/probe-frank-archon-kv.py b/scripts/probe-frank-archon-kv.py index 0351309..eac6a76 100755 --- a/scripts/probe-frank-archon-kv.py +++ b/scripts/probe-frank-archon-kv.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""probe-frank-archon-kv.py — verify the 2 x 200,000 KV contract on a LIVE +"""probe-frank-archon-kv.py — verify the seqs x max-model-len KV contract on a frank-archon (radiance) service. Run it ON dankserver: it talks to 127.0.0.1:7770 and builds its prompts from the repo checkout there. No scratch container, no mask: the service under test is the service. @@ -10,10 +10,11 @@ Three probes, in increasing cost: A decode at depth — one ~51k-token chat completion, 512 tokens, greedy. Reports the engine's own `timings` (prefill, decode, draft acceptance). The bench law applies: >=128 generated tokens, greedy, real text. - B the capacity contract — TWO concurrent chats of ~197k prompt tokens - (~394k tokens resident) each carrying a unique marker at its head. Both - must answer their own marker. This is what a 2 x 200,000 endpoint owes: - two full-height conversations resident at once, exact retrieval in both. + B the capacity contract — as many CONCURRENT chats of ~197k prompt tokens as + the deployed `FRANK_ARCHON_MAX_SEQS` (read from the repo env file on this + host), each carrying a unique marker at its head. All must answer their own + marker. This is what the endpoint owes: N full-height conversations + resident at once, exact retrieval in every one of them. C prefix-cache restore — probe A's cold prompt again, so the disk tier has to give the blocks back (cache_n > 0, TTFT collapses). @@ -114,23 +115,51 @@ def show(tag, j, wall, expect=None): print(line) +def deployed_shape(): + """The contract is the deployed env, not a constant: slot count and window + come from the checkout on this host, so the probe cannot drift from the + service it is testing (the 2026-10-13 shape is 4 x 200,000; before that + 2 x 200,000, and the fp8 era 3 x 200,000).""" + shape = {} + with open(f"{REPO}/scripts/frank-archon.env") as fh: + for line in fh: + line = line.strip() + if line.startswith("FRANK_ARCHON_") and "=" in line: + key, _, val = line.partition("=") + shape[key] = val.strip().strip('"') + return (int(shape["FRANK_ARCHON_MAX_SEQS"]), + int(shape["FRANK_ARCHON_MAX_MODEL_LEN"]), + shape["FRANK_ARCHON_KV_DTYPE"]) + + def main(): + seqs, maxlen, dtype = deployed_shape() + print(f"shape: {seqs} x {maxlen:,} tokens, kv {dtype}") print("health:", json.dumps(get("/health"))) print("models:", [m.get("id") for m in get("/v1/models").get("data", [])]) deep = corpus(160_000) wide = corpus(1_100_000) - facts = {"B1": "cobalt-harbor-13", "B2": "quartz-meadow-42"} - bodies = {s: chat(wide, 646_000, s, f, 24) for s, f in facts.items()} + # 3.28 chars/token is the measured ratio of this corpus through the + # froggeric template (646,000 chars -> 196,919 tokens at the 2026-10-11 + # deploy); 98.5 % of the window leaves the response room under the ceiling. + chars = int(0.985 * maxlen * 3.28) + # Each marker's FIRST hyphen-separated word is unique, because show() + # scores the hit on that word — a shared prefix would score every slot HIT + # on any answer that says the word. + words = ["cobalt", "quartz", "basalt", "ember", "willow", "cobweb", "flint"] + facts = {f"B{i+1}": f"{words[i % len(words)]}-flag-{(i + 1) * 137}" + for i in range(seqs)} + bodies = {s: chat(wide, chars, s, f, 24) for s, f in facts.items()} print(f"[B] {len(bodies['B1']['messages'][0]['content']):,} chars per " - f"request, x2 concurrent") + f"request, x{seqs} concurrent") - # A + C first (cheap), B last (the expensive pair). + # A + C first (cheap), B last (the expensive set). for tag, salt in (("A-cold", "A1-9137"), ): j, wall, _ = post(chat(deep, 140_000, salt, "zephyr-lantern-7", 512)) show(tag, j, wall, "zephyr-lantern-7") t0 = time.time() - with concurrent.futures.ThreadPoolExecutor(max_workers=2) as ex: + with concurrent.futures.ThreadPoolExecutor(max_workers=seqs) as ex: futs = {s: ex.submit(post, b) for s, b in bodies.items()} for s, fut in futs.items(): try: diff --git a/scripts/serve-frank-archon.sh b/scripts/serve-frank-archon.sh index a96c1be..35e38ea 100755 --- a/scripts/serve-frank-archon.sh +++ b/scripts/serve-frank-archon.sh @@ -62,7 +62,7 @@ set +a : "${FRANK_ARCHON_PREFIX_DISK_MIB:?FRANK_ARCHON_PREFIX_DISK_MIB must be set}" : "${FRANK_ARCHON_NGRAM_PLACEMENT:?FRANK_ARCHON_NGRAM_PLACEMENT must be set}" : "${FRANK_ARCHON_TTM_MIN_MIB:?FRANK_ARCHON_TTM_MIN_MIB must be set}" -:"${FRANK_ARCHON_TTM_TARGET_MIB:?FRANK_ARCHON_TTM_TARGET_MIB must be set}" +: "${FRANK_ARCHON_TTM_TARGET_MIB:?FRANK_ARCHON_TTM_TARGET_MIB must be set}" : "${FRANK_ARCHON_RAD_MIN_MIB:?FRANK_ARCHON_RAD_MIN_MIB must be set}" # The image gate is the digest-pinned ref itself: a name@sha256 ref exists