inference: frank-large returns to halogen, frank-small takes the 3090
Dankclaw's endpoints are re-split. frank-large is the halogen-flash-server quadlet again -- the 2026-09-13 unit file was never touched by the two-day ik_llama.cpp interlude, so the runtime was still in the repo and its image is still in local storage -- retuned to the shape asked for: HALOGEN_CTX 200192 with a 400,384-position pool and two slots, exactly two 200k conversations. frank-small is new: mainline llama.cpp on the RTX 3090 serving ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF at IQ3_S, using the -mtp build (it carries blk.64.nextn.*, which is what lets --spec-type draft-mtp run on the main model's own weights instead of a second 12 GiB copy) plus the BF16 vision projector, as one 131,072-token slot with f16 KV on port 7771. The host is shared now, and that is what the halogen retune pays for: HALOGEN_MAX_TOK 24576 (arena 20.6 -> 16.4 GiB, ~4-5% prefill) and the vision tower dropped (1.97 GiB) hand ~6 GiB back to the host, and the reserve rises to 20 to keep the fit check honest. Vision moved wholesale to frank-small. Retired to scripts/.AGENTTRASH/: the ik trio (env, service, serve script), the Qwen-Sharp template only that runtime read, and four benchmark harnesses bound to retired runtimes. Because retiring a file makes any test that greps it pass by absence, the seams assert the archive exists AND the live path does not.
This commit is contained in:
Frank
+229
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Benchmark decode throughput across speculative decoding configs.
|
||||
|
||||
Sweeps 4 configs: none, ngram-mod, draft-mtp, draft-mtp+ngram-mod.
|
||||
Each config generates 16k tokens from a minimal prompt at temp 0.
|
||||
|
||||
Runs ON the target host (Dankclaw). Service management uses local sudo.
|
||||
Usage:
|
||||
python3 bench-spec-decode.py [--max-tokens N] [--host H] [--port P]
|
||||
"""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
SERVICE = "frank-large"
|
||||
ENV_FILE = "/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ALIAS = "frank-large"
|
||||
DEFAULT_HOST = "localhost"
|
||||
DEFAULT_PORT = 7770
|
||||
DEFAULT_MAX_TOKENS = 16384
|
||||
HEALTH_TIMEOUT = 1200 # 20 min for cold load
|
||||
HEALTH_INTERVAL = 5
|
||||
|
||||
PROMPT = (
|
||||
"Write a comprehensive, detailed essay about the complete history of "
|
||||
"computing, from the earliest mechanical calculators through modern AI. "
|
||||
"Cover every major era in detail, including key figures, machines, and "
|
||||
"breakthroughs. Be as thorough as possible."
|
||||
)
|
||||
|
||||
CONFIGS = [
|
||||
("none", "none"),
|
||||
("ngram-mod", "ngram-mod"),
|
||||
("draft-mtp", "draft-mtp"),
|
||||
("draft-mtp+ngram-mod", "draft-mtp,ngram-mod"),
|
||||
]
|
||||
|
||||
|
||||
def local(cmd):
|
||||
"""Run a command locally (the script runs on the target host)."""
|
||||
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=120)
|
||||
if r.returncode != 0:
|
||||
print(f" WARNING: {cmd} -> exit {r.returncode}: {r.stderr.strip()}",
|
||||
file=sys.stderr)
|
||||
return r
|
||||
|
||||
|
||||
def sed_spec_type(value):
|
||||
local(f"sudo sed -i 's/^SPEC_TYPE=.*/SPEC_TYPE={value}/' {ENV_FILE}")
|
||||
|
||||
|
||||
def wait_health(host, port, timeout=HEALTH_TIMEOUT):
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
try:
|
||||
resp = urllib.request.urlopen(
|
||||
f"http://{host}:{port}/health", timeout=10
|
||||
)
|
||||
if resp.status == 200:
|
||||
return True
|
||||
except (urllib.error.URLError, urllib.error.HTTPError, ConnectionError,
|
||||
OSError):
|
||||
pass
|
||||
time.sleep(HEALTH_INTERVAL)
|
||||
return False
|
||||
|
||||
|
||||
def benchmark(host, port, alias, max_tokens):
|
||||
body = json.dumps({
|
||||
"model": alias,
|
||||
"messages": [{"role": "user", "content": PROMPT}],
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": 0,
|
||||
"stream": False,
|
||||
}).encode()
|
||||
req = urllib.request.Request(
|
||||
f"http://{host}:{port}/v1/chat/completions",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
t0 = time.time()
|
||||
resp = urllib.request.urlopen(req, timeout=1200)
|
||||
wall = time.time() - t0
|
||||
data = json.loads(resp.read())
|
||||
return data, wall
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--host", default=DEFAULT_HOST)
|
||||
ap.add_argument("--port", type=int, default=DEFAULT_PORT)
|
||||
ap.add_argument("--max-tokens", type=int, default=DEFAULT_MAX_TOKENS)
|
||||
args = ap.parse_args()
|
||||
|
||||
# Read original SPEC_TYPE from the env file
|
||||
r = local(f"grep '^SPEC_TYPE=' {ENV_FILE}")
|
||||
original_spec = r.stdout.strip()
|
||||
orig_val = (original_spec.split("=", 1)[1]
|
||||
if "=" in original_spec else "draft-mtp,ngram-mod")
|
||||
print(f"Original: {original_spec}")
|
||||
print(f"Max tokens: {args.max_tokens}")
|
||||
print(f"Configs: {[c[0] for c in CONFIGS]}")
|
||||
print()
|
||||
|
||||
# Stop + mask the production service
|
||||
print("Stopping + masking production service...")
|
||||
local(f"sudo systemctl stop {SERVICE}")
|
||||
local(f"sudo systemctl mask {SERVICE}")
|
||||
time.sleep(2)
|
||||
|
||||
results = []
|
||||
for name, spec_type in CONFIGS:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Config: {name} (SPEC_TYPE={spec_type})")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# Set SPEC_TYPE
|
||||
sed_spec_type(spec_type)
|
||||
print(f"Set SPEC_TYPE={spec_type}")
|
||||
|
||||
# Start the service
|
||||
local(f"sudo systemctl unmask {SERVICE}")
|
||||
local(f"sudo systemctl start {SERVICE}")
|
||||
print("Starting service...")
|
||||
|
||||
# Wait for health
|
||||
print(f"Waiting for health (up to {HEALTH_TIMEOUT}s)...")
|
||||
if not wait_health(args.host, args.port):
|
||||
print(f"FAILED: service did not become healthy for {name}")
|
||||
results.append({
|
||||
"config": name, "spec_type": spec_type,
|
||||
"error": "health timeout",
|
||||
})
|
||||
local(f"sudo systemctl stop {SERVICE}")
|
||||
local(f"sudo systemctl mask {SERVICE}")
|
||||
continue
|
||||
|
||||
print("Healthy. Running benchmark...")
|
||||
|
||||
# Run benchmark
|
||||
try:
|
||||
data, wall = benchmark(
|
||||
args.host, args.port, ALIAS, args.max_tokens
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"FAILED: {e}")
|
||||
results.append({
|
||||
"config": name, "spec_type": spec_type,
|
||||
"error": str(e),
|
||||
})
|
||||
local(f"sudo systemctl stop {SERVICE}")
|
||||
local(f"sudo systemctl mask {SERVICE}")
|
||||
continue
|
||||
|
||||
timings = data.get("timings", {})
|
||||
usage = data.get("usage", {})
|
||||
pred_n = timings.get("predicted_n", usage.get("completion_tokens", 0))
|
||||
pred_ms = timings.get("predicted_ms", 0)
|
||||
pred_ps = timings.get("predicted_per_second", 0)
|
||||
prompt_n = timings.get("prompt_n", usage.get("prompt_tokens", 0))
|
||||
prompt_ms = timings.get("prompt_ms", 0)
|
||||
|
||||
print(f" Generated: {pred_n} tokens")
|
||||
print(f" Wall time: {wall:.1f}s")
|
||||
print(f" Decode: {pred_ps:.2f} tok/s")
|
||||
print(f" Prefill: {prompt_n} tokens in {prompt_ms:.0f}ms"
|
||||
f" ({prompt_n / max(prompt_ms, 1) * 1000:.1f} tok/s)")
|
||||
|
||||
results.append({
|
||||
"config": name,
|
||||
"spec_type": spec_type,
|
||||
"predicted_tokens": pred_n,
|
||||
"predicted_ms": pred_ms,
|
||||
"predicted_per_second": pred_ps,
|
||||
"prompt_tokens": prompt_n,
|
||||
"prompt_ms": prompt_ms,
|
||||
"wall_time": wall,
|
||||
})
|
||||
|
||||
# Stop + mask between configs
|
||||
local(f"sudo systemctl stop {SERVICE}")
|
||||
local(f"sudo systemctl mask {SERVICE}")
|
||||
time.sleep(2)
|
||||
|
||||
# Restore original SPEC_TYPE
|
||||
sed_spec_type(orig_val)
|
||||
print(f"\nRestoring SPEC_TYPE={orig_val}")
|
||||
print("Unmasking + starting production service...")
|
||||
local(f"sudo systemctl unmask {SERVICE}")
|
||||
local(f"sudo systemctl start {SERVICE}")
|
||||
|
||||
# Health check
|
||||
if wait_health(args.host, args.port):
|
||||
print("Production service healthy.")
|
||||
else:
|
||||
print("WARNING: production service not healthy after restore!")
|
||||
|
||||
# Print results table
|
||||
print(f"\n{'='*70}")
|
||||
print("RESULTS")
|
||||
print(f"{'='*70}")
|
||||
print(f"{'Config':<30} {'Tokens':>8} {'Wall(s)':>10} {'Decode tok/s':>14} "
|
||||
f"{'Error':>20}")
|
||||
print(f"{'-'*30} {'-'*8} {'-'*10} {'-'*14} {'-'*20}")
|
||||
for r in results:
|
||||
if "error" in r:
|
||||
print(f"{r['config']:<30} {'':>8} {'':>10} {'':>14} "
|
||||
f"{r['error']:>20}")
|
||||
else:
|
||||
print(f"{r['config']:<30} {r['predicted_tokens']:>8} "
|
||||
f"{r['wall_time']:>10.1f} {r['predicted_per_second']:>14.2f} "
|
||||
f"{'':>20}")
|
||||
print(f"{'='*70}")
|
||||
|
||||
# Save JSON results
|
||||
ts = time.strftime("%Y%m%d-%H%M%S")
|
||||
out = f"/tmp/bench-spec-decode-{ts}.json"
|
||||
with open(out, "w") as f:
|
||||
json.dump({"results": results, "max_tokens": args.max_tokens,
|
||||
"timestamp": ts}, f, indent=2)
|
||||
print(f"Results saved: {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user