diff --git a/dankclaw/authentik/quadlet/authentik-outpost-ldap.container b/dankclaw/authentik/quadlet/authentik-outpost-ldap.container index 2fefc43..ca3922d 100644 --- a/dankclaw/authentik/quadlet/authentik-outpost-ldap.container +++ b/dankclaw/authentik/quadlet/authentik-outpost-ldap.container @@ -23,7 +23,7 @@ Network=dankclaw_default # Image=ghcr.io/goauthentik/ldap:2026.8.3 (amd64 digest; podman 5.7.0 refuses # the tag's nested manifest list, so pin the single-platform digest). EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env -Image=ghcr.io/goauthentik/ldap@sha256:2e7b259eb8c37faac916946ddf48ec681ee9d3a9f724fe8521bf161973f13cf9 +Image=ghcr.io/goauthentik/ldap:2026.8.5@sha256:4c75bd27918418d2d6fbbfe4581bbd5a199fe38942d24559c4483edd64a1ea87 Environment=AUTHENTIK_HOST=https://auth.franksdank.cloud Environment=AUTHENTIK_INSECURE=false Environment=AUTHENTIK_TOKEN=${AUTHENTIK_LDAP_OUTPOST_TOKEN} diff --git a/dankclaw/authentik/quadlet/authentik-outpost.container b/dankclaw/authentik/quadlet/authentik-outpost.container index 20c242e..49b7685 100644 --- a/dankclaw/authentik/quadlet/authentik-outpost.container +++ b/dankclaw/authentik/quadlet/authentik-outpost.container @@ -20,7 +20,7 @@ Wants=authentik-server.service [Container] ContainerName=authentik-outpost -Image=ghcr.io/goauthentik/server:2026.8.3@sha256:ab9b4e8cc4ab3f8d1198d2db6aeea66bafea1963b3f2843589e0d163f97d9849 +Image=ghcr.io/goauthentik/server:2026.8.5@sha256:602227a86455d6772e3d16b7d852713c4f864f40df5df894abd1285bb53979a5 Exec=worker Network=dankclaw_default EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env diff --git a/dankclaw/authentik/quadlet/authentik-postgres.container b/dankclaw/authentik/quadlet/authentik-postgres.container index 53c43d0..9673f16 100644 --- a/dankclaw/authentik/quadlet/authentik-postgres.container +++ b/dankclaw/authentik/quadlet/authentik-postgres.container @@ -14,7 +14,7 @@ After=network.target [Container] ContainerName=authentik-postgres -Image=docker.io/library/postgres:16-alpine@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea +Image=docker.io/library/postgres:16-alpine@sha256:0cefaabd882e42bcb2a2f3dc56d3ce4770560ba62fd7f197f1a2e008fd5731ab Network=dankclaw_default EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env Environment=POSTGRES_DB=authentik diff --git a/dankclaw/authentik/quadlet/authentik-server.container b/dankclaw/authentik/quadlet/authentik-server.container index d71c234..35116bc 100644 --- a/dankclaw/authentik/quadlet/authentik-server.container +++ b/dankclaw/authentik/quadlet/authentik-server.container @@ -19,7 +19,7 @@ Wants=authentik-postgres.service [Container] ContainerName=authentik-server -Image=ghcr.io/goauthentik/server:2026.8.3@sha256:ab9b4e8cc4ab3f8d1198d2db6aeea66bafea1963b3f2843589e0d163f97d9849 +Image=ghcr.io/goauthentik/server:2026.8.5@sha256:602227a86455d6772e3d16b7d852713c4f864f40df5df894abd1285bb53979a5 Exec=server Network=dankclaw_default EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env diff --git a/dankclaw/authentik/quadlet/authentik-worker.container b/dankclaw/authentik/quadlet/authentik-worker.container index 81a572f..5d634cc 100644 --- a/dankclaw/authentik/quadlet/authentik-worker.container +++ b/dankclaw/authentik/quadlet/authentik-worker.container @@ -16,7 +16,7 @@ Wants=authentik-server.service [Container] ContainerName=authentik-worker -Image=ghcr.io/goauthentik/server:2026.8.3@sha256:ab9b4e8cc4ab3f8d1198d2db6aeea66bafea1963b3f2843589e0d163f97d9849 +Image=ghcr.io/goauthentik/server:2026.8.5@sha256:602227a86455d6772e3d16b7d852713c4f864f40df5df894abd1285bb53979a5 Exec=worker Network=dankclaw_default EnvironmentFile=/home/dank/serverAppConfigs/dankclaw/.env diff --git a/dankclaw/gitea/quadlet/gitea-postgres.container b/dankclaw/gitea/quadlet/gitea-postgres.container index 663c162..1d7e10a 100644 --- a/dankclaw/gitea/quadlet/gitea-postgres.container +++ b/dankclaw/gitea/quadlet/gitea-postgres.container @@ -15,7 +15,7 @@ After=network.target [Container] ContainerName=gitea-postgres Network=gitea.network -Image=docker.io/library/postgres:16-alpine@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea +Image=docker.io/library/postgres:16-alpine@sha256:0cefaabd882e42bcb2a2f3dc56d3ce4770560ba62fd7f197f1a2e008fd5731ab EnvironmentFile=/home/dank/Containers/gitea/env Environment=POSTGRES_USER=gitea Environment=POSTGRES_PASSWORD=${GITEA_DB_PASSWORD} diff --git a/dankclaw/hermes/quadlet/chrome-devtools-protocol.container b/dankclaw/hermes/quadlet/chrome-devtools-protocol.container index 2204bf7..732b3a9 100644 --- a/dankclaw/hermes/quadlet/chrome-devtools-protocol.container +++ b/dankclaw/hermes/quadlet/chrome-devtools-protocol.container @@ -25,7 +25,7 @@ Description=Chrome DevTools Protocol daemon (headless-shell) for hermes [Container] ContainerName=chrome-devtools-protocol Network=host -Image=docker.io/chromedp/headless-shell@sha256:2d349b544a1ea6b5b5fd7c0fe99215ff662339c57407ee2e8c0a11af93516b04 +Image=docker.io/chromedp/headless-shell@sha256:7bee201478b2f6266aae01243740ee85c16281e32f42d3aed9fdf04f4eace7a1 Entrypoint=/headless-shell/headless-shell Exec=--no-sandbox --disable-gpu --use-gl=angle --use-angle=swiftshader --remote-debugging-address=127.0.0.1 --remote-debugging-port=9222 ShmSize=1073741824 diff --git a/dankclaw/marreta/quadlet/marreta-browser.container b/dankclaw/marreta/quadlet/marreta-browser.container index f2f40dc..3229d53 100644 --- a/dankclaw/marreta/quadlet/marreta-browser.container +++ b/dankclaw/marreta/quadlet/marreta-browser.container @@ -23,7 +23,7 @@ After=gluetun.service network.target [Container] ContainerName=marreta_browser Network=gluetun.container -Image=docker.io/lightpanda/browser:nightly@sha256:979c36ed869f1016b8dcf22939a0633c262f2c30e892a03a9f720eba2ff1931e +Image=docker.io/lightpanda/browser:nightly@sha256:e72730e364d1fbf352c1149f29ffa4b278f3b1a55ff0dec3262298ebe624ec7a [Service] TimeoutStartSec=300 diff --git a/dankclaw/observability/quadlet/alertmanager.container b/dankclaw/observability/quadlet/alertmanager.container index a21ebb3..78fcad6 100644 --- a/dankclaw/observability/quadlet/alertmanager.container +++ b/dankclaw/observability/quadlet/alertmanager.container @@ -33,7 +33,7 @@ Wants=network-online.target [Container] ContainerName=alertmanager -Image=docker.io/prom/alertmanager:latest@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a +Image=docker.io/prom/alertmanager:latest@sha256:39d8bb3966c694a124428a3a88fbcffd40bd069129ecd47c059f8a3f17c55f3b Network=dankclaw_default User=0:0 Exec=--config.file=/etc/alertmanager/alertmanager.yml --storage.path=/alertmanager diff --git a/dankclaw/observability/quadlet/grafana.container b/dankclaw/observability/quadlet/grafana.container index 17770ec..162d64f 100644 --- a/dankclaw/observability/quadlet/grafana.container +++ b/dankclaw/observability/quadlet/grafana.container @@ -34,7 +34,7 @@ Wants=network-online.target prometheus.service [Container] ContainerName=grafana -Image=docker.io/grafana/grafana:latest@sha256:dbbc38ea21103cefb50396340cd88afec779c7e070b0a7bf5a75b4ce46651d3f +Image=docker.io/grafana/grafana:latest@sha256:488984b716c0862639f91a698adbe2862677a5a07bad42da5f03f3030883b205 Network=dankclaw_default User=0:0 PublishPort=127.0.0.1:3002:3000 diff --git a/dankclaw/open-webui/quadlet/open-webui.container b/dankclaw/open-webui/quadlet/open-webui.container index 51105e1..f728193 100644 --- a/dankclaw/open-webui/quadlet/open-webui.container +++ b/dankclaw/open-webui/quadlet/open-webui.container @@ -27,7 +27,7 @@ After=network.target [Container] ContainerName=open-webui Network=open-webui.network -Image=ghcr.io/open-webui/open-webui:main@sha256:4860ba005ae8f19f08d1e12b0868855b7b339ea0dad9927f196820fc0de04575 +Image=ghcr.io/open-webui/open-webui:main@sha256:0b399a8ba34558e181d316750455f2a95376a38cd60a61b9c942e138898944bd PublishPort=3005:8080 Environment=OPENAI_API_BASE_URL=http://192.168.0.143:7770 Environment=ANONYMIZED_TELEMETRY=False diff --git a/dankclaw/searxng/quadlet/gluetun.container b/dankclaw/searxng/quadlet/gluetun.container index a226e08..2c6360f 100644 --- a/dankclaw/searxng/quadlet/gluetun.container +++ b/dankclaw/searxng/quadlet/gluetun.container @@ -16,7 +16,7 @@ After=searxng-network.service network.target [Container] ContainerName=gluetun Network=searxng.network -Image=docker.io/qmcgaw/gluetun:latest@sha256:c2ee4191762fca6b6a3c3359a731198a95d9c9af4db32d466e223bf3a84e779b +Image=docker.io/qmcgaw/gluetun:latest@sha256:4af0c804003481e100180e1f8d4d5e9c48163fedc0f88d061881b615aef9d183 AddCapability=NET_ADMIN AddDevice=/dev/net/tun:/dev/net/tun Environment=VPN_SERVICE_PROVIDER=${VPN_SERVICE_PROVIDER} diff --git a/dankclaw/searxng/quadlet/searxng-valkey.container b/dankclaw/searxng/quadlet/searxng-valkey.container index 9a58f0e..6c1a055 100644 --- a/dankclaw/searxng/quadlet/searxng-valkey.container +++ b/dankclaw/searxng/quadlet/searxng-valkey.container @@ -16,7 +16,7 @@ After=searxng-network.service network.target ContainerName=searxng-valkey Network=searxng.network NetworkAlias=valkey -Image=docker.io/valkey/valkey:9-alpine@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11 +Image=docker.io/valkey/valkey:9-alpine@sha256:bf09d58e96d6b4524a15396b65b242bbe18beebbcc15a04d8ee75ddac62fae0b Volume=/opt/searxng/valkey:/data [Service] diff --git a/dankclaw/searxng/quadlet/searxng.container b/dankclaw/searxng/quadlet/searxng.container index 7961486..2f539a6 100644 --- a/dankclaw/searxng/quadlet/searxng.container +++ b/dankclaw/searxng/quadlet/searxng.container @@ -16,7 +16,7 @@ After=gluetun.service searxng-valkey.service network.target [Container] ContainerName=searxng Network=gluetun.container -Image=ghcr.io/searxng/searxng:latest@sha256:5ed0323b5f5f8701264ef65e239a2bd83cb57098f2b971fa30bf79c06eeae271 +Image=ghcr.io/searxng/searxng:latest@sha256:f4aa660c54d062dedb268be84e821ea24e39f48c7a496af8dd02cc9eedb93847 Environment=SEARXNG_SECRET=${SEARXNG_SECRET} Environment=SEARXNG_BASE_URL=${SEARXNG_BASE_URL} Volume=/opt/searxng/config:/etc/searxng diff --git a/dankclaw/seerr/quadlet/seerr.container b/dankclaw/seerr/quadlet/seerr.container index f2adf47..aa77d7f 100644 --- a/dankclaw/seerr/quadlet/seerr.container +++ b/dankclaw/seerr/quadlet/seerr.container @@ -29,7 +29,7 @@ Wants=network-online.target [Container] ContainerName=seerr -Image=ghcr.io/seerr-team/seerr:v3.4.1@sha256:f4768de5f616248d723e05891f3345a1402123775d03bf0890dbfedc0831bda1 +Image=ghcr.io/seerr-team/seerr:v3.5.1@sha256:d8fab675b55ad293ded98fd71efcfd46054cbad9746534ae1cc4794a88c0b3be Network=dankclaw_default PublishPort=127.0.0.1:5055:5055 Volume=/home/dank/Containers/seerr/data:/app/config diff --git a/dankserver/calibre-server/quadlet/calibre-server.container b/dankserver/calibre-server/quadlet/calibre-server.container index e9d3762..bd526a3 100644 --- a/dankserver/calibre-server/quadlet/calibre-server.container +++ b/dankserver/calibre-server/quadlet/calibre-server.container @@ -43,7 +43,7 @@ After=network.target [Container] ContainerName=calibre-server -Image=lscr.io/linuxserver/calibre:9.15.0@sha256:59e4b0e036e62f228f8bcfae1ade42f47047719885fe454ddbad38e4597b987b +Image=lscr.io/linuxserver/calibre:9.17.0@sha256:b198fc03a6abc6be7e251a9958e50e74634b00e447c7b25e92c191c8077ba378 Network=pasta EnvironmentFile=/home/dank/Containers/calibre-server/env Environment=PUID=1000 diff --git a/dankserver/calibre-web/quadlet/calibre-web.container b/dankserver/calibre-web/quadlet/calibre-web.container index 75a3c46..4177268 100644 --- a/dankserver/calibre-web/quadlet/calibre-web.container +++ b/dankserver/calibre-web/quadlet/calibre-web.container @@ -16,7 +16,7 @@ After=network.target [Container] ContainerName=calibre-web -Image=lscr.io/linuxserver/calibre-web:0.6.27-ls402@sha256:802729966faed5e32d671cd4f92ba6196e9e754a6c0f740ce1ee9acd86b10e55 +Image=lscr.io/linuxserver/calibre-web:0.6.27-ls407@sha256:470b4ed9a7d6ea1475fef17dbcf957cb60880910296d226bd0dca652c05f68ca Network=pasta Environment=DOCKER_MODS=linuxserver/mods:universal-calibre Environment=LANG=en_US.UTF-8 diff --git a/dankserver/jellyfin/quadlet/jellyfin.container b/dankserver/jellyfin/quadlet/jellyfin.container index 88bc693..afa45a5 100644 --- a/dankserver/jellyfin/quadlet/jellyfin.container +++ b/dankserver/jellyfin/quadlet/jellyfin.container @@ -16,7 +16,7 @@ After=network.target [Container] ContainerName=jellyfin -Image=ghcr.io/jellyfin/jellyfin:latest@sha256:c3efa69a22f2691fc88d6bb035e87a8560e395a6047f062fb381c61c3185e8f5 +Image=ghcr.io/jellyfin/jellyfin:latest@sha256:fea863d0cd9ec445985d9c75fc6ed64e38a1d2766cf0c250a33d553f4fc1b771 Network=dankserver_appnet Environment=DEBIAN_FRONTEND=noninteractive Environment=DOTNET_SYSTEM_IO_DISABLEFILELOCKING=true diff --git a/dankserver/plex/quadlet/plex.container b/dankserver/plex/quadlet/plex.container index 1a72e56..60a36d6 100644 --- a/dankserver/plex/quadlet/plex.container +++ b/dankserver/plex/quadlet/plex.container @@ -4,7 +4,7 @@ After=network-online.target Wants=network-online.target [Container] -Image=lscr.io/linuxserver/plex:1.43.5.11029-6bf8af3f0-ls329@sha256:8f655bad7dcbc5398a78f70a5af12600b4ed3424f901566ed1eaa6710e9e2334 +Image=lscr.io/linuxserver/plex:1.43.5.11079-b3b13e506-ls330@sha256:01a53d9c7adcbb3701fbb0a4230c0a37c85453759b6b05a67356c3c7a09de08d ContainerName=plex Network=dankserver_appnet:alias=plex Environment=PUID=0 diff --git a/dankserver/romm/quadlet/romm-mariadb.container b/dankserver/romm/quadlet/romm-mariadb.container index 130ebf8..1540065 100644 --- a/dankserver/romm/quadlet/romm-mariadb.container +++ b/dankserver/romm/quadlet/romm-mariadb.container @@ -14,7 +14,7 @@ After=network.target [Container] ContainerName=romm-mariadb -Image=docker.io/library/mariadb:11@sha256:f16d7a344413e519dff518f27bdb221388f84b0d274c97c4fb5f2cb0dc1b34fa +Image=docker.io/library/mariadb:11@sha256:c4124d1afa875e47f2892edf26b482fa54f796e2f342d3717a04ca091a22a9bd Network=dankserver_appnet:alias=mariadb EnvironmentFile=/home/dank/Containers/romm/env Environment=MARIADB_DATABASE=romm diff --git a/dankserver/romm/quadlet/romm-redis.container b/dankserver/romm/quadlet/romm-redis.container index e935b37..3fd15a9 100644 --- a/dankserver/romm/quadlet/romm-redis.container +++ b/dankserver/romm/quadlet/romm-redis.container @@ -12,7 +12,7 @@ After=network.target [Container] ContainerName=romm-redis -Image=docker.io/library/redis:7-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 +Image=docker.io/library/redis:7-alpine@sha256:8da19c7399dca900c6822a1cf30f9d2285585e2c4b457ff4095b0f4500054d3f Network=dankserver_appnet:alias=redis Volume=/home/dank/Containers/romm/redis:/data HealthCmd=redis-cli ping diff --git a/dankserver/romm/quadlet/romm.container b/dankserver/romm/quadlet/romm.container index 858fc31..19c78a6 100644 --- a/dankserver/romm/quadlet/romm.container +++ b/dankserver/romm/quadlet/romm.container @@ -18,7 +18,7 @@ After=romm-mariadb.service romm-redis.service network.target [Container] ContainerName=romm -Image=ghcr.io/rommapp/romm:latest@sha256:03295f95a6290e752641f9553ac4198b0e5b71b879d57f7b2371b98f608feb54 +Image=ghcr.io/rommapp/romm:latest@sha256:8677fb7b81b1d3db20b849e989a1dadc2a11159e51eb34a2acdcb01102274edf Network=dankserver_appnet EnvironmentFile=/home/dank/Containers/romm/env Environment=DB_HOST=mariadb diff --git a/dankserver/tautulli/quadlet/tautulli.container b/dankserver/tautulli/quadlet/tautulli.container index 8ab4ae3..001b0de 100644 --- a/dankserver/tautulli/quadlet/tautulli.container +++ b/dankserver/tautulli/quadlet/tautulli.container @@ -4,7 +4,7 @@ After=network-online.target Wants=network-online.target [Container] -Image=lscr.io/linuxserver/tautulli:v2.18.1-ls245@sha256:c66d3afa8f7a701a48dd1b1f57b22eca0483d895d670fe11ecff783f6a9bbd73 +Image=lscr.io/linuxserver/tautulli:v2.18.2-ls250@sha256:529c7eabcc40554c1dc48f33f1f3cc3b1438462d73e0ec2dac8ac0b66aeeb8d4 ContainerName=tautulli Network=dankserver_appnet:alias=tautulli Environment=PUID=0 diff --git a/scripts/calib-push.py b/scripts/calib-push.py index beffdfb..6bd8ee1 100755 --- a/scripts/calib-push.py +++ b/scripts/calib-push.py @@ -22,9 +22,17 @@ Mechanics notes: prompts do not bias the Hessians (the corpus is drawn round-robin over source classes, so loss is class-neutral as long as failures are rare; a systematic 4xx is loud in the log and must be fixed, not ignored). -- a round that seals no tap bytes is a FAILURE, not a retry: the engine - answering 200 while the tap directory is wrong, full or read-only is - how a corpus gets pushed four times for nothing. +- a round that seals no tap bytes is NORMAL and continues: RADIANCE_CALIB_TOKENS + is a pooled floor and this campaign's corpus is 10.03M tokens against a 12M + floor, so round 1 seals nothing by construction. A tap that LOSES bytes fails + the run, and a run that never sees the engine's completion line exits non-zero. + The first version failed on a flat round, which aborted exactly one round too + early (2026-10-29). +- before any prompt, the tap must carry the rig's own marker + (`--tap-marker`, written by run-qwen38-bf16.sh calib) naming THIS + container, and that container must be running. Measured 2026-10-29: + with neither check, four rounds of 10M tokens went into restored + production after the rig's boot had died, and recorded nothing. Usage: calib-push.py [--port 7770] [--parallel 64] [--container arbench] [--corpus /home/dank/corpora/out/calib.jsonl] [--max-rounds 4] @@ -107,6 +115,61 @@ def tap_bytes(path): return total +def round_verdict(before, after, done): + """What one round's tap bytes mean. + + 'done' the engine logged that it sealed the tap — the run is over + 'shrink' the tap LOST bytes: reset, pruned or replaced under us + 'grew' sealed bytes this round + 'flat' sealed nothing — which is legitimate: RADIANCE_CALIB_TOKENS is a + pooled floor, and a round that does not reach it has nothing to + write yet (the campaign's corpus is 10.03M tokens against a 12M + floor, so round 1 is always flat). Failing on it aborted the run + exactly one round too early. + """ + if done: + return "done" + if after < before: + return "shrink" + if after == before: + return "flat" + return "grew" + + +def require_rig(a): + """Refuse unless the rig that opened this tap is up and serving right now. + + Three facts, in order: the runner wrote its marker into the tap dir, the + marker names the container this pusher targets, and that container is + running. A marker without a live container is a boot that died; a live + container without a marker is some other engine on the port — on 2026-10-29 + it was restored production, which took four rounds of 10M tokens before the + old driver admitted it had recorded nothing. + """ + marker = os.path.join(a.tap_dir, a.tap_marker) + if not os.path.isfile(marker): + qc.refuse(f"no rig marker at {marker}: this tap was not opened by " + "run-qwen38-bf16.sh calib, so a push here would feed whatever " + "answers the port and record nothing") + try: + lines = open(marker, encoding="utf-8", errors="replace").read().splitlines() + except OSError as exc: + qc.refuse(f"cannot read the rig marker {marker}: {exc}") + meta = dict(line.split("=", 1) for line in lines if "=" in line) + who = meta.get("container", "") + if who != a.container: + qc.refuse(f"the rig marker names container {who!r} and this pusher targets " + f"{a.container!r} — wrong tap, or a stale marker") + try: + p = subprocess.run(["podman", "inspect", "-f", "{{.State.Running}}", a.container], + capture_output=True, text=True, timeout=30) + except Exception as exc: + qc.refuse(f"cannot inspect the rig container {a.container}: {exc}") + if p.returncode != 0 or p.stdout.strip() != "true": + qc.refuse(f"the rig container {a.container} is not running — whatever is " + "answering the port, it is not the calibration rig") + + def parse_args(): ap = argparse.ArgumentParser() ap.add_argument("--port", type=int, default=7770) @@ -114,6 +177,8 @@ def parse_args(): ap.add_argument("--container", default="arbench") ap.add_argument("--model", default="frank-archon") ap.add_argument("--tap-dir", default="/home/dank/campaign/qwen38-w5nl64/calib") + ap.add_argument("--tap-marker", default=".rig-boot", + help="the file run-qwen38-bf16.sh calib writes into the tap dir; without it the pusher refuses, because an unmarked tap is not a rig") ap.add_argument("--corpus", default="/home/dank/corpora/out/calib.jsonl") ap.add_argument("--max-rounds", type=int, default=4) return ap.parse_args() @@ -136,6 +201,7 @@ def main(): a = parse_args() url = f"http://127.0.0.1:{a.port}/v1/completions" prompts = load_prompts(a) + require_rig(a) chars = sum(len(p) for p in prompts) print(f"corpus: {len(prompts)} prompts, ~{chars // 4} tokens (chars/4)", flush=True) @@ -156,11 +222,15 @@ def main(): if tap_done(a.container): break after = tap_bytes(a.tap_dir) - if after <= before: - print(f"FAIL: the tap at {a.tap_dir} sealed nothing this round " - f"({before} B before, {after} B after) — the engine is serving " - "without recording (wrong dir, full disk, read-only mount)", flush=True) + verdict = round_verdict(before, after, False) + if verdict == "shrink": + print(f"FAIL: the tap at {a.tap_dir} LOST bytes this round " + f"({before} B before, {after} B after) — it was reset or replaced " + "under us", flush=True) sys.exit(1) + if verdict == "flat": + print(f"round {rnd}: the tap sealed nothing yet (byte floor not reached; " + f"{before} B) — continuing", flush=True) else: print("max rounds exhausted, tap not done", flush=True) sys.exit(1) diff --git a/scripts/kld-gate-qwen38.sh b/scripts/kld-gate-qwen38.sh index 6ad5738..e60834f 100755 --- a/scripts/kld-gate-qwen38.sh +++ b/scripts/kld-gate-qwen38.sh @@ -1,44 +1,53 @@ #!/usr/bin/env bash # kld-gate-qwen38.sh — RUNS ON DANKSERVER (or anywhere the reports are). The campaign's -# acceptance gate: three KLD reports against ONE reference recording, one verdict. +# acceptance gate: one verdict from the KLD reports of the containers that ride one reference +# recording. # # The reference is the BF16 container built from the vendor checkpoint (run-qwen38-bf16.sh # kldref, in the same drained window that records the calibration grams). Every scored # container is measured against that one recording, so the KL scale runs from 0 = the -# checkpoint itself downward in faithfulness. Three containers ride it: +# checkpoint itself downward in faithfulness. # -# fp8.json the published FP8-native release — the achievable end of the interval -# iq4r.json the payload frank-archon serves — the bar to beat -# cand.json the arm under test — the subject +# TWO MODES, because the disk decides how many anchors can exist at once: # -# The FP8 number is what makes the interval absolute instead of relative: "half the gap" -# is half of the distance between the 4-bit payload and the near-lossless release, not half -# of an invented distance to the checkpoint. That is also why the FP8 payload is FETCHED and -# scored, never converted: it is a second opinion about what this checkpoint tolerates, not a -# step of the campaign. It lands LAST (disk sequencing) — by then the 335 GiB bf16 container -# is gone, so its 172.8 GiB fits. +# gate IQ4R.json CAND.json candidate vs the served payload +# gate FP8.json IQ4R.json CAND.json the same, inside the absolute interval # -# The law, in order of severity, on the mean KL of one slice (--slice all | a corpus source): -# 1. KL(fp8) < KL(iq4r) the anchors are ordered. If the near-lossless -# release does NOT sit closer to the checkpoint than -# the 4-bit payload, the corpus, the tokenizer or the -# boot shape is wrong and no arm number means anything. -# 2. KL(fp8) < KL(cand) < KL(iq4r) the candidate lies strictly inside the interval — -# outside it, something is mis-measured, not better -# 3. KL(cand) <= KL(iq4r) - R x gap it closes at least R (--recover, default 0.5) of the -# gap = KL(iq4r) - KL(fp8): at least half the way from -# what is served toward the release (agreed 2026-10-27) -# 4. top-1(cand) >= top-1(iq4r) the agreement the harness keys on does not fall +# iq4r.json is the payload frank-archon serves — the bar to beat. When the FP8-native release +# (172.8 GiB, fetched last because of the peak-disk law) is on disk, pass its report first: +# "half the gap" is then half of the distance between the 4-bit payload and the near-lossless +# release, not half of an invented distance to the checkpoint. Without it the gate still +# answers the campaign's question — does the candidate beat what is served — and says so. +# +# The law, on the summary of one slice (--slice all | a corpus source): +# +# candidate vs served (2 reports) +# 1. mean KL(cand) < mean KL(iq4r) the served payload is beaten on the corpus +# 2. p99.9(cand) <= p99.9(iq4r) and the tail is no worse — a mean can hide a few +# positions taking the whole hit +# 3. top-1(cand) >= top-1(iq4r) the agreement the harness keys on does not fall +# 4. every source slice passes 1-3 a win bought by regressing one distribution +# (code, prose, tools) is not a win +# +# inside the interval (3 reports), laws 1-4 above (1-3 renumbered to the interval and the +# recovery fraction, as agreed 2026-10-27) plus: +# 5. p99.9(cand) <= p99.9(iq4r) the tail law +# 6. every source slice passes the slice law # # Booting the containers is NOT this script's job: `run-qwen38-bf16.sh score RAD OUT.json` # writes the reports and owns the single engine argv (same pin, --tp 2, --kv-cache-dtype fp8, -# same step size, --deterministic, no speculation) that makes the three numbers comparable. +# same step size, --deterministic, no speculation) that makes the numbers comparable. # This script is arithmetic, so it re-runs against old reports and is testable without a GPU. # -# Usage: kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F] +# Usage: kld-gate-qwen38.sh IQ4R.json CAND.json [--slice NAME] +# kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F] set -uo pipefail -[ "$#" -ge 3 ] || { echo "usage: kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]" >&2; exit 2; } +if [ "$#" -lt 2 ]; then + echo "usage: kld-gate-qwen38.sh IQ4R.json CAND.json [--slice NAME]" >&2 + echo " kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]" >&2 + exit 2 +fi python3 - "$@" <<'PY' import json, sys @@ -52,13 +61,15 @@ while i < len(a): else: files.append(a[i]) i += 1 -if len(files) != 3: - sys.exit("usage: kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]") +if len(files) not in (2, 3): + sys.exit("usage: kld-gate-qwen38.sh IQ4R.json CAND.json [--slice NAME]\n" + " kld-gate-qwen38.sh FP8.json IQ4R.json CAND.json [--slice NAME] [--recover F]") o = dict(zip(opts[0::2], opts[1::2])) slice_ = o.get("--slice", "all") recover = float(o.get("--recover", 0.5)) if not 0.0 < recover <= 1.0: sys.exit(f"--recover must be in (0,1], got {recover}") +interval = len(files) == 3 def die(msg): @@ -66,11 +77,15 @@ def die(msg): raise SystemExit(2) -def read(path): +def load(path): try: - d = json.load(open(path)) + return json.load(open(path)) except Exception as e: die(f"cannot read the KLD report {path}: {e}") + + +def summary(path, d): + """The slice this gate grades: 'all', or one corpus source.""" if slice_ == "all": g = d.get("all") else: @@ -80,23 +95,25 @@ def read(path): f"{', '.join(sorted(d.get('by_source') or {})) or '(none)'}") if g is None or "kld" not in g or "top1_agreement" not in g: die(f"{path} has no {slice_!r} summary — wrong report version?") - return d, g + return g +docs = {label: load(path) for label, path in + zip(("fp8", "iq4r", "cand") if interval else ("iq4r", "cand"), files)} rows = [] -for label, path in zip(("fp8", "iq4r", "cand"), files): - d, g = read(path) - rows.append((label, path, d.get("model", "?"), g["kld"]["mean"], g["kld"]["p99_9"], - g["top1_agreement"], g["positions"])) +for label, path in zip(("fp8", "iq4r", "cand") if interval else ("iq4r", "cand"), files): + g = summary(path, docs[label]) + rows.append((label, path, docs[label].get("model", "?"), g["kld"]["mean"], + g["kld"]["p99_9"], g["top1_agreement"], g["positions"])) -print(f"reference: BF16 container; slice: {slice_}") +print(f"reference: BF16 container; slice: {slice_}; " + + ("inside the FP8 interval" if interval else "candidate vs served (no FP8 anchor)")) print(f"{'':6} {'model':38} {'KL mean':>10} {'KL p99.9':>10} {'top-1':>8} positions") for label, path, model, kl, p999, top1, n in rows: print(f"{label:6} {str(model)[:38]:38} {kl:10.6f} {p999:10.4f} {100*top1:7.2f}% {n}") -r = {label: (kl, top1) for label, _p, _m, kl, _q, top1, _n in rows} -kl_fp8, kl_iq4r, kl_cand = r["fp8"][0], r["iq4r"][0], r["cand"][0] -t_iq4r, t_cand = r["iq4r"][1], r["cand"][1] -gap = kl_iq4r - kl_fp8 +r = {label: (kl, p999, top1) for label, _p, _m, kl, p999, top1, _n in rows} +kl_cand, p_cand, t_cand = r["cand"] +kl_iq4r, p_iq4r, t_iq4r = r["iq4r"] ok = True @@ -106,17 +123,64 @@ def law(n, text, passed): print(f" [{'PASS' if passed else 'FAIL'}] {n}. {text}") -law(1, f"anchors ordered: fp8 {kl_fp8:.6f} < iq4r {kl_iq4r:.6f} (gap {gap:.6f})", - gap > 0) -law(2, f"candidate inside the interval: {kl_fp8:.6f} < {kl_cand:.6f} < {kl_iq4r:.6f}", - kl_fp8 < kl_cand < kl_iq4r) -need = kl_iq4r - recover * gap -law(3, f"candidate closes {100*max((kl_iq4r - kl_cand)/gap, 0):.1f}% of the " - f"{gap:.6f} gap (needs {100*recover:.0f}%, i.e. KL <= {need:.6f})" - if gap > 0 else "gap is not positive; cannot score recovery", - gap > 0 and kl_cand <= need) -law(4, f"candidate top-1 {100*t_cand:.2f}% >= served iq4r top-1 {100*t_iq4r:.2f}%", - t_cand >= t_iq4r) +def tail_law(n): + law(n, f"candidate KL p99.9 {p_cand:.4f} <= served iq4r {p_iq4r:.4f}", + p_cand <= p_iq4r) + + +# --- the slice law: every corpus source must pass the metric laws ------------- +def slice_law(n): + """A win that regresses one source class is not a win: check them all.""" + mine = docs["cand"].get("by_source") or {} + theirs = docs["iq4r"].get("by_source") or {} + if not mine or not theirs: + law(n, "no per-source summaries to check", True) + return + missing = sorted(set(theirs) - set(mine)) + extra = sorted(set(mine) - set(theirs)) + if missing or extra: + law(n, "same source slices in both reports" + + (f" (missing from cand: {', '.join(missing)})" if missing else "") + + (f" (only in cand: {', '.join(extra)})" if extra else ""), False) + return + bad = [] + for src in sorted(theirs): + c, q = mine[src], theirs[src] + worse_mean = c["kld"]["mean"] >= q["kld"]["mean"] + worse_tail = c["kld"]["p99_9"] > q["kld"]["p99_9"] + worse_top1 = c["top1_agreement"] < q["top1_agreement"] + if worse_mean or worse_tail or worse_top1: + why = ",".join(m for m, w in (("mean", worse_mean), ("p99.9", worse_tail), + ("top-1", worse_top1)) if w) + bad.append(f"{src} ({why})") + law(n, "every source slice passes (mean, p99.9, top-1)" + + (f" — regressed: {'; '.join(bad)}" if bad else ""), not bad) + + +if interval: + kl_fp8 = r["fp8"][0] + gap = kl_iq4r - kl_fp8 + law(1, f"anchors ordered: fp8 {kl_fp8:.6f} < iq4r {kl_iq4r:.6f} (gap {gap:.6f})", + gap > 0) + law(2, f"candidate inside the interval: {kl_fp8:.6f} < {kl_cand:.6f} < {kl_iq4r:.6f}", + kl_fp8 < kl_cand < kl_iq4r) + need = kl_iq4r - recover * gap + law(3, f"candidate closes {100*max((kl_iq4r - kl_cand)/gap, 0):.1f}% of the " + f"{gap:.6f} gap (needs {100*recover:.0f}%, i.e. KL <= {need:.6f})" + if gap > 0 else "gap is not positive; cannot score recovery", + gap > 0 and kl_cand <= need) + law(4, f"candidate top-1 {100*t_cand:.2f}% >= served iq4r top-1 {100*t_iq4r:.2f}%", + t_cand >= t_iq4r) + tail_law(5) + slice_law(6) +else: + law(1, f"candidate beats the served payload: KL {kl_cand:.6f} < iq4r {kl_iq4r:.6f}", + kl_cand < kl_iq4r) + tail_law(2) + law(3, f"candidate top-1 {100*t_cand:.2f}% >= served iq4r top-1 {100*t_iq4r:.2f}%", + t_cand >= t_iq4r) + slice_law(4) + print("KLD GATE " + ("PASS" if ok else "FAIL")) sys.exit(0 if ok else 1) PY diff --git a/scripts/qwen38-calib-window.sh b/scripts/qwen38-calib-window.sh new file mode 100755 index 0000000..15e3c51 --- /dev/null +++ b/scripts/qwen38-calib-window.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# qwen38-calib-window.sh — RUNS ON DANKSERVER. The calibration window, and nothing else: +# boot the BF16 rig, wait until it is actually serving, push the corpus until the engine says +# the tap is full, stop the rig, restore frank-archon, and bank the reference container. +# +# The reference recording is already taken (kld-ref/kld.json exists), so this window is the +# calibration tap alone — the one job that needs both cards after it. The kldref phase is NOT +# re-run here: the chained launcher that used to do that moved the banked kld-ref into trash +# first, which is how a 16-hour phase gets paid for twice. +# +# Every phase is gated on the previous one, because the last time this ran unguarded +# (2026-10-29) the rig's boot died of an OOM, the health loop quietly fell through, and the +# pusher fed four rounds of 10M tokens into RESTORED PRODUCTION with nothing recorded: +# +# 1. the runner must still be alive and its container must answer /health, or this script +# kills it, restores the service and exits non-zero — no push, ever; +# 2. the pusher refuses on its own unless the rig wrote its marker into the tap dir, the +# marker names this container, and the container is running (scripts/calib-push.py +# --tap-marker), so a second lock survives a mistake in this file; +# 3. the reference container is banked into trash only when BOTH the serve and the push +# exited 0 and the tap dir holds grams. The checkpoint that reproduces it stays. +# +# knobs: Q38_WINDOW_CALIB_CORPUS (calib-7168.jsonl — the re-chunked file the rig's pool can +# hold), Q38_WINDOW_HEALTH_TIMEOUT (seconds to wait for /health), Q38_WINDOW_ROUNDS +# (pusher rounds; one is 10M tokens), Q38_WINDOW_KEEP_RAD=1 to leave the .rad in place. +# Usage: scripts/qwen38-calib-window.sh [--dry-run] +set -uo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 9 + +DRY=0 +[ "${1:-}" = "--dry-run" ] && DRY=1 +say() { printf '%s %s\n' "$(date -Is)" "$*"; } + +MODELS="${FRANK_QWEN38_MODELS:-/home/dank/models}" +CAMPAIGN="${FRANK_QWEN38_CAMPAIGN:-/home/dank/campaign/qwen38-w5nl64}" +RAD="${FRANK_QWEN38_RAD:-$MODELS/rad/qwen38-bf16.rad}" +CORPUS="${Q38_WINDOW_CALIB_CORPUS:-/home/dank/corpora/out/calib-7168.jsonl}" +KLDREF="$CAMPAIGN/kld-ref/kld.json" +CALIB="$CAMPAIGN/calib" +HEALTH_TIMEOUT="${Q38_WINDOW_HEALTH_TIMEOUT:-3600}" +ROUNDS="${Q38_WINDOW_ROUNDS:-6}" +RIG=qwen38-bf16 +SERVICE=frank-archon +PY="${PY:-python3}" + +# ---- preflight: everything this window needs, before a card is touched ------- +[ -f "$KLDREF" ] || { echo "REFUSED: no reference recording at $KLDREF — record it first (scripts/run-qwen38-bf16.sh kldref); this window only fills the calibration tap" >&2; exit 2; } +[ -f "$RAD" ] || { echo "REFUSED: no BF16 reference container at $RAD — nothing to serve; scripts/build-qwen38-bf16-rad.sh" >&2; exit 2; } +[ -f "$CORPUS" ] || { echo "REFUSED: no calibration corpus at $CORPUS" >&2; exit 2; } +[ -d "$CALIB" ] || { echo "REFUSED: no tap dir at $CALIB" >&2; exit 2; } +[ -w "$CALIB" ] || { echo "REFUSED: tap dir $CALIB is not writable by $(id -un)" >&2; exit 2; } +! [[ "$HEALTH_TIMEOUT" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_HEALTH_TIMEOUT must be seconds (got: $HEALTH_TIMEOUT)" >&2; exit 2; } +! [[ "$ROUNDS" =~ ^[0-9]+$ ]] && { echo "REFUSED: Q38_WINDOW_ROUNDS must be a round count (got: $ROUNDS)" >&2; exit 2; } + +if [ "$DRY" = 1 ]; then + echo "dry run: calib window — the service is left running, nothing is mounted or written" + echo "would run: bash scripts/run-qwen38-bf16.sh calib (rig boots, writes $CALIB/.rig-boot)" + echo "would wait: /health up within ${HEALTH_TIMEOUT}s, else kill the rig, restore $SERVICE, exit 1" + echo "would run: $PY scripts/calib-push.py --container $RIG --corpus $CORPUS --max-rounds $ROUNDS" + echo "would bank: $RAD -> ~/.AGENTTRASH/$(date +%Y%m%d)-qwen38-bf16-rad/ (only if both phases exit 0)" + exit 0 +fi + +say "calib window start pid=$$ corpus=$CORPUS rounds=$ROUNDS" +[ -n "$(ls -A "$CALIB" 2>/dev/null)" ] && say "tap dir $CALIB is not empty; the engine's token floor is pooled, so an earlier partial tap still counts" + +# ---- phase 1: the rig ------------------------------------------------------- +say "phase=calib serve begin" +bash scripts/run-qwen38-bf16.sh calib & spid=$! + +rig_up() { + podman inspect -f '{{.State.Running}}' "$RIG" 2>/dev/null | grep -qx true \ + && curl -sf --max-time 10 -o /dev/null http://127.0.0.1:7770/health +} + +waited=0 +until rig_up; do + if ! kill -0 "$spid" 2>/dev/null; then + wait "$spid"; crc=$? + say "phase=calib serve end rc=$crc (the rig exited before it ever answered /health)" + echo "FAIL: the rig boot died (rc=$crc) — nothing was pushed; read /home/dank/logs/qwen38_bf16_calib.log" >&2 + exit 1 + fi + if [ "$waited" -ge "$HEALTH_TIMEOUT" ]; then + say "phase=calib health never came after ${HEALTH_TIMEOUT}s; killing the rig" + podman stop "$RIG" >/dev/null 2>&1 || true + wait "$spid" + echo "FAIL: the rig never answered /health within ${HEALTH_TIMEOUT}s — nothing was pushed" >&2 + exit 1 + fi + sleep 15 + waited=$((waited + 15)) +done +say "phase=calib health ok waited=${waited}s" + +# ---- phase 2: the tap ------------------------------------------------------- +say "phase=calib push begin" +"$PY" scripts/calib-push.py --port 7770 --parallel 8 --container "$RIG" \ + --tap-dir "$CALIB" --corpus "$CORPUS" --max-rounds "$ROUNDS" +prc=$? +say "phase=calib push end rc=$prc" + +podman stop "$RIG" >/dev/null 2>&1 || true +wait "$spid"; crc=$? +say "phase=calib serve end rc=$crc" + +if [ "$prc" -ne 0 ] || [ "$crc" -ne 0 ]; then + echo "FAIL: serve rc=$crc push rc=$prc — the tap is NOT banked and $RAD stays: $(basename "$RAD")" >&2 + exit 1 +fi +[ -n "$(ls -A "$CALIB" 2>/dev/null)" ] || { echo "FAIL: both phases exited 0 but the tap dir $CALIB is empty" >&2; exit 1; } +say "TAP BANKED: $(ls -1 "$CALIB" | wc -l) file(s) in $CALIB" + +# ---- phase 3: bank the reference container ---------------------------------- +if [ "${Q38_WINDOW_KEEP_RAD:-0}" = "1" ]; then + say "reference .rad kept (Q38_WINDOW_KEEP_RAD=1)" +else + T="$HOME/.AGENTTRASH/$(date +%Y%m%d)-qwen38-bf16-rad" + mkdir -p "$T" || { echo "FAIL: cannot create trash dir $T" >&2; exit 1; } + mv "$RAD" "$T/" && say "reference .rad banked in $T (the checkpoint that rebuilds it stays)" +fi +say "calib window end" diff --git a/scripts/run-qwen38-bf16.sh b/scripts/run-qwen38-bf16.sh index fe358b6..e1db5a9 100755 --- a/scripts/run-qwen38-bf16.sh +++ b/scripts/run-qwen38-bf16.sh @@ -92,7 +92,8 @@ # the container reads through /models), FRANK_QWEN38_LOG_DIR, and the step # and slot counts inherited as FRANK_ARCHON_MAX_BATCHED_TOKENS / # FRANK_ARCHON_MAX_SEQS — no per-mode literals (the audit law: a mode that -# carries its own step measures a shape nothing serves) — +# carries its own step measures a shape nothing serves; calib is exempt, see +# Q38_CALIB_MNBT / Q38_CALIB_SEQS below — its output is never compared) — # and FRANK_QWEN38_MAX_MODEL_LEN — the window every mode boots at (32,768; # see the corpus census above, and raise it together with the corpus). # kldref refuses (exit 2) when the longest record of the corpus it is about to @@ -185,6 +186,24 @@ if ! [[ "$MNBT" =~ ^[0-9]+$ ]]; then exit 2 fi +# Calib's shape is NOT a measuring shape. kldref and score produce KL numbers that are only +# comparable across boots at an identical shape, which is why they inherit production's step and +# slots to the letter. The tap's output is layer Hessians, never a number any report compares, so +# its step and slot count are its own — and they are the tap's dominant cost: a 7,168-token +# document cut into four 2,048-token steps re-streams each layer's routed expert union four +# times from the container, so one wide step is worth up to ~4x the read volume. Measured +# 2026-10-29 at the inherited 2,048/4: prefill 71 tok/s and a 47-hour tap; the probe exists to +# find out what the wide step is worth before the campaign pays for two days of drained serving. +# Both default to the inherited value, so nothing moves unless a probe asks it to. +CALIB_MNBT="${Q38_CALIB_MNBT:-$MNBT}" +CALIB_SEQS="${Q38_CALIB_SEQS:-$MAX_SEQS}" +for pair in "Q38_CALIB_MNBT=$CALIB_MNBT" "Q38_CALIB_SEQS=$CALIB_SEQS"; do + if ! [[ "${pair#*=}" =~ ^[0-9]+$ ]]; then + echo "REFUSED: ${pair%%=*} must be a number (got: ${pair#*=})" >&2 + exit 2 + fi +done + # The window every mode boots at. Over-length is a REFUSAL, not a truncation (the # env file says so for production), so this is the number the corpus must fit in. MAX_MODEL_LEN="${FRANK_QWEN38_MAX_MODEL_LEN:-32768}" @@ -193,6 +212,25 @@ if ! [[ "$MAX_MODEL_LEN" =~ ^[0-9]+$ ]]; then exit 2 fi +# Where the n-gram table lives. On the bf16 payload it is 95.37 GiB and `ram` +# PINS it: the calibration boot died at "startup stopped: out of memory" with +# 230.22 GiB pinned of 246.07 GiB and 8.06 GiB available (measured 2026-10-29), +# while the reference recording and every score boot had already succeeded with +# the same pool and slots. The difference that matters is the job: calib only +# needs the routed weights prefilled, so it reads the table from the container's +# own file tier and the pin never happens; kldref and score keep production's +# placement, because their numbers are only comparable when every other term of +# the shape is identical. FRANK_QWEN38_NGRAM_PLACEMENT overrides either default. +case "$MODE" in + calib) NGRAM_PLACEMENT="${FRANK_QWEN38_NGRAM_PLACEMENT:-disk}" ;; + *) NGRAM_PLACEMENT="${FRANK_QWEN38_NGRAM_PLACEMENT:-ram}" ;; +esac +case "$NGRAM_PLACEMENT" in + auto|vram|ram|disk) ;; + *) echo "REFUSED: FRANK_QWEN38_NGRAM_PLACEMENT must be auto|vram|ram|disk (got: $NGRAM_PLACEMENT)" >&2 + exit 2 ;; +esac + # Every container this campaign boots is read through the ro /models mount, so # the host path must live under it; the container-side spelling is what goes # into the argv. Same rule for the container a score pass opens. @@ -230,7 +268,7 @@ ARGS=(--tp 2 --placement expert_tiered --weights-disk-tier --host-pool-mib "$HOST_POOL_MIB" --gpu-headroom-mib "$GPU_HEADROOM_MIB" --deterministic --num-speculative-tokens 0 --kv-cache-dtype fp8 --max-model-len "$MAX_MODEL_LEN" - --ngram-placement ram + --ngram-placement "$NGRAM_PLACEMENT" --override-chat-template /models/tpl.jinja) MOUNTS=(-v "$MODELS:/models:ro") TAPENV=() @@ -277,7 +315,7 @@ case "$MODE" in # VRAM knob. The slots follow frank-archon.env's FRANK_ARCHON_MAX_SEQS # through MAX_SEQS, exactly as kldref and score do. ARGS+=(--model "$MODEL_IN" --port 7770 - --max-num-seqs "$MAX_SEQS" --max-num-batched-tokens "$MNBT") + --max-num-seqs "$CALIB_SEQS" --max-num-batched-tokens "$CALIB_MNBT") # The tap is only real if this directory exists and the engine can write it: # without the check a typo'd CALIB_DIR auto-creates a root-owned empty dir in # whatever path was given, and the run burns hours serving prompts into a tap @@ -289,6 +327,17 @@ case "$MODE" in MOUNTS+=(-v "$CALIB_DIR:/data/calib") TAPENV=(-e RADIANCE_CALIB_DIR=/data/calib -e "RADIANCE_CALIB_TOKENS=${FRANK_QWEN38_CALIB_TOKENS:-12000000}") + # The tap's own authorization. A pusher must see this marker before it sends + # a prompt (calib-push.py --tap-marker), it names the container so a push + # aimed anywhere else refuses, and a boot that fails removes it again below. + # Measured 2026-10-29: with no marker, the chain pushed 4 rounds x 10M tokens + # into RESTORED PRODUCTION after the rig's boot died, and nothing recorded. + if [ "$DRY" = 1 ]; then + : + else + printf 'container=%s\nimage=%s\nmode=calib\nstarted=%s\npid=%s\n' \ + "$C" "$IMAGE" "$(date -Is)" "$$" > "$CALIB_DIR/.rig-boot" + fi ;; esac @@ -377,5 +426,12 @@ podman rm -f "$C" >/dev/null 2>&1 || true "${CMD[@]}" 2>&1 | tee "$LOG" rc=${PIPESTATUS[0]} +# A boot that failed must not leave the tap marked as rig-authorized: the marker +# is what calib-push.py trusts, and the point of it is that a dead rig can never +# be pushed into again. +if [ "$MODE" = calib ] && [ "$rc" -ne 0 ] && [ -f "$CALIB_DIR/.rig-boot" ]; then + mv -f "$CALIB_DIR/.rig-boot" "$CALIB_DIR/.rig-boot.failed-$(date +%Y%m%d-%H%M%S)" 2>/dev/null || true + echo "the rig boot failed (rc=$rc): the tap marker is retired, so no pusher can take this tap for a live rig" +fi echo "run-qwen38-bf16 $MODE exited rc=$rc; $SERVICE restored; log: $LOG" exit "$rc" diff --git a/serverAppConfigs-docs/LLM/frank-archon.md b/serverAppConfigs-docs/LLM/frank-archon.md index a449d85..c1442a5 100644 --- a/serverAppConfigs-docs/LLM/frank-archon.md +++ b/serverAppConfigs-docs/LLM/frank-archon.md @@ -562,6 +562,29 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10). fork bomb inside the container. Kill with `podman rm -f` (the cgroup owns every generation) never pattern-kill an interpreter name on this host; use `TORCH_LOGS` for dynamo questions. +### The quantization pipeline + +- **A calibration tap is only a tap when the rig that opened it is running.** The runner writes + `.rig-boot` into the tap dir (container, image, pid) and retires it when its boot exits non-zero; + `calib-push.py` refuses without that marker, refuses when it names another container, and refuses + when the named container is not running; `qwen38-calib-window.sh` stops on a failed phase instead + of falling through its health wait. All three exist because on 2026-10-29 the unguarded chain's + rig boot died of an OOM, the health loop expired without failing, and the pusher fed four rounds + of 10M tokens into **restored production** — 86 minutes of live prefill that recorded nothing. +- **The n-gram table's pin is what starves a rig boot, and the tier is a knob.** On the BF16 payload + the table is 95.37 GiB, `--ngram-placement ram` pins it, and the calibration boot died at + `startup stopped: out of memory` with 230.22 GiB pinned of 246.07 GiB and 8.06 GiB available — + while the reference recording and every score boot succeeded at the same pool and slot count. + Calibration only needs the routed weights prefilled, so it reads the table from the container's + file tier (`--ngram-placement disk`); kldref and score keep production's placement, because their + numbers are only comparable at an identical shape. +- **Acceptance is three metrics and every slice, not a mean.** `scripts/kld-gate-qwen38.sh` grades + mean KL, the p99.9 tail, top-1 agreement and every `by_source` slice (code, prose, tools): + a mean can hide a few positions taking the whole hit, and an overall win can be bought by + regressing one distribution. The FP8-native anchor is an interval, not a requirement — without + the 172.8 GiB fetch the gate runs in two-report mode (candidate vs the served payload) and says + so on its first line. + ## Contract (the wire) - **Live surface:** `/health` (== `/ping`), `/v1/models`, `/v1/chat/completions` @@ -638,14 +661,19 @@ vllm-radiance era (this table opens on 09-26; the era page claims 10-10). rebuild), `scripts/download-frank-archon-swift15.sh` (the BF16 source a w4dn8 re-convert needs), `scripts/hfget.sh` / `scripts/hf-fetch.sh`; template `scripts/qwen38-froggeric-v22.5.jinja`. -- Convert campaign (dormant, kept for a future quant): `scripts/convert-swift15-w4dn8.sh` - `scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe`, `scripts/run-swift15-bf16.sh` - `scripts/kld-gate-swift15.sh`, `scripts/kld-then-calib.sh`, `scripts/calib-push.py` +- The campaign (qwen38-w5nl64, live): `scripts/run-qwen38-bf16.sh` (kldref / calib / score — the + ONE measuring shape), `scripts/qwen38-calib-window.sh` (the gated calibration window), + `scripts/convert-qwen38-w5nl64.sh` with + `scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe` (the arm driver and its template), + `scripts/kld-gate-qwen38.sh` (acceptance), `scripts/calib-push.py`, `scripts/make-calib-corpus.py`, `scripts/rechunk-kld-corpus.py`, `scripts/qwen38_common.py` (the tokenizer, JSONL and refusal plumbing those three corpus tools share — one `n_tokens`, so every tool counts the same way), - `scripts/derive-protected-experts.py` - `scripts/apply-protected-cast.sh`, `scripts/verify-swift15-ple.py`. + `scripts/derive-protected-experts.py`, `scripts/apply-protected-cast.sh`. +- Retired swift15-era campaign scripts, superseded by the above and kept only as reference: + `scripts/convert-swift15-w4dn8.sh`, `scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe`, + `scripts/run-swift15-bf16.sh`, `scripts/kld-gate-swift15.sh`, `scripts/kld-then-calib.sh`, + `scripts/verify-swift15-ple.py`. - Probes: `scripts/probe-frank-archon-kv.py` (reads slots/window/dtype from the deployed env so it cannot drift; leg D = decode), `scripts/frank-archon-expert-activity.py` `scripts/autoresearch-frank-archon.sh` + `scripts/autoresearch/frank-archon-measure.py`. diff --git a/tests/qwen38_campaign.bats b/tests/qwen38_campaign.bats index f061a8d..fdaece6 100755 --- a/tests/qwen38_campaign.bats +++ b/tests/qwen38_campaign.bats @@ -200,6 +200,9 @@ DERIVE="$REPO_ROOT/scripts/derive-protected-experts.py" APPLY="$REPO_ROOT/scripts/apply-protected-cast.sh" CONVERT="$REPO_ROOT/scripts/convert-qwen38-w5nl64.sh" GATE="$REPO_ROOT/scripts/kld-gate-qwen38.sh" +RUNNER="$REPO_ROOT/scripts/run-qwen38-bf16.sh" +PUSHER="$REPO_ROOT/scripts/calib-push.py" +WINDOW="$REPO_ROOT/scripts/qwen38-calib-window.sh" TEMPLATE="$REPO_ROOT/scripts/recipes/qwen4exp-qwen38-w5nl64-bf16trunk.recipe" # make_gram — a RADGRAM2 down-Gram whose @@ -525,3 +528,206 @@ gate_case() { # gate_case [extra flags. [[ "$output" == *"has no slice 'code'"* ]] [[ "$output" == *"it has: ops"* ]] } + +# --- the n-gram tier: the pin that starved the calibration boot --------------- + +@test "the calibration boot reads the n-gram table from the file tier" { + # 2026-10-29: 95.37 GiB pinned beside a 65.59 GiB/rank pool left 8.06 GiB of + # RAM and the boot died "startup stopped: out of memory". calib only needs the + # routed weights prefilled, so it reads the table off the container's file tier. + run bash "$RUNNER" calib --dry-run + [ "$status" -eq 0 ] + grep -A1 -x -- '--ngram-placement' <<<"$output" | grep -qx 'disk' +} + +@test "the recording and every score pass keep production's n-gram placement" { + # Those numbers are only comparable when the shape is identical; the tier is + # part of the shape. + run bash "$RUNNER" kldref --dry-run + [ "$status" -eq 0 ] + grep -A1 -x -- '--ngram-placement' <<<"$output" | grep -qx 'ram' +} + +@test "an unknown n-gram placement is refused by name before the boot" { + run env FRANK_QWEN38_NGRAM_PLACEMENT=ssd bash "$RUNNER" calib --dry-run + [ "$status" -eq 2 ] + [[ "$output" == *"auto|vram|ram|disk"* ]] +} + +# --- the pusher: a tap without a live rig is not a tap ------------------------ + +# stub_podman — a podman that answers the one question the +# pusher asks: is the rig container running? +stub_podman() { + mkdir -p "$1/bin" + if [ "$2" = running ]; then + printf '#!/usr/bin/env bash\necho true\n' > "$1/bin/podman" + elif [ "$2" = stopped ]; then + printf '#!/usr/bin/env bash\necho false\n' > "$1/bin/podman" + else + printf '#!/usr/bin/env bash\nexit 125\n' > "$1/bin/podman" + fi + chmod +x "$1/bin/podman" +} + +pusher_fixture() { # pusher_fixture + D="$BATS_TEST_TMPDIR/push" + mkdir -p "$D/tap" "$D/bin" + printf '{"prompt":"hello"}\n' > "$D/corpus.jsonl" + if [ -n "$1" ]; then + printf 'container=%s\nimage=radiance@sha256:test\nmode=calib\npid=1\n' "$1" \ + > "$D/tap/.rig-boot" + fi + echo "$D" +} + +@test "the pusher refuses a tap that carries no rig marker: production is not a rig" { + # With no marker, the old driver fed four rounds of 10M tokens into restored + # production after the rig's boot had died, and recorded nothing. + D="$(pusher_fixture '')" + run python3 "$PUSHER" --corpus "$D/corpus.jsonl" --tap-dir "$D/tap" --port 1 + [ "$status" -eq 2 ] + [[ "$output" == *"no rig marker"* ]] +} + +@test "the pusher refuses a marker that names another container" { + D="$(pusher_fixture arbench)" + run python3 "$PUSHER" --corpus "$D/corpus.jsonl" --tap-dir "$D/tap" \ + --container qwen38-bf16 --port 1 + [ "$status" -eq 2 ] + [[ "$output" == *"names container 'arbench'"* ]] + [[ "$output" == *"targets 'qwen38-bf16'"* ]] +} + +@test "the pusher refuses when the rig container is not running" { + D="$(pusher_fixture qwen38-bf16)" + stub_podman "$D" stopped + run env PATH="$D/bin:$PATH" python3 "$PUSHER" --corpus "$D/corpus.jsonl" \ + --tap-dir "$D/tap" --container qwen38-bf16 --port 1 + [ "$status" -eq 2 ] + [[ "$output" == *"is not running"* ]] +} + +# --- the acceptance gate without an FP8 anchor ------------------------------- + +# kld_report_slices +kld_report_slices() { + python3 - "$@" <<'PY' +import json, sys +path, model, kl_all, kl_code, t_all, t_code = sys.argv[1:7] +def g(kl, t, n=25000): + return {"kld": {"mean": float(kl), "p99_9": float(kl) * 4}, + "top1_agreement": float(t), "positions": n} +json.dump({"model": model, "all": g(kl_all, t_all), + "by_source": {"code": g(kl_code, t_code), "ops": g(kl_all, t_all)}}, + open(path, "w")) +PY +} + +gate2() { # gate2 [extra] + D="$BATS_TEST_TMPDIR/kl2" + mkdir -p "$D" + kld_report_slices "$D/iq4r.json" iq4r-moe "$1" "$1" "$3" "$3" + kld_report_slices "$D/cand.json" w5nl64 "$2" "$2" "$4" "$4" + run bash "$GATE" "$D/iq4r.json" "$D/cand.json" "${@:5}" +} + +@test "a candidate that beats the served payload on mean, tail and top-1 passes without an anchor" { + gate2 0.0190 0.0090 0.962 0.971 + [ "$status" -eq 0 ] + [[ "$output" == *"candidate vs served (no FP8 anchor)"* ]] + [[ "$output" == *"KLD GATE PASS"* ]] +} + +@test "a better mean with a worse tail fails: the tail is where a mean hides its damage" { + D="$BATS_TEST_TMPDIR/kl3" + mkdir -p "$D" + kld_report_slices "$D/iq4r.json" iq4r-moe 0.0190 0.0190 0.962 0.962 + python3 - "$D/cand.json" <<'PY' +import json, sys +# mean better than the served payload, p99.9 worse: the shape a mean-only gate waves through +g = {"kld": {"mean": 0.0090, "p99_9": 0.4000}, "top1_agreement": 0.971, "positions": 25000} +json.dump({"model": "w5nl64", "all": g, "by_source": {"code": dict(g), "ops": dict(g)}}, + open(sys.argv[1], "w")) +PY + run bash "$GATE" "$D/iq4r.json" "$D/cand.json" + [ "$status" -eq 1 ] + [[ "$output" == *"[FAIL] 2."* ]] + [[ "$output" == *"KL p99.9"* ]] +} + +@test "a candidate that wins overall but regresses one source slice fails" { + D="$BATS_TEST_TMPDIR/kl4" + mkdir -p "$D" + kld_report_slices "$D/iq4r.json" iq4r-moe 0.0190 0.0190 0.962 0.962 + # 'all' is better, 'code' is worse: exactly the trade a mean-only gate cannot see + kld_report_slices "$D/cand.json" w5nl64 0.0090 0.0300 0.971 0.950 + run bash "$GATE" "$D/iq4r.json" "$D/cand.json" + [ "$status" -eq 1 ] + [[ "$output" == *"[PASS] 1."* ]] + [[ "$output" == *"[FAIL] 4."* ]] + [[ "$output" == *"regressed: code"* ]] +} + +# --- the calibration window: gated, and it says so --------------------------- + +window_fixture() { + D="$BATS_TEST_TMPDIR/win" + mkdir -p "$D/campaign/kld-ref" "$D/campaign/calib" "$D/rad" "$D/corpora" + echo '{}' > "$D/campaign/kld-ref/kld.json" + : > "$D/rad/qwen38-bf16.rad" + : > "$D/corpora/calib.jsonl" + echo "$D" +} + +@test "the calibration window refuses before touching a card without a reference recording" { + D="$BATS_TEST_TMPDIR/win-less" + mkdir -p "$D/campaign/calib" + run env FRANK_QWEN38_CAMPAIGN="$D/campaign" bash "$WINDOW" + [ "$status" -eq 2 ] + [[ "$output" == *"no reference recording"* ]] +} + +@test "the calibration window's dry run prints the gated plan, including what it would bank" { + D="$(window_fixture)" + run env FRANK_QWEN38_CAMPAIGN="$D/campaign" \ + FRANK_QWEN38_RAD="$D/rad/qwen38-bf16.rad" \ + Q38_WINDOW_CALIB_CORPUS="$D/corpora/calib.jsonl" bash "$WINDOW" --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"run-qwen38-bf16.sh calib"* ]] + [[ "$output" == *"else kill the rig"* ]] + [[ "$output" == *"calib-push.py"* ]] + [[ "$output" == *"AGENTTRASH"* ]] +} + +@test "a flat calibration round is not a failure: the tap's floor is pooled" { + # round 1 of a 10.03M-token corpus against a 12M floor seals nothing, and the + # first version of this gate aborted the run there — one round too early. + run python3 - "$PUSHER" <<'PY' +import importlib.util, sys +spec = importlib.util.spec_from_file_location("cp", sys.argv[1]) +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) +print(m.round_verdict(8192, 8192, False), # flat: continue + m.round_verdict(8192, 4096, False), # shrink: fail + m.round_verdict(8192, 20000, False), # grew: continue + m.round_verdict(0, 0, True)) # done: stop +PY + [ "$status" -eq 0 ] + [ "$output" = "flat shrink grew done" ] +} + +@test "the calibration boot's step is its own knob; the KL modes keep production's" { + # Calib is the only phase whose cost is read volume per document: four 2,048-token steps + # re-stream each layer's routed experts four times. Measured 2026-10-29: 71 tok/s, 47 h tap. + run env Q38_CALIB_MNBT=8192 bash "$RUNNER" calib --dry-run + [ "$status" -eq 0 ] + grep -A1 -x -- '--max-num-batched-tokens' <<<"$output" | grep -qx '8192' + # and the same knob must NOT reach the measuring shape those numbers are compared in + run env Q38_CALIB_MNBT=8192 bash "$RUNNER" kldref --dry-run + [ "$status" -eq 0 ] + ! grep -qx '8192' <<<"$output" + run env Q38_CALIB_MNBT=wide bash "$RUNNER" calib --dry-run + [ "$status" -eq 2 ] + [[ "$output" == *"Q38_CALIB_MNBT must be a number"* ]] +}