diff --git a/scripts/convert-swift15-w4dn8.sh b/scripts/convert-swift15-w4dn8.sh index 6a19690..d005cf6 100755 --- a/scripts/convert-swift15-w4dn8.sh +++ b/scripts/convert-swift15-w4dn8.sh @@ -27,7 +27,8 @@ CALIB=/home/dank/corpora/calib OUT=/home/dank/models/rad/qwen4exp-swift15-w4dn8-i8-hc8m.rad LOG=/home/dank/logs/swift15_convert.log # gate_up ~37.6 + down ~18.8 + PLE 47.7 + trunk/embeddings/head ~10 GiB (all-w4 expert table; -# the plugin's one-encoding law dropped the block-fp8 down tier -- see the recipe header). + # the format assigns ONE encoding per expert table, so a per-tensor block-fp8 + # down tier cannot be served beside a w4 gate_up tier -- see the recipe header). MIN_MIB=${FRANK_SWIFT15_RAD_MIN_MIB:-108000} # Gate 1: inputs. A missing Gram directory converts UNCALIBRATED with a diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 4444b51..065ef6d 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -947,7 +947,7 @@ deploy_plex() { # accepted and then answered with nothing (curl 52, Empty reply) against a # perfectly healthy server. Measured 2026-10-02 — the gate burned its full # 600s and failed a plex whose https://127.0.0.1:32400/identity answered - # 200 in 44 ms the whole time. The same law is why the Caddyfile dials the + # 200 in 44 ms the whole time. The same requirement is why the Caddyfile dials the # plex upstream over TLS (tests/caddyfile.bats pins it). # -k: PMS presents its own self-signed cert on 32400. # @@ -1207,8 +1207,9 @@ deploy_llm_dragoon() { # on the FULL container name, which appears in the argv of a manual # `podman run --name frank-dragoon-halogen` / `podman wait` but NOT in # this deployer's own command line (`bash deploy-remote.sh - # frank-dragoon`) — so it cannot shoot itself in the foot (the fleet - # pkill law), and it can never match frank-zealot's llama-server. + # frank-dragoon`) — so the sweep can never match the deployer's own argv + # (`pkill -f` kills the invoker when the pattern appears in the invoker's + # own command line), and it can never match frank-zealot's llama-server. run_privileged systemctl stop frank-dragoon.service 2>/dev/null || true pkill -f 'frank-dragoon-halogen' 2>/dev/null || true podman rm -f frank-dragoon frank-dragoon-halogen >/dev/null 2>&1 || true diff --git a/scripts/derive-protected-experts.py b/scripts/derive-protected-experts.py index 4a3503b..b473cac 100755 --- a/scripts/derive-protected-experts.py +++ b/scripts/derive-protected-experts.py @@ -2,7 +2,7 @@ """derive-protected-experts.py — rewrite the recipe's protected-expert section from a calibration recording. RUNS ON DANKSERVER (venv: numpy). -The law (docs/MOE-W4.md "Protected experts", restated in the recipe): + The protection rule (docs/MOE-W4.md "Protected experts", restated in the recipe): an expert whose share of its layer's down-input energy is at least 3% AND at least three times its share of the routing is kept bf16; the count must be EVEN on each rank's half of the layer (the four-bit GEMM takes the rest diff --git a/scripts/download-frank-archon-swift15.sh b/scripts/download-frank-archon-swift15.sh index 273d4d2..0332e94 100755 --- a/scripts/download-frank-archon-swift15.sh +++ b/scripts/download-frank-archon-swift15.sh @@ -14,7 +14,7 @@ # Fetch uses the vendored scripts/hfget.sh (StillDeadcode's Apache-2.0 # listing/resume downloader, curl+jq only): per-file size checks against # the Hub listing, `-C -` resume, pickle-format duplicates skipped. -# Verify (default) is the AGENTS.md law in three parts: every listed file + # Verify (default) enforces the download contract in three parts: every listed file # present with the listed size, the shard count matches the weight-map # index, and every shard's sha256 equals the Hub's LFS oid. Verify needs # only the public API. diff --git a/scripts/download-frank-archon.sh b/scripts/download-frank-archon.sh index 0f5ff83..4fa4970 100755 --- a/scripts/download-frank-archon.sh +++ b/scripts/download-frank-archon.sh @@ -12,7 +12,7 @@ # Fetch uses the vendored scripts/hfget.sh (StillDeadcode's Apache-2.0 # listing/resume downloader, curl+jq only): per-file size checks against # the Hub listing, `-C -` resume, pickle-format duplicates skipped. -# Verify (default) is the AGENTS.md law in three parts: every listed file + # Verify (default) enforces the download contract in three parts: every listed file # present with the listed size, the shard count matches the weight-map # index, and every shard's sha256 equals the Hub's LFS oid. Verify needs # only the public API. diff --git a/scripts/download-frank-dragoon-flashnext.sh b/scripts/download-frank-dragoon-flashnext.sh index 5c096cf..53ac491 100755 --- a/scripts/download-frank-dragoon-flashnext.sh +++ b/scripts/download-frank-dragoon-flashnext.sh @@ -4,9 +4,10 @@ # .hgn, the REQUIRED lookup table and the optional vision tower from # peonist-ai/halogen-qwen3.8-flash-next (both at the revisions the model # card pins), and the flat tokenizer the two repos share. All public, no -# token — and per the fleet secret law, none would ever touch argv. + # token — and a token never rides a command line: argv is readable by every + # process on the host through /proc, so a key placed there is a key exposed. # -# The fetch law, same as every sibling script: curl -f (an access error + # The fetch discipline, same as every sibling script: curl -f (an access error # must not become a "successful" 153-byte artifact), exact byte count from # the Hub tree API, LFS sha256 verification, --verify-only to re-check an # existing payload without downloading. @@ -26,7 +27,8 @@ # tokenizer/* @e053f488 — byte-identical to the 27B-era tokenizer # already on disk (tokenizer.json LFS # 0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3); -# fetched fresh rather than symlinked, per the no-symlink law. + # fetched fresh rather than symlinked, so the payload dir is + # self-contained — every gate and every mount sees real files inside it. set -euo pipefail DEST="${FRANK_DRAGOON_MODEL_DIR:-$HOME/models/halogen-flashnext-v2}" diff --git a/scripts/download-frank-dragoon-halogen.sh b/scripts/download-frank-dragoon-halogen.sh index ad36204..7ef7107 100755 --- a/scripts/download-frank-dragoon-halogen.sh +++ b/scripts/download-frank-dragoon-halogen.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash # download-frank-dragoon-halogen.sh — fetch frank-dragoon's halogen payload: # the qwen3.8-27b .hgn checkpoint and the export tokenizer, from the -# PUBLIC peonist-ai/halogen-qwen3.8-27b repo (no token — and per the fleet -# secret law, none would ever touch argv). -# -# The fetch law, same as every sibling script: curl -f (an access error + # PUBLIC peonist-ai/halogen-qwen3.8-27b repo (no token — and a token never + # rides a command line: argv is readable by every process on the host + # through /proc, so a key placed there is a key exposed). + # + # The fetch discipline, same as every sibling script: curl -f (an access error # must not become a "successful" 153-byte artifact), exact byte count from # the model card, LFS sha256 verification, --verify-only to re-check an # existing payload without downloading. diff --git a/scripts/frank-archon-expert-activity.py b/scripts/frank-archon-expert-activity.py index d9b7b9d..a5cb021 100644 --- a/scripts/frank-archon-expert-activity.py +++ b/scripts/frank-archon-expert-activity.py @@ -11,8 +11,8 @@ Tensor extents are DERIVED from consecutive GGUF header offsets — no quant type-size table (those are easy to get subtly wrong; a wrong per-element size silently under-covers each tensor and dumps tail samples into a junk bucket). -Fleet law this instrument established (2026-09-26, 75 s window, 4 mixed -streams, 2.7M expert-tensor samples): per-BLOCK read volume is FLAT + What this instrument shows (75 s window, 4 mixed streams, 2.7M + expert-tensor samples): per-BLOCK read volume is FLAT (0.93x-1.08x of mean across all 40 CPU-resident blocks) because MoE top-k routing fires every block every token — there are no "most active layers" at `-ot` granularity. The real skew is per-EXPERT (top 10% of (block,expert) diff --git a/scripts/frank-archon.env b/scripts/frank-archon.env index 3910b83..f6aac8a 100644 --- a/scripts/frank-archon.env +++ b/scripts/frank-archon.env @@ -13,7 +13,7 @@ # +25 % at the full window) while KV beyond the contract buys nothing. # Consequence, deliberate: a FOURTH full-height conversation does not fit this # pair, and the engine refuses it — it never truncates. See the KV block for the -# arithmetic and the measured dtype law. The fp8-native payload this file was + # arithmetic and the measured per-dtype cost. The fp8-native payload this file was # written for is the rollback target and keeps its own trio under the # fp8-native runbook. # @@ -124,13 +124,12 @@ FRANK_ARCHON_TP=2 FRANK_ARCHON_TP_WIRE=wht6 # Decode slots: THREE, so the served window is a CAPACITY CONTRACT — three # concurrent conversations of --max-model-len fit the KV pool (see the KV -# block below). The "seqs 2 cannot boot" law is NOT radiance's: it was the -# tcclaviger vLLM fork's torch.compile shape specialization on -# query_start_loc (ConstraintViolationError, measured 2026-10-02, retired -# with that runtime). Radiance booted seqs 2 at the fp8-native first boot, -# and the 2026-10-06 decode sweep measured seqs 2 and 3 statistically tied -# (30.9-33.1 t/s, inside the +-3% noise floor) — slot count buys concurrency, -# not speed, so a slot costs KV, never decode. + # block below). A runtime that compiles its kernels specialized to the exact + # batch shape can refuse a slot count it has never compiled for; a + # shape-agnostic engine accepts any count that fits memory. Radiance is the + # latter: seqs 2 boots, and measured decode at seqs 2 and 3 is statistically + # tied (30.9-33.1 t/s, inside the +-3% noise floor) — slot count buys + # concurrency, not speed, so a slot costs KV, never decode. # Cost of each extra slot: 110 MiB of per-slot recurrent state inside the KV # pool, and the fused TP all-reduce is declared over seqs x (1 + spec) rows — # 3 x 3 = 9 here, under the 25-row bound at which the wht6 wire stays EXACT @@ -255,9 +254,10 @@ FRANK_ARCHON_MAX_BATCHED_TOKENS=2048 # available with the service loaded). FRANK_ARCHON_PREFIX_HOST_MIB=8192 FRANK_ARCHON_PREFIX_DISK_MIB=65536 -# Frank's pin law: the PLE n-gram table (47.7 GiB) serves from pinned host -# RAM the cards read. `ram` REFUSES the start if the memlock limit is too -# small — the unit's LimitMEMLOCK=infinity is the other half of this law. + # The PLE n-gram table (47.7 GiB) serves from pinned host RAM the cards + # read directly, so it cannot be evicted or page-faulted mid-token. `ram` + # placement REFUSES the start if the memlock limit is too small, and the + # unit's LimitMEMLOCK=infinity is the matching half of that pairing. FRANK_ARCHON_NGRAM_PLACEMENT=ram # TTM preflight: the pinned ledger (2 x 44 GiB pools + 47.7 GiB PLE + 8 GiB # prefix host = ~147 GiB) rides TTM's shared pages, capped by ttm's diff --git a/scripts/frank-dragoon.env b/scripts/frank-dragoon.env index c5103f7..daef8e2 100644 --- a/scripts/frank-dragoon.env +++ b/scripts/frank-dragoon.env @@ -42,10 +42,10 @@ # form of the engine's default, so the seam test can pin the product. # KV costs ~28 KiB/position (FLAGS.md) => 524,288 x 28 KiB = 14.0 GiB. # HALOGEN_KV_SLOTS=2 restores the w4b-era shape on this box: the MTP -# drafter is NOT slot-locked on this engine (batching and speculation -# coexist — DFlash-era slots>1 law does NOT carry over; the card's own -# quickstart ships 2 slots WITH the head), so the second slot is -# concurrent capacity, not a speculation tax. + # drafter is NOT slot-locked on this engine: some engines silently + # disable drafting the moment a second slot is enabled, this one does not + # (the card's own quickstart ships 2 slots WITH the head), so the second + # slot is concurrent capacity, not a speculation tax. # Memory ledger against the box's 121.2 GiB unified (the n-gram table is # mmap'd page cache beside the checkpoint, not a pinned charge — the # card's own quickstart arithmetic does not bill it): @@ -88,8 +88,9 @@ # before. Kernel needs CONFIG_HSA_AMD_SVM (CachyOS ships it) and # amdgpu.noretry=0 (check at deploy). -# Image: digest-pinned (fetch law: tag pulls silently re-point; the digest -# is what was qualified). 0.16.2 = the version the model card names as its + # Image: digest-pinned (a tag is a mutable pointer — the same tag can + # silently deliver different bits — so the digest pins exactly the bits + # that were qualified). 0.16.2 = the version the model card names as its # quickstart pin; RepoDigest captured at pull on 2026-10-06. FRANK_DRAGOON_IMAGE=ghcr.io/peonist-ai/halogen-flash-server@sha256:0c61bf84ac22308a53f5d1ca6b86806702d7039e5ebc51cae4c66621b92fe04a # Container identity: kept from the 27B era (unit ExecStop names it). diff --git a/scripts/frank-zealot.env b/scripts/frank-zealot.env index 32e22db..f15a806 100644 --- a/scripts/frank-zealot.env +++ b/scripts/frank-zealot.env @@ -16,7 +16,7 @@ # custom-quant lock-in like the ROCmFPX era). NO nextn/mtp tensors exist, so # in-file MTP is impossible here; SPEC_TYPE=none is forced by the payload. # -# KV rate law: 8 full layers × 4 KV heads × 256 dim = 8,192 elements per + # KV rate arithmetic: 8 full layers × 4 KV heads × 256 dim = 8,192 elements per # token per cache. f16 K + f16 V = 32,768 B/token = 32 KiB/token, so the # 401,408-token pool is 12.25 GiB. The 24 DeltaNet layers cost a fixed # recurrent state per SLOT (not per token) — that is why two 200k slots fit diff --git a/scripts/patches/glm53-dflash-01-aux-hidden-states.patch b/scripts/patches/glm53-dflash-01-aux-hidden-states.patch index 27cafc2..def9c35 100644 --- a/scripts/patches/glm53-dflash-01-aux-hidden-states.patch +++ b/scripts/patches/glm53-dflash-01-aux-hidden-states.patch @@ -29,7 +29,8 @@ # its base SupportsEagleBase (has_own_lm_head, has_own_embed_tokens) -- and # getattr_static()s each one off the instance, breaking on the first miss. # So BOTH wrappers need all three attributes. -# * MRO law: INHERITING SupportsEagle3 as a base dies at class creation with + # * Method-resolution order forbids the obvious inheritance: adding + # SupportsEagle3 as a base dies at class creation with # "Cannot create a consistent method resolution order (MRO) for bases ..." # (GLM's wrappers already carry four/five Protocol-derived bases). The # attribute route avoids the MRO entirely. Also proven by simulation: diff --git a/scripts/probe-frank-archon-kv.py b/scripts/probe-frank-archon-kv.py index a08b480..4b1cdcb 100755 --- a/scripts/probe-frank-archon-kv.py +++ b/scripts/probe-frank-archon-kv.py @@ -9,7 +9,8 @@ No scratch container, no mask: the service under test is the service. Four probes, in increasing cost: A decode at depth — one ~51k-token chat completion, 512 tokens, greedy. Reports the engine's own `timings` (prefill, decode, draft acceptance). - The bench law applies: >=128 generated tokens, greedy, real text. + Measurement discipline: generate at least 128 tokens (a short run + measures first-token noise, not the server), greedy, real text. B the capacity contract — as many CONCURRENT chats of ~197k prompt tokens as the deployed `FRANK_ARCHON_MAX_SEQS` (read from the repo env file on this host), each carrying a unique marker at its head. All must answer their own diff --git a/scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe b/scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe index d3d6a2a..2b6a504 100644 --- a/scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe +++ b/scripts/recipes/qwen4exp-swift15-w4dn8-i8-hc8m.recipe @@ -1,9 +1,9 @@ # qwen4exp-swift15-w4dn8-i8-hc8m.recipe -- the Swift 1.5 finetune of Qwen3.8-Flash-Next # (ukisai/Swift1.5-Qwen3.8-Flash-Next, BF16 source, 335.3 GiB, key set identical to the official # base) as a radiance container for frank-archon: EVERY routed expert -- gate_up AND down -- -# through GPTQ into the w4nl64a8h table. The plugin's one-table-one-encoding law + # through GPTQ into the w4nl64a8h table. The format assigns exactly ONE encoding per table # ("one table of experts takes one encoding, and only a whole expert kept plain bf16 is served -# beside it", qwen4exp_fp8 rad_arch_declare) forbids the original split tier (down block-fp8 + + # beside it", qwen4exp_fp8 rad_arch_declare — so the original split tier (down block-fp8 + # gate_up w4): the served kernel is w4nl64a8h for the whole expert set, bf16 experts beside it. # The rest is qwen4exp-w5dn64-i8-hc8m.recipe verbatim: int8 trunk linears, block-fp8 hc mixers # and MTP fc, the fixed-scale n-gram table, the 2-bit draft head. @@ -26,8 +26,8 @@ # The protected experts, cast to bf16 ahead of both expert rules. THE NAMES BELOW ARE THE BASE # CHECKPOINT'S (from its 1M-token recording, qwen4exp-w5dn64-i8-hc8m.recipe) -- a warm start only. # The Swift 1.5 finetune's routing distribution is not assumed equal: the derivation step rewrites -# this section from Swift's OWN calibration recording before this recipe is ever run, by the base's -# law -- an expert whose share of its layer's down-input energy is at least 3% and at least three + # this section from Swift's OWN calibration recording before this recipe is ever run, by the + # protection rule -- an expert whose share of its layer's down-input energy is at least 3% and at least three # times its share of the routing, an even count on each rank's half of the layer (the four-bit GEMM # takes the rest as two equal tables by parity) by adding that half's next most energetic expert. # PROTECTED-BEGIN (rewritten from Swift's OWN grams by derive-protected-experts.py) diff --git a/scripts/serve-frank-dragoon.sh b/scripts/serve-frank-dragoon.sh index 135fe38..98381b3 100755 --- a/scripts/serve-frank-dragoon.sh +++ b/scripts/serve-frank-dragoon.sh @@ -70,7 +70,9 @@ fi # Size-exact gates on all three payload files: a truncated curl exits 0 on # some failures and a partial .hgn loads far enough to look alive, so the -# byte counts from the Hub tree API are the contract (fleet fetch law). + # byte counts from the Hub tree API are the contract: curl exits 0 on an + # HTTP error and writes the error body as the file, so a truncated download + # is caught by size here and never boots. ck() { # ck if [ ! -f "$2" ]; then diff --git a/scripts/serve-frank-zealot.sh b/scripts/serve-frank-zealot.sh index f186f4d..5bd34bc 100755 --- a/scripts/serve-frank-zealot.sh +++ b/scripts/serve-frank-zealot.sh @@ -63,8 +63,9 @@ fi : "${CACHE_TYPE_K:=f16}" : "${CACHE_TYPE_V:=f16}" # Speculative decoding is OFF: this Q6_K file carries NO nextn/mtp tensors, so -# there is no in-file head (the law: MTP is a property of the file, not of the -# flags). SPEC_TYPE=none strips BOTH speculative flags (flags move as a set); + # there is no in-file head — this kind of drafter exists only as tensors + # inside the checkpoint file, and no flag can supply one the file lacks. + # SPEC_TYPE=none strips BOTH speculative flags (they move as a set); # naming a drafter requires SPEC_DRAFT_FILE alongside SPEC_TYPE — the Gemma4 # assistant on disk is arch gemma4-assistant and cannot drafter-match qwen35. : "${SPEC_TYPE:=none}" diff --git a/scripts/stack-qwen38-official-fp8.py b/scripts/stack-qwen38-official-fp8.py index 106e983..c1e1763 100755 --- a/scripts/stack-qwen38-official-fp8.py +++ b/scripts/stack-qwen38-official-fp8.py @@ -298,7 +298,7 @@ def main(): and list(dssh) == [(dsh[0] + 127) // 128, (dsh[1] + 127) // 128]) if not scale_law: log(f"ERROR: unexpected expert shapes: gate {gdt}{gsh} scale {gsd}{gssh} " - f"down {ddt}{dsh} scale {dsd}{dssh} (scale law 128x128)") + f"down {ddt}{dsh} scale {dsd}{dssh} (128x128 block scales)") return 2 shapes = {"ffn2_h": (gsh[0] * 2, gsh[1]), "ffn": (dsh[1],), "gs": tuple(gssh), "ds": tuple(dssh)} diff --git a/serverAppConfigs-docs/Operations/LLM Usage Attribution.md b/serverAppConfigs-docs/Operations/LLM Usage Attribution.md new file mode 100644 index 0000000..b69a3ad --- /dev/null +++ b/serverAppConfigs-docs/Operations/LLM Usage Attribution.md @@ -0,0 +1,199 @@ +--- +type: reference +title: LLM Usage Attribution +description: Research catalog of every viable way to answer "who is using the fleet LLMs" — engine-native identity, gateways, auth proxies, tracing platforms, network/host attribution — with the measured fleet baseline (2026-10-13) and a phased adoption plan. +tags: [llm, observability, auth, gateway, litellm, tracking, identity] +timestamp: 2026-10-14T00:00:00-07:00 +--- + +# LLM Usage Attribution + +Deep-research catalog (2026-10-13/14): solutions for tracking WHO uses the +fleet endpoints (`frank-archon`, `frank-dragoon`, `frank-zealot`). Full web +citations; live-fleet measurements marked **(measured)**. + +## Baseline: what the fleet answers today + +- All three endpoints are `auth: none`, plain HTTP, LAN-only + (`models.yml` providers; no vhost on 7770/7771). +- **(measured)** Live `/metrics` label census: `frank-archon` exports + `vllm:*`+`radiance:*` with labels only `{model_name, engine, le, + finished_reason, position, class}` — zero identity labels. + `frank-dragoon` exports `halogen:*`+`llamacpp:*`. + `frank-zealot` answers **501** — `serve-frank-zealot.sh` never passes + `--metrics`, so the Prometheus job is structurally never-green + (`LlmEndpointDown` fires). Fix: add `--metrics` to the ARGS array. +- **(measured)** frank-zealot verbosity-4 journal lines carry slot/task/timing + only — no client IP, no user. Alloy ships journal + podman ctr.log to Loki, + so what lands there has no identity either. +- Caddy has no `log` directive anywhere → no per-request access logs on the + OpenWebUI vhost. +- Net: fleet-level token telemetry exists per ENDPOINT; per-CONSUMER + identity exists nowhere. + +## The identity layers (pick the layer, get its resolution) + +| # | Layer | Identity | Answers | +|---|---|---|---| +| L0 | DHCP/ARP | MAC, hostname, lease | which device was on the LAN | +| L1 | DNS (Pi-hole v6) | device → queries | which device resolved/used a name | +| L2 | L3/L4 (nftables/conntrack, JA3/JA4) | IP, TLS client lib | bytes per client, client software | +| L3 | Transport (API keys, mTLS, SPIFFE, Tailscale) | key/cert/node | which credential connected | +| L4 | App gateway (virtual keys → user/team/end_user) | named consumer | tokens/spend/cost per who | +| L5 | Trace/content (Langfuse, Helicone, halogen records) | user_id, session, prompt text | what they asked, per session | +| L6 | Client host (auditd, eBPF, netdata) | pid/uid/comm per connect | which process on which host | + +Keys/headers/`user` fields are CLIENT-DECLARED — they group traffic, they +prove nothing about a human (halogen's own docs warn this verbatim: +"they group requests, and they prove nothing about who sent them"). + +## Catalog + +### A. Engine-native (per-consumer identity inside the inference server) + +| Engine | AuthN | Identity in logs | Per-identity metrics | OTel | Notes | +|---|---|---|---|---|---| +| llama.cpp | `--api-key` (multi, comma) / `--api-key-file` **(measured in the deployed build's `--help`)** | request `user` field drives slot assignment/eviction; `--log-requests` dumps bodies | `--metrics` exports `llamacpp:*`, NO key/user label | none upstream | [server README](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md) | +| vLLM | `--api-key` (multi) | request log; no user persistence | `vllm:*`; NO `api_key` label (cardinality, issue #53926) | native (`--otlp-traces-endpoint`) | [metrics design](https://github.com/vllm-project/vllm/blob/main/docs/design/metrics.md) | +| radiance | no public auth docs **[unverified]** | — | **(measured)** no identity labels on live /metrics | none found | closed deployment surface; treat authN as absent until probed | +| halogen-flash-server | `HALOGEN_API_KEY` / `HALOGEN_API_KEY_FILE` (multi, comma) | **`HALOGEN_RECORD_DIR` request records**: `turns/*.jsonl` per request — `client_family` (User-Agent), tokens, timings, finish, tool calls, plus HASHES of `user` / `prompt_cache_key` / `safety_identifier` / `metadata.user_id` (`.salt`-keyed); `units/` content-addressed text; caps `HALOGEN_RECORD_GIB`=2, `HALOGEN_RECORD_DAYS`=90; `HALOGEN_RECORD_TEXT=none` for metadata-only; `stats`/`prune` tools in-container | `halogen:*`+`llamacpp:*`, no per-key label | none | [configuration.md](https://github.com/peonist-ai/halogen-flash-server/blob/main/docs/configuration.md) — verbatim-verified 2026-10-13. Best native ledger of the fleet. | +| SGLang | `--api-key` (single) | `--log-requests` | `sglang:*`, no key label | native (`--enable-trace`) | for any future swap | +| Ollama | single key | request log | `ollama:` no key label | none | not in fleet | + +Wire fields usable from clients today (all client-declared): OpenAI +`user`, `safety_identifier`, `prompt_cache_key`, Messages +`metadata.user_id`; LiteLLM `end_user_id`; gateway header conventions +(`X-Forwarded-User`, `Helicone-User-Id`, `cf-aig-metadata`). + +### B. LLM gateways (front door with virtual keys — the only layer with +per-consumer TOKEN counts) + +| Solution | Identity | Usage lands | License | Footprint | +|---|---|---|---|---| +| **LiteLLM Proxy** | virtual keys → `user_id`/`team_id`/`end_user`; budgets, rate limits | Postgres spend ledger; `/metrics` labels `api_key_alias,user,team,end_user,model,client_ip,user_agent` (`litellm_*_tokens_metric`, spend, budget gauges); OTel/Langfuse callbacks | MIT core (SSO/RBAC/audit-log enterprise) | ~150–300 MiB + Postgres | +| Kong AI Gateway | Consumer + key-auth/JWT/OIDC/mTLS; AI consumer groups | Postgres; `kong_*` per-consumer; file-log/zipkin/otel | OSS core; token-aware AI plugins enterprise | 0.5–1 GiB | +| Apache APISIX | Consumer + key-auth/JWT | `apisix_llm_prompt_tokens`/`completion_tokens` with `consumer` label; http-logger/kafka | Apache-2.0 | 200–400 MiB | +| Bifrost (maximhq) | virtual keys `sk-bf-*`/`x-bf-vk`, budgets | Prometheus/OTel, config store (SQLite/Postgres) | Apache-2.0, Go | ~150–300 MiB | +| Portkey OSS gateway | virtual keys | logs/OTel | MIT | 200–400 MiB | +| Higress AI GW | keys (+OIDC ent) | `higress_*`, ai-logging | Apache-2.0 CE | Istio-based, 0.5 GiB+ | +| Envoy AI GW / Gateway API Inference Extension / LLM Gatekeeper | delegated auth | Envoy metrics/access logs | Apache-2.0 | K8s-native | +| Gloo / KrakenD / Gravitee AI gateways | keys, OIDC | metrics/logs | OSS core, AI features enterprise | heavy | +| Cloudflare AI Gateway | Access JWT `cf.user_id`, `cf-aig-metadata` | Cloudflare-managed analytics/logs | SaaS (exotic for LAN; Tunnel possible) | edge | + +([LiteLLM virtual keys](https://docs.litellm.ai/docs/proxy/virtual_keys), +[prometheus](https://docs.litellm.ai/docs/proxy/prometheus), +[APISIX ai-proxy](https://apisix.apache.org/docs/apisix/plugins/ai-proxy/), +[Bifrost](https://github.com/maximhq/bifrost), +[Kong AI](https://developer.konghq.com/index/ai-gateway/)) + +### C. Auth proxies / identity infrastructure (identity without token counts) + +- **Ory Oathkeeper** (MIT, ~50–100 MiB): per-consumer api-key/JWT/ + client-credentials → header mutation (`X-User: {{ .Subject }}`) + + decision logs. +- **oauth2-proxy** (Apache-2.0): OIDC (Authentik) incl. client_credentials, + `--pass-user-headers` → `X-Forwarded-User`; ~100 MiB. +- **Authentik itself**: proxy-provider custom headers (already injects + `X-Authentik-Username`/`-Groups`, spoof-stripped — the /whoami seam proves + the pattern on this host); event log + webhook forwarding. +- **Caddy `tls client_auth`** (mTLS per client cert) + access-log fields — + Caddy already fronts both hosts. +- **Smallstep step-ca**: short-lived per-device certs, ACME-style. +- **SPIFFE/SPIRE**: per-workload SVIDs (k8s-flavored; heavy for quadlets). +- **Tailscale / NetBird**: per-node identity, ACL + network-flow logs, + headscale for self-hosted control. Best for off-LAN laptops. +- **ApiAuth** (nginx auth_request): the minimal per-consumer apikey sidecar. + +### D. LLM observability / tracing (prompt-level per-user audit) + +| Platform | Identity | Content | Self-host | Shape | +|---|---|---|---|---| +| **Langfuse** | `user_id`,`session_id`, tags | full input/output, masking hook | ✅ ClickHouse, Apache-2.0 | fed via LiteLLM callback / OTLP / SDK | +| **Helicone** | api-key→user, `Helicone-User-Id` | full, tiered redaction | ✅ compose (ClickHouse+PG+Redis+Jawn), Apache-2.0 | PROXY — can be the front door itself | +| Arize Phoenix | `user.id`,`session.id` | opt-in `gen_ai.*` | ✅, Apache-2.0 | OTLP collector | +| OpenLIT | `user.id`,`session.id` | 3-tier capture | ✅, Apache-2.0 | OTLP+ClickHouse | +| SigNoz LLM | OTel attrs | opt-in | ✅, Apache-2.0 | OTLP | +| OpenLLMetry/Traceloop | assoc props | opt-in | ✅ Helm | OTel | +| Lunary / W&B Weave / LangSmith | user/session | full | enterprise-gated | — | + +OTel GenAI semconv (Development-stability): `gen_ai.operation.name`, +`gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.usage.input_tokens`/ +`output_tokens`, `gen_ai.conversation.id`, content attrs opt-in; `user.id` +from the registry. vLLM and SGLang emit OTLP natively; llama.cpp only via +external SDK — **the fleet's engines are OTel-silent at request level**, +which is why the gateway callback (LiteLLM→Langfuse) is the practical feed. + +### E. Network/host passive attribution (no client cooperation) + +- **Pi-hole v6** (already on Dankclaw): query log API per device, + `pihole6_exporter` + query-logger→Loki (GPLv3 exporter). +- **Caddy access logs**: add a `log` directive (JSON) → Alloy→Loki; with the + auth chain it carries `X-Authentik-Username` for browser users. +- **OpenWebUI admin**: per-user daily token stats, events/logs, analytics API + (`/api/v1/analytics/*`) — per-USER identity exists here already (MIT). +- **auditd** connect-syscall rules (uid/pid/comm per connect to 7770/7771), + ship via audisp→Loki. +- **eBPF**: Tetragon (process+socket events, policies), Coroot, DeepFlow, + bpftrace `tracepoint:syscalls:sys_enter_connect` one-liners. K8s-shaped + for single hosts, but agents.plugin-style tools work. +- **nftables named counters** per consumer IP (bytes; reload resets). +- **netdata apps.plugin**: per-process network bytes, auto-discovered. +- **JA3/JA4 TLS fingerprinting**: client library only, TLS-only; the LAN is + plaintext HTTP today → use the HTTP **User-Agent census** instead (free + from any access log). +- **DHCP/ARP census** (leases + `ip neigh`): device roster. + +## Phased plan for this fleet + +**Phase 0 — repo-only, no new services** +1. frank-zealot: add `--metrics` (fixes the measured 501 + the + structurally-red Prometheus job + the firing `LlmEndpointDown`). +2. frank-dragoon: set `HALOGEN_API_KEY_FILE` (multi-key per consumer) + + mount `HALOGEN_RECORD_DIR` (start `HALOGEN_RECORD_TEXT=none`; flip to + `all` if prompt audit is wanted) + Alloy ship of `turns/`. +3. frank-zealot: `--api-key-file` (multi-key, verified in the deployed + build); radiance: probe for `--api-key` support (contract facts are + silent — add to `fleet-probes`). +4. models.yml: per-consumer bearer keys (`auth: none` → key) — the provider + schema supports apiKey (pi-era conversion note). +5. Caddy: `log` on `ai.franksdank.cloud` (OpenWebUI browser census, JSON → + Loki). + Limits: keys gate access, and only the engines' request records (halogen) + see them per-request; llama.cpp/radiance/vLLM metrics stay identity-blind. + +**Phase 1 — LiteLLM Proxy quadlet on Dankclaw = the front door.** +One virtual key per consumer (omp- ×N, openwebui, hermes, +agents, bench). Per-key/user/team Prometheus labels into the existing +Grafana+Alertmanager; Postgres spend ledger; budgets + rate limits per +key. This is the ONLY layer that yields tokens-per-who without engine +cooperation. (Bifrost is the lighter Apache-2.0 Go alternative; Helicone +the proxy-native-observability alternative.) + +**Phase 2 — prompt-level audit.** Langfuse self-hosted fed by the LiteLLM +callback: per-user traces, sessions, cost; masking + retention knobs. + +**Phase 3 — ambitious identity infra.** step-ca + Caddy mTLS per device; +Tailscale (+flow logs) for off-LAN; auditd/eBPF process attribution on the +agent hosts; Pi-hole query logger; nft counters per consumer IP. Browser +users stay owned by OpenWebUI analytics + its per-user API keys; a pipeline +can forward `X-Forwarded-User` so the gateway sees them too. + +## Cautions + +- Client-declared identity (keys, `user`, headers) groups traffic; it does + not authenticate humans. Authenticated identity = Authentik (browser) / + mTLS (device) / Tailscale (node). +- Plain-HTTP LAN = bearer keys sniffable by any tenant; treat keys as + identity labels on the LAN, not secrets, until TLS/mTLS at the door. +- Prompt-content logging stores a plaintext copy of everything — set + retention caps (halogen GIB/days; Langfuse masking; Loki retention) and + gitignore the record dirs. +- Household-scale cardinality is safe for per-key Prometheus labels; + the engines' refusal of key labels is a multi-tenant concern, not ours. + +## Related +- [[Fleet Audit and Posture]] — monitoring gaps census +- [Observability](../Infrastructure/Observability.md) — the Loki/Prometheus + pipes every sink above uses +- [LLM index](../LLM/index.md) — endpoint runbooks (env trios the Phase-0 + edits touch) diff --git a/serverAppConfigs-docs/Operations/index.md b/serverAppConfigs-docs/Operations/index.md index ec58fa1..d9ce3eb 100644 --- a/serverAppConfigs-docs/Operations/index.md +++ b/serverAppConfigs-docs/Operations/index.md @@ -23,6 +23,9 @@ Procedures for operating the fleet. The LLM-specific pages live under posture, whole-fleet audit, bulk redaction, stash audits - [[SSH and Remote Access]] — passwordless bootstrap without sshpass, fish login shells, GUI-hang diagnosis over ssh +- [[LLM Usage Attribution]] — who-is-using-the-fleet-LLMs research catalog: + engine-native identity, gateways, tracing, network/host attribution, and + the phased plan ## Related - [Observability Provisioning](../Infrastructure/Observability%20Provisioning.md) diff --git a/serverAppConfigs-docs/index.md b/serverAppConfigs-docs/index.md index c56d0f9..162fc49 100644 --- a/serverAppConfigs-docs/index.md +++ b/serverAppConfigs-docs/index.md @@ -134,6 +134,9 @@ skill-cluster. See [log.md](log.md) for the wave plan and results. posture, whole-fleet audit, bulk redaction - [[SSH and Remote Access]] — passwordless bootstrap, fish login shells, GUI-hang diagnosis over ssh +- [[LLM Usage Attribution]] — who-is-using-the-fleet-LLMs research catalog: + engine-native identity, gateways, tracing, network/host attribution, + phased plan **Architecture:** diff --git a/tests/frank_archon_unit.bats b/tests/frank_archon_unit.bats index 8ce2be3..f16ddc8 100755 --- a/tests/frank_archon_unit.bats +++ b/tests/frank_archon_unit.bats @@ -14,7 +14,7 @@ # half of RAM), the served window is what models.yml must # quote, the host-networking contract (no --publish), the image gate on a # name@sha256 ref, the .rad payload gate with its size floor, the -# chat-template override mount, the self-healing law (the run returns before + # chat-template override mount, the self-healing requirement (the run returns before # the engine is up, so the serve script must wait on the container for # Restart=always to mean anything), the deployer's false-green MainPID check, # the vendored hfget fetch path (no token for this ungated repo), and the @@ -136,10 +136,11 @@ nosudo_bin() { # Concurrency: 3 slots x the served window is the capacity contract, and # the fused TP all-reduce is declared over seqs x (1 + spec) = 9 rows, # under the 25-row bound at which the wht6 wire stays bit-exact. The - # "seqs 2 cannot boot" law belongs to the retired tcclaviger vLLM fork - # (torch.compile query_start_loc constraint, 2026-10-02); radiance booted - # seqs 2 in the fp8-native era and the 2026-10-06 sweep had seqs 2 and 3 - # tied inside the +-3% noise floor — slots buy concurrency, not speed. + # "seqs 2 cannot boot" is the failure mode of a runtime that compiles + # kernels specialized to the batch shape — it refuses a slot count it has + # never compiled for. This engine is shape-agnostic: seqs 2 boots, and + # measured decode ties seqs 2 and 3 inside the +-3% noise floor — slots + # buy concurrency, not speed. grep -q '^FRANK_ARCHON_MAX_SEQS=3$' "$ENV_FILE" grep -q '^FRANK_ARCHON_TP=2$' "$ENV_FILE" grep -q '^FRANK_ARCHON_TP_WIRE=wht6$' "$ENV_FILE" @@ -147,8 +148,8 @@ nosudo_bin() { # expert_tiered placement: vram_weights + the pinned host pool serve the # per-rank resident set TOGETHER, so cutting the VRAM slab to pay for the # KV pool moves the pool with it. KV is QUANTIZED fp8 (the engine takes - # only bf16|fp8 — no per-tensor K/V split), the PLE n-gram table is pinned - # in host RAM per Frank's law. + # only bf16|fp8 — no per-tensor K/V split), and the PLE n-gram table is + # pinned in host RAM so the cards read it without page-faulting mid-token. grep -q '^FRANK_ARCHON_PLACEMENT=expert_tiered$' "$ENV_FILE" grep -q '^FRANK_ARCHON_HOST_POOL_MIB=44928$' "$ENV_FILE" grep -q '^FRANK_ARCHON_VRAM_WEIGHTS_MIB=[1-9]' "$ENV_FILE" diff --git a/tests/frank_dragoon_unit.bats b/tests/frank_dragoon_unit.bats index 9ce038d..f0f892c 100755 --- a/tests/frank_dragoon_unit.bats +++ b/tests/frank_dragoon_unit.bats @@ -11,8 +11,9 @@ # vision file names AND byte counts inside the /models mount, and the # tokenizer's flat-file shape (a HF cache snapshot's symlinks dangle inside # the container). A serve gate and a deployer gate that disagree is the -# 2026-10-01 crash-loop; both sides are pinned here, plus the one engine -# law the card is emphatic about: HALOGEN_SPEC_ADAPT=0 — the default + # crash-loop class this file exists to kill: both sides are pinned here, + # plus the one engine rule the card is emphatic about: HALOGEN_SPEC_ADAPT=0 — + # the default # adaptive policy silently disables the MTP head per request and HALVES # decode. # @@ -111,9 +112,10 @@ good_env() { @test "the head stays on: adaptive spec must stay 0 and the window shape pinned" { # The card is emphatic: the default adaptive policy turns the MTP head # off for whole requests after low acceptance and HALVES decode. The - # two-slot law does NOT carry over from the 27B engine — on this line - # batching and speculation coexist (the card's quickstart ships 2 - # slots WITH the head), so 2 is concurrent capacity, not a tax. + # two-slot coupling of the 27B engine — which disabled drafting whenever a + # second slot existed — does NOT carry over: on this line batching and + # speculation coexist (the card's quickstart ships 2 slots WITH the head), + # so 2 is concurrent capacity, not a tax. grep -qE '^HALOGEN_SPEC_ADAPT=0$' "$ENV_FILE" grep -qE '^HALOGEN_KV_SLOTS=2$' "$ENV_FILE" # The cap/timeout pair is coupled (FLAGS.md table): a request at the @@ -178,8 +180,9 @@ good_env() { } @test "the serve script refuses a checkpoint of the wrong byte count" { - # A truncated curl exit-0 artifact must not boot; the Hub's byte - # count is the contract (fleet fetch law). + # A truncated curl exit-0 artifact must not boot: curl exits 0 on an HTTP + # error and writes the error body as the file, so the Hub's declared byte + # count is the contract that catches it. podman_stub good_env run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ @@ -280,7 +283,7 @@ $(cat "$TEST_TMP/serve.log")" ! grep -q 'PODMAN wait' "$TEST_TMP/serve.log" } -@test "the shipped env keeps the harness fit law inside the ceiling" { + @test "the shipped env keeps the harness fit budget inside the engine ceiling" { # 262144 (the checkpoint's native window) is the per-slot hard ceiling # (a longer prompt is a 400 naming the limit, never a silent # truncation). The harness clamps max_tokens = min(45056, 217088 − est