chore: stop tracking .AGENTTRASH — the purger deletes tracked files, dirtying every host
The deletion policy moves in-repo deletions into .AGENTTRASH/ beside the item, and .gitignore line 25 already declares it ignored. But 37 retired payloads were committed before that rule existed, and gitignore never applies to tracked paths. Consequence measured 2026-10-31: Dankserver's deploy checkout reported 37 tracked deletions — exactly every tracked trash file — because the approved purger had done its job. Any host that purges is permanently dirty, and a future pull that touches one of those paths aborts mid-deploy with 'local changes would be overwritten' while the tree is in fact clean of real edits. Untrack the four dirs; the bytes stay on disk and in history (they always were recoverable from git, which is what the trash policy's 'temporary hold, not an archive' means). tests/deployment_scoping.bats asserts the staged chat template exists on disk — it still does, so the seam is untouched.
This commit is contained in:
dank committed
1 parent
6a4291f224
commit
8ae4d1c710
37 files changed
-4680
No files matched your search
@@ -1,156 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""HTTP front door for the frank-medium single-GPU replicas.
|
||||
|
||||
Why two replicas instead of TP=2: one vLLM process spanning both R9700s needs
|
||||
RCCL collectives, and RCCL's kernels require PCIe atomics on every
|
||||
participating GPU. The second card sits behind chipset bridges advertising
|
||||
`AtomicOpsCap: Routing-`, which no software can enable (see the runbook), so
|
||||
no collective can launch on it. Two single-GPU replicas use both cards; this
|
||||
service keeps the fleet contract of one endpoint on :7770, streaming included.
|
||||
|
||||
Routing decisions live in routing.py — spread single-turn requests, pin
|
||||
continuations to the replica whose prefix cache holds their context. This file
|
||||
only moves bytes: parse enough of the request to route it, forward it verbatim,
|
||||
stream the answer back, and report replica health on /health.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from contextlib import suppress
|
||||
from aiohttp import ClientSession, ClientTimeout, web
|
||||
|
||||
import routing
|
||||
|
||||
PORT = int(os.environ.get("PORT", "7770"))
|
||||
HEALTH_TTL = float(os.environ.get("HEALTH_TTL", "10"))
|
||||
# Hop-by-hop headers must not be forwarded (RFC 9110 §7.6.1) — forwarding
|
||||
# transfer-encoding or content-length of a rewritten body breaks clients.
|
||||
HOP_BY_HOP = {
|
||||
"host", "content-length", "transfer-encoding", "connection",
|
||||
"keep-alive", "proxy-authenticate", "proxy-authorization", "te",
|
||||
"trailers", "upgrade",
|
||||
}
|
||||
|
||||
UPSTREAMS = routing.UPSTREAMS
|
||||
# Replica health, refreshed by the watcher. Module state rather than aiohttp app
|
||||
# state: the application is frozen once the site is up.
|
||||
_UP: list[bool] = [True] * len(UPSTREAMS)
|
||||
|
||||
|
||||
async def health(session: ClientSession) -> web.Response:
|
||||
results = await asyncio.gather(
|
||||
*(session.get(u + "/health", timeout=ClientTimeout(total=5)) for u in UPSTREAMS),
|
||||
return_exceptions=True,
|
||||
)
|
||||
lines, up = [], 0
|
||||
for url, res in zip(UPSTREAMS, results):
|
||||
if isinstance(res, Exception) or res.status != 200:
|
||||
lines.append(f"{url} DOWN")
|
||||
else:
|
||||
up += 1
|
||||
lines.append(f"{url} UP")
|
||||
return web.Response(status=200 if up else 503, text="\n".join(lines) + "\n")
|
||||
|
||||
|
||||
async def watch_health() -> None:
|
||||
"""Refresh the replica up/down table so routing never targets a dead one.
|
||||
|
||||
One HTTP request per replica per interval, in exchange for never handing a
|
||||
client a 502 after a restart.
|
||||
"""
|
||||
global _UP
|
||||
session = ClientSession()
|
||||
try:
|
||||
while True:
|
||||
res = await asyncio.gather(
|
||||
*(session.get(u + "/health", timeout=ClientTimeout(total=5)) for u in UPSTREAMS),
|
||||
return_exceptions=True,
|
||||
)
|
||||
_UP = [not isinstance(r, Exception) and r.status == 200 for r in res]
|
||||
await asyncio.sleep(HEALTH_TTL)
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def proxy(request: web.Request) -> web.StreamResponse:
|
||||
session: ClientSession = request.app["session"]
|
||||
if request.path == "/health":
|
||||
return await health(session)
|
||||
|
||||
body = await request.read()
|
||||
headers = {k: v for k, v in request.headers.items() if k.lower() not in HOP_BY_HOP}
|
||||
|
||||
# Fail over before anything is written to the client. A replica that
|
||||
# restarts is invisible to the health watcher for up to HEALTH_TTL, and a
|
||||
# deploy restarts both replicas — without this retry a 10 s window of
|
||||
# requests would 502 on a service that has a live card the whole time.
|
||||
# vLLM's OpenAI endpoints are stateless per request, so re-sending a
|
||||
# request that never reached an engine is safe; a request that failed
|
||||
# mid-stream is not retried (that path is below).
|
||||
tried: set[int] = set()
|
||||
upstream = None
|
||||
last_error = "no healthy frank-medium replica"
|
||||
while len(tried) < len(UPSTREAMS):
|
||||
idx = routing.route(body, request.remote, _UP)
|
||||
if idx < 0 or idx in tried:
|
||||
break
|
||||
tried.add(idx)
|
||||
target = UPSTREAMS[idx] + request.path_qs
|
||||
try:
|
||||
upstream = await session.request(
|
||||
request.method, target, headers=headers, data=body or None,
|
||||
timeout=ClientTimeout(total=None, sock_connect=10, sock_read=3600),
|
||||
)
|
||||
break
|
||||
except Exception as exc:
|
||||
# Demote at once: the watcher will confirm, but every request until
|
||||
# then should skip this replica rather than pay the same timeout.
|
||||
# Logged because a silent failover hides a card that is down.
|
||||
_UP[idx] = False
|
||||
last_error = f"{target} unreachable: {exc}"
|
||||
print(f"proxy: failover — {target} unreachable ({exc}); retrying "
|
||||
f"another replica", flush=True)
|
||||
if upstream is None:
|
||||
# 503 when no replica was even reachable through the routing table,
|
||||
# 502 when replicas were tried and all of them failed.
|
||||
status = 502 if tried else 503
|
||||
return web.Response(status=status, text=f"proxy: {last_error}\n")
|
||||
|
||||
out = web.StreamResponse(status=upstream.status, reason=upstream.reason)
|
||||
for k, v in upstream.headers.items():
|
||||
if k.lower() not in HOP_BY_HOP:
|
||||
out.headers[k] = v
|
||||
await out.prepare(request)
|
||||
try:
|
||||
async for chunk in upstream.content.iter_any():
|
||||
await out.write(chunk)
|
||||
except Exception:
|
||||
# The client went away mid-stream (an aborted generation). Nothing to
|
||||
# report: the engine sees its own cancellation.
|
||||
pass
|
||||
finally:
|
||||
upstream.release()
|
||||
with suppress(Exception):
|
||||
await out.write_eof()
|
||||
return out
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
app = web.Application(client_max_size=1024 ** 3)
|
||||
app["session"] = ClientSession()
|
||||
app.router.add_route("*", "/{tail:.*}", proxy)
|
||||
runner = web.AppRunner(app)
|
||||
await runner.setup()
|
||||
await web.TCPSite(runner, "0.0.0.0", PORT).start()
|
||||
print(f"proxy: :{PORT} -> {', '.join(UPSTREAMS)} routing={routing.ROUTING} "
|
||||
f"key={routing.AFFINITY_BYTES}B known_max={routing.KNOWN_MAX}", flush=True)
|
||||
watcher = asyncio.create_task(watch_health())
|
||||
with suppress(asyncio.CancelledError):
|
||||
await asyncio.gather(watcher)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with suppress(KeyboardInterrupt):
|
||||
asyncio.run(main())
|
||||
@@ -1,98 +0,0 @@
|
||||
# Quadlet unit: frank-medium replica A (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankserver: ~/.config/containers/systemd/frank-medium-a.container
|
||||
# Unit name: frank-medium-a.service (container name: frank-medium-a)
|
||||
#
|
||||
# WHY TWO REPLICAS INSTEAD OF TP=2. The intended shape for frank-medium is one
|
||||
# tensor-parallel process across both R9700s (see frank-medium.container, kept
|
||||
# as the target). That shape cannot run on this host: RCCL's collective kernels
|
||||
# carry a hidden hostcall-buffer parameter, and the ROCr runtime refuses to
|
||||
# dispatch them on a device without PCIe atomics ("Pcie atomics not enabled,
|
||||
# hostcall not supported" -> AQL dispatch failed -> hipErrorIllegalState). The
|
||||
# second R9700 sits behind the chipset bridges 05:00.0/06:09.0, which advertise
|
||||
# `AtomicOpsCap: Routing-` — a read-only capability, so no software can enable
|
||||
# atomics on that path; the amdgpu driver's pci_enable_atomic_ops_to_root()
|
||||
# therefore fails for that GPU and KFD marks the CPU<->GPU2 link NO_ATOMICS.
|
||||
# The first R9700 sits on CPU root port 00:01.1 (Routing+) and does have
|
||||
# atomics. Result: every collective that touches GPU 2 dies at launch.
|
||||
# Two independent single-GPU replicas use both cards instead; the proxy unit
|
||||
# keeps the single :7770 endpoint. Full evidence: the frank-medium runbook.
|
||||
#
|
||||
# SHAPE. vLLM binds 0.0.0.0:7771 on the host network directly; no PublishPort.
|
||||
# MAXLEN is 114688, not the TP=2 deployment's 262144: one card carries the
|
||||
# whole model (~20 GiB of weights + drafter), leaving ~5.2 GiB of KV, and
|
||||
# 131072 does not fit (measured: needs 5.58 GiB). KV_MEM=0 lets vLLM profile
|
||||
# this shape instead of using the 2x7551-32624 pin, which is only valid at
|
||||
# TP=2/262144.
|
||||
#
|
||||
# DEVICES. /dev/kfd + /dev/dri; HIP_VISIBLE_DEVICES=0 (the first R9700). The
|
||||
# 2 GiB Raphael iGPU enumerates as index 2 and is excluded by the pin.
|
||||
# Only HIP_VISIBLE_DEVICES is set, never ROCR_VISIBLE_DEVICES: ROCR filters
|
||||
# first and HIP indexes into the filtered list, so a single-valued pair
|
||||
# (ROCR=1,HIP=1) leaves an empty device set.
|
||||
#
|
||||
# DO NOT name an environment variable TEMP, HOST or GPU_MAX_HEAP_SIZE: the
|
||||
# HIP runtime reads bare TEMP for its JIT cache path (2026-08-13 incident).
|
||||
[Unit]
|
||||
Description=frank-medium replica A — vllm-radiance MXFP4 Qwen3.8-27B on R9700 #1
|
||||
After=network-online.target
|
||||
|
||||
[Container]
|
||||
ContainerName=frank-medium-a
|
||||
Image=localhost/radiance-vllm:0.9.3
|
||||
AddDevice=/dev/kfd
|
||||
AddDevice=/dev/dri
|
||||
GroupAdd=keep-groups
|
||||
# The upstream launcher's exact podman posture (rootless).
|
||||
PodmanArgs=--privileged --ipc=host --security-opt seccomp=unconfined --cap-add SYS_PTRACE --network=host
|
||||
Environment=HIP_VISIBLE_DEVICES=0
|
||||
Environment=SNAP=/models/Qwen3.8-27B-Uncensored-MXFP4-awq
|
||||
Environment=DRAFTER=/models/Qwen3.8-27B-DFlash2-FP8
|
||||
Environment=CHAT_TEMPLATE=/patches/qwen-fixed-v22.3.jinja
|
||||
Environment=SERVED_NAMES=frank-medium
|
||||
Environment=PORT=7771
|
||||
Environment=TP=1
|
||||
Environment=MAXSEQS=4
|
||||
Environment=CHUNK=8192
|
||||
Environment=MAXLEN=114688
|
||||
Environment=GPU_UTIL=0.98
|
||||
Environment=KV_MEM=0
|
||||
Environment=SPEC_METHOD=dflash
|
||||
Environment=SPEC=7
|
||||
# Checkpoints (rw: nothing downloads at runtime, but vLLM mmaps the shards).
|
||||
Volume=/home/dank/models:/models
|
||||
# Tokenizer files ride the HF cache; HF_HUB_OFFLINE=1 is set in the entrypoint.
|
||||
Volume=/home/dank/.cache/huggingface:/root/.cache/huggingface
|
||||
# Compile cache. Shared with the other replica and with the TP=2 target: the
|
||||
# entries are content-addressed, so shapes simply add subdirectories.
|
||||
Volume=/home/dank/.radiance-cache-w4a8-093-gdnm-nqft-fp8s-gnq:/cache
|
||||
# The pinned radiance clone: patches applied at every container start.
|
||||
Volume=/home/dank/GitPackages/radiance-vllm-mxfp4:/patches:z
|
||||
# The patched libr4d (pin b9e42ab + repo extras, key b9e42ab-rx6). The image's
|
||||
# shipped r4d.so predates the GDN overflow fix and NaNs this model.
|
||||
Volume=/home/dank/.cache/radiance-libr4d/b9e42ab-rx6:/r4d:z
|
||||
# Entrypoint + healthcheck from this checkout, mounted read-only.
|
||||
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/radiance-entrypoint.sh:/usr/local/bin/frank-medium-entrypoint.sh:ro
|
||||
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/frank-medium-healthcheck.py:/usr/local/bin/frank-medium-healthcheck.py:ro
|
||||
# The entrypoint must be the container command; the image's own entrypoint
|
||||
# (which the patched script execs at the end) must not run unpatched.
|
||||
PodmanArgs=--entrypoint /usr/local/bin/frank-medium-entrypoint.sh
|
||||
# 200 only once the engine serves; during first-start compilation the
|
||||
# container is legitimately unhealthy. HealthCmd is split on spaces into an
|
||||
# argv array by the podman 5.x v1 generator, hence the file-based probe.
|
||||
HealthCmd=python3 /usr/local/bin/frank-medium-healthcheck.py
|
||||
HealthInterval=60s
|
||||
HealthTimeout=30s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=1800s
|
||||
Pull=never
|
||||
|
||||
[Service]
|
||||
# First start compiles Triton/inductor kernels (several minutes) before the
|
||||
# engine loads ~20 GiB of weights.
|
||||
TimeoutStartSec=1800
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,98 +0,0 @@
|
||||
# Quadlet unit: frank-medium replica B (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankserver: ~/.config/containers/systemd/frank-medium-b.container
|
||||
# Unit name: frank-medium-b.service (container name: frank-medium-a)
|
||||
#
|
||||
# WHY TWO REPLICAS INSTEAD OF TP=2. The intended shape for frank-medium is one
|
||||
# tensor-parallel process across both R9700s (see frank-medium.container, kept
|
||||
# as the target). That shape cannot run on this host: RCCL's collective kernels
|
||||
# carry a hidden hostcall-buffer parameter, and the ROCr runtime refuses to
|
||||
# dispatch them on a device without PCIe atomics ("Pcie atomics not enabled,
|
||||
# hostcall not supported" -> AQL dispatch failed -> hipErrorIllegalState). The
|
||||
# second R9700 sits behind the chipset bridges 05:00.0/06:09.0, which advertise
|
||||
# `AtomicOpsCap: Routing-` — a read-only capability, so no software can enable
|
||||
# atomics on that path; the amdgpu driver's pci_enable_atomic_ops_to_root()
|
||||
# therefore fails for that GPU and KFD marks the CPU<->GPU2 link NO_ATOMICS.
|
||||
# The first R9700 sits on CPU root port 00:01.1 (Routing+) and does have
|
||||
# atomics. Result: every collective that touches GPU 2 dies at launch.
|
||||
# Two independent single-GPU replicas use both cards instead; the proxy unit
|
||||
# keeps the single :7770 endpoint. Full evidence: the frank-medium runbook.
|
||||
#
|
||||
# SHAPE. vLLM binds 0.0.0.0:7772 on the host network directly; no PublishPort.
|
||||
# MAXLEN is 114688, not the TP=2 deployment's 262144: one card carries the
|
||||
# whole model (~20 GiB of weights + drafter), leaving ~5.2 GiB of KV, and
|
||||
# 131072 does not fit (measured: needs 5.58 GiB). KV_MEM=0 lets vLLM profile
|
||||
# this shape instead of using the 2x7551-32624 pin, which is only valid at
|
||||
# TP=2/262144.
|
||||
#
|
||||
# DEVICES. /dev/kfd + /dev/dri; HIP_VISIBLE_DEVICES=1 (the second R9700). The
|
||||
# 2 GiB Raphael iGPU enumerates as index 2 and is excluded by the pin.
|
||||
# Only HIP_VISIBLE_DEVICES is set, never ROCR_VISIBLE_DEVICES: ROCR filters
|
||||
# first and HIP indexes into the filtered list, so a single-valued pair
|
||||
# (ROCR=1,HIP=1) leaves an empty device set.
|
||||
#
|
||||
# DO NOT name an environment variable TEMP, HOST or GPU_MAX_HEAP_SIZE: the
|
||||
# HIP runtime reads bare TEMP for its JIT cache path (2026-08-13 incident).
|
||||
[Unit]
|
||||
Description=frank-medium replica B — vllm-radiance MXFP4 Qwen3.8-27B on R9700 #2
|
||||
After=network-online.target
|
||||
|
||||
[Container]
|
||||
ContainerName=frank-medium-b
|
||||
Image=localhost/radiance-vllm:0.9.3
|
||||
AddDevice=/dev/kfd
|
||||
AddDevice=/dev/dri
|
||||
GroupAdd=keep-groups
|
||||
# The upstream launcher's exact podman posture (rootless).
|
||||
PodmanArgs=--privileged --ipc=host --security-opt seccomp=unconfined --cap-add SYS_PTRACE --network=host
|
||||
Environment=HIP_VISIBLE_DEVICES=1
|
||||
Environment=SNAP=/models/Qwen3.8-27B-Uncensored-MXFP4-awq
|
||||
Environment=DRAFTER=/models/Qwen3.8-27B-DFlash2-FP8
|
||||
Environment=CHAT_TEMPLATE=/patches/qwen-fixed-v22.3.jinja
|
||||
Environment=SERVED_NAMES=frank-medium
|
||||
Environment=PORT=7772
|
||||
Environment=TP=1
|
||||
Environment=MAXSEQS=4
|
||||
Environment=CHUNK=8192
|
||||
Environment=MAXLEN=114688
|
||||
Environment=GPU_UTIL=0.98
|
||||
Environment=KV_MEM=0
|
||||
Environment=SPEC_METHOD=dflash
|
||||
Environment=SPEC=7
|
||||
# Checkpoints (rw: nothing downloads at runtime, but vLLM mmaps the shards).
|
||||
Volume=/home/dank/models:/models
|
||||
# Tokenizer files ride the HF cache; HF_HUB_OFFLINE=1 is set in the entrypoint.
|
||||
Volume=/home/dank/.cache/huggingface:/root/.cache/huggingface
|
||||
# Compile cache. Shared with the other replica and with the TP=2 target: the
|
||||
# entries are content-addressed, so shapes simply add subdirectories.
|
||||
Volume=/home/dank/.radiance-cache-w4a8-093-gdnm-nqft-fp8s-gnq:/cache
|
||||
# The pinned radiance clone: patches applied at every container start.
|
||||
Volume=/home/dank/GitPackages/radiance-vllm-mxfp4:/patches:z
|
||||
# The patched libr4d (pin b9e42ab + repo extras, key b9e42ab-rx6). The image's
|
||||
# shipped r4d.so predates the GDN overflow fix and NaNs this model.
|
||||
Volume=/home/dank/.cache/radiance-libr4d/b9e42ab-rx6:/r4d:z
|
||||
# Entrypoint + healthcheck from this checkout, mounted read-only.
|
||||
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/radiance-entrypoint.sh:/usr/local/bin/frank-medium-entrypoint.sh:ro
|
||||
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/frank-medium-healthcheck.py:/usr/local/bin/frank-medium-healthcheck.py:ro
|
||||
# The entrypoint must be the container command; the image's own entrypoint
|
||||
# (which the patched script execs at the end) must not run unpatched.
|
||||
PodmanArgs=--entrypoint /usr/local/bin/frank-medium-entrypoint.sh
|
||||
# 200 only once the engine serves; during first-start compilation the
|
||||
# container is legitimately unhealthy. HealthCmd is split on spaces into an
|
||||
# argv array by the podman 5.x v1 generator, hence the file-based probe.
|
||||
HealthCmd=python3 /usr/local/bin/frank-medium-healthcheck.py
|
||||
HealthInterval=60s
|
||||
HealthTimeout=30s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=1800s
|
||||
Pull=never
|
||||
|
||||
[Service]
|
||||
# First start compiles Triton/inductor kernels (several minutes) before the
|
||||
# engine loads ~20 GiB of weights.
|
||||
TimeoutStartSec=1800
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,61 +0,0 @@
|
||||
# Quadlet unit: frank-medium proxy (rootless, boot-persistent).
|
||||
#
|
||||
# Install path on dankserver: ~/.config/containers/systemd/frank-medium-proxy.container
|
||||
# Unit name: frank-medium-proxy.service (container name: frank-medium-proxy)
|
||||
#
|
||||
# WHAT IT IS. A prefix-affinity reverse proxy in front of the two single-GPU
|
||||
# frank-medium replicas (:7771 on R9700 #1, :7772 on R9700 #2). It preserves
|
||||
# the fleet contract: one endpoint on host port 7770 answering as the served
|
||||
# model name frank-medium, OpenAI /v1, streaming included.
|
||||
#
|
||||
# WHY IT EXISTS. TP=2 — one vLLM process spanning both cards — is impossible
|
||||
# on this host (see frank-medium-a.container and the runbook: RCCL's kernels
|
||||
# need PCIe atomics, and the second card's chipset path advertises
|
||||
# AtomicOpsCap: Routing-). Two replicas are how both cards get used.
|
||||
#
|
||||
# ROUTING. Requests are keyed on a hash of their leading 4 KiB of prompt text,
|
||||
# so a conversation family — an agent loop re-sending the same system prompt,
|
||||
# tool list and file context every turn — keeps landing on the same replica,
|
||||
# where --enable-prefix-caching has that prefix warm. Plain round-robin threw
|
||||
# half of those cache hits away. A replica that is down is skipped, and
|
||||
# ROUTING=roundrobin restores the old behaviour for A/B measurement. /health
|
||||
# answers 200 while at least one replica is up and names the ones that are
|
||||
# down. The proxy runs on host networking so it can bind 7770 and reach the
|
||||
# replicas on loopback.
|
||||
#
|
||||
# The script is a few dozen lines of aiohttp, mounted read-only out of
|
||||
# /home/dank/serverAppConfigs (deploy.sh pulls that checkout before dispatch).
|
||||
# It reuses the same image as the replicas: aiohttp is already there and no
|
||||
# extra image has to be pulled on this host.
|
||||
[Unit]
|
||||
Description=frank-medium proxy — prefix-affinity in front of the two single-GPU replicas
|
||||
After=network-online.target
|
||||
Wants=frank-medium-a.service frank-medium-b.service
|
||||
After=frank-medium-a.service frank-medium-b.service
|
||||
|
||||
[Container]
|
||||
ContainerName=frank-medium-proxy
|
||||
Image=localhost/radiance-vllm:0.9.3
|
||||
PodmanArgs=--network=host
|
||||
Environment=PORT=7770
|
||||
Environment=UPSTREAMS=http://127.0.0.1:7771,http://127.0.0.1:7772
|
||||
# The proxy is two files: routing.py holds the replica-selection policy, the
|
||||
# server holds the HTTP plumbing. Both are mounted because the server imports
|
||||
# the policy by name from its own directory (/usr/local/bin is sys.path[0]).
|
||||
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/proxy/routing.py:/usr/local/bin/routing.py:ro
|
||||
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/proxy/affinity-proxy.py:/usr/local/bin/affinity-proxy.py:ro
|
||||
Volume=/home/dank/serverAppConfigs/dankserver/frank-medium/frank-medium-healthcheck.py:/usr/local/bin/frank-medium-healthcheck.py:ro
|
||||
PodmanArgs=--entrypoint /usr/local/bin/affinity-proxy.py
|
||||
HealthCmd=python3 /usr/local/bin/frank-medium-healthcheck.py
|
||||
HealthInterval=60s
|
||||
HealthTimeout=30s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=60s
|
||||
Pull=never
|
||||
|
||||
[Service]
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -1,129 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Routing policy for the frank-medium proxy: which replica serves a request.
|
||||
|
||||
Kept apart from the HTTP plumbing in affinity-proxy.py so the decisions can be
|
||||
tested without aiohttp, a network or a GPU — the first version of these rules
|
||||
shipped with a bug that only appeared on a continuation request, and a test
|
||||
that loads this module catches that class of mistake directly.
|
||||
|
||||
Two cards, two behaviours, and the workload decides which is right:
|
||||
|
||||
* A **single-turn** request is one member of a batch (a benchmark run, a burst
|
||||
of parallel subagents). Spreading those over both replicas lets both cards
|
||||
prefill at once; measured on this host, a 16-request burst of 15k-token
|
||||
prompts finishes in 18.6 s spread versus 30.1 s pinned.
|
||||
* A **continuation** re-sends a prefix the replica has already prefilled. Pinned
|
||||
to that replica the turn costs a cache lookup; spread, half the turns pay a
|
||||
cold prefill again (measured: 1096 ms versus 2676 ms mean warm-turn TTFT on
|
||||
90k-token contexts).
|
||||
|
||||
ROUTING picks the strategy; the default `adaptive` applies both rules above.
|
||||
`affinity` and `roundrobin` exist so the choice can be re-measured, and are not
|
||||
production settings.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
|
||||
UPSTREAMS = [u.strip() for u in os.environ.get(
|
||||
"UPSTREAMS", "http://127.0.0.1:7771,http://127.0.0.1:7772").split(",") if u.strip()]
|
||||
ROUTING = os.environ.get("ROUTING", "adaptive").strip().lower()
|
||||
# Leading bytes of prompt text that identify a conversation's shared prefix.
|
||||
# 4 KiB covers a system prompt plus the start of the first user turn, which is
|
||||
# where agent loops keep the text they resend every turn.
|
||||
AFFINITY_BYTES = int(os.environ.get("AFFINITY_BYTES", "4096"))
|
||||
# Prefixes remembered for pinning. Bounded so a long-lived proxy cannot grow
|
||||
# without limit; 4096 conversations is far beyond what this fleet runs at once.
|
||||
KNOWN_MAX = int(os.environ.get("KNOWN_MAX", "4096"))
|
||||
|
||||
_counter = itertools.count()
|
||||
# digest -> replica index, insertion-ordered for O(1) LRU eviction. A prefix is
|
||||
# remembered as soon as any request carries it; only a continuation is pinned
|
||||
# there, because only a continuation can profit from the cache entry.
|
||||
_known: "collections.OrderedDict[bytes, int]" = collections.OrderedDict()
|
||||
|
||||
|
||||
def affinity_key(body: bytes, remote: str | None) -> tuple[bytes, bool]:
|
||||
"""Key a request by its shared prefix, and report whether it continues one.
|
||||
|
||||
Chat and completions bodies both carry their prompt text in JSON, so the
|
||||
leading bytes of that text are the prefix; a body we cannot parse (or an
|
||||
endpoint with no prompt, e.g. /v1/models) falls back to the client address,
|
||||
which still beats nothing for a single-client loop.
|
||||
"""
|
||||
fallback = (remote or "anonymous").encode()
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
except Exception:
|
||||
return fallback, False
|
||||
|
||||
text = ""
|
||||
continuation = False
|
||||
if isinstance(payload, dict):
|
||||
if isinstance(payload.get("prompt"), str):
|
||||
text = payload["prompt"]
|
||||
elif isinstance(payload.get("messages"), list):
|
||||
continuation = len(payload["messages"]) > 1
|
||||
parts = []
|
||||
for msg in payload["messages"]:
|
||||
if isinstance(msg, dict):
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str):
|
||||
parts.append(content)
|
||||
elif isinstance(content, list): # multimodal content blocks
|
||||
parts.extend(
|
||||
blk.get("text", "") for blk in content
|
||||
if isinstance(blk, dict) and isinstance(blk.get("text"), str)
|
||||
)
|
||||
text = "\n".join(parts)
|
||||
if not text:
|
||||
return fallback, continuation
|
||||
return text[:AFFINITY_BYTES].encode("utf-8", "replace"), continuation
|
||||
|
||||
|
||||
def pick(key: bytes, continuation: bool, up: list[bool]) -> int:
|
||||
"""Replica for a request, or -1 when none is available.
|
||||
|
||||
Down replicas are never selected — a rollout must not send traffic to an
|
||||
engine that is still compiling its graphs — and a pin whose replica is down
|
||||
is re-established on a live one rather than kept pointing at a corpse.
|
||||
"""
|
||||
digest = hashlib.sha1(key).digest()
|
||||
|
||||
if ROUTING == "roundrobin":
|
||||
start = next(_counter) % len(UPSTREAMS)
|
||||
elif ROUTING == "adaptive":
|
||||
entry = _known.get(digest)
|
||||
if continuation and entry is not None and up[entry]:
|
||||
_known.move_to_end(digest)
|
||||
return entry
|
||||
# Anything that is not a continuation spreads by counter — including a
|
||||
# batch of requests that happen to share one prefix. Hashing the prefix
|
||||
# would send every member of such a batch to whichever card the hash
|
||||
# names (measured: 29.8 s for a 16-request burst, against 18.6 s when
|
||||
# spread). The counter is only advanced by these requests, so batch
|
||||
# members alternate across the cards.
|
||||
start = next(_counter) % len(UPSTREAMS)
|
||||
else: # affinity
|
||||
start = int.from_bytes(digest[:8], "big") % len(UPSTREAMS)
|
||||
|
||||
for offset in range(len(UPSTREAMS)):
|
||||
idx = (start + offset) % len(UPSTREAMS)
|
||||
if up[idx]:
|
||||
if ROUTING == "adaptive":
|
||||
_known[digest] = idx
|
||||
_known.move_to_end(digest)
|
||||
if len(_known) > KNOWN_MAX:
|
||||
_known.popitem(last=False)
|
||||
return idx
|
||||
return -1
|
||||
|
||||
|
||||
def route(body: bytes, remote: str | None, up: list[bool]) -> int:
|
||||
"""Convenience wrapper: decode the request body and choose a replica."""
|
||||
return pick(*affinity_key(body, remote), up)
|
||||
Reference in new issue
Block a user