Frank
4fca739bf8
The radiance MXFP4 stack cannot run TP=2 on this host. RCCL's collective
kernels carry a hidden hostcall-buffer parameter, and ROCr refuses to
dispatch them on a device without PCIe atomics: the AQL dispatch fails with
hipErrorIllegalState ("the operation cannot be performed in the present
state"), which is what every two-rank collective hit. The second R9700 sits
behind the chipset bridges 05:00.0/06:09.0, which advertise
`AtomicOpsCap: Routing-` — a read-only capability, so no software can enable
atomic routing on that path and the driver's
pci_enable_atomic_ops_to_root() fails for that card. The first card, on CPU
root port 00:01.1, has atomics enabled.
Ruled out: kernel version (7.0.0-28/-30/-31 identical, after a full upgrade),
iommu=pt, P2P hardware (45.3 GB/s between the cards), the shipped librccl (a
from-source rebuild reproduces the failure at the same call path), vLLM and
pynccl (a bare two-rank torch.distributed allreduce fails the same way), all
NCCL transport knobs, and single-rank tests (NCCL short-circuits at
world_size=1, so one-card success proves nothing).
Deployment: three rootless Quadlets — frank-medium-a (:7771, card 1),
frank-medium-b (:7772, card 2), both TP=1 at 114688 context (262144 and
131072 do not fit one card's KV) with vLLM-profiled KV — and
frank-medium-proxy (:7770), a round-robin that preserves the fleet contract
of one endpoint and one served model name. frank-medium.container stays as
the TP=2 target for a host whose second card is atomic-capable; the deploy
handler does not install it and trashes any stale copy.
Measured through the proxy (vllm bench serve, random dataset, --ignore-eos):
conc 1 42.8 tok/s, conc 4 147.6 tok/s, conc 8 230.8 tok/s, 32k prefill at
~2.8k tok/s per card, TPOT p50 23.6-28.8 ms.
Also: healthcheck reads PORT so one probe serves all three units, the hermes
example and the harness models.yml drop to the real 114688 context, and the
runbook records the full evidence chain.
97 lines
3.4 KiB
Python
Executable File
97 lines
3.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Round-robin proxy in front of the frank-medium TP=1 replicas.
|
|
|
|
Why this exists: TP=2 (one vLLM process spanning both R9700s) needs RCCL
|
|
collectives, and RCCL's kernels require PCIe atomics on every participating
|
|
GPU. On this host the second R9700 sits behind chipset bridges that advertise
|
|
`AtomicOpsCap: Routing-` (see the runbook), so no collective can launch on it
|
|
and TP=2 is impossible. Two single-GPU replicas use both cards instead; this
|
|
proxy keeps the fleet contract of one endpoint on :7770.
|
|
|
|
The vLLM OpenAI API is stateless per request, so round-robin is sound. /health
|
|
returns 200 while at least one replica is up and names the ones that are not.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import itertools
|
|
import os
|
|
from aiohttp import ClientSession, ClientTimeout, web
|
|
|
|
UPSTREAMS = [u.strip() for u in os.environ.get(
|
|
"UPSTREAMS", "http://127.0.0.1:7771,http://127.0.0.1:7772").split(",") if u.strip()]
|
|
PORT = int(os.environ.get("PORT", "7770"))
|
|
# Hop-by-hop headers must not be forwarded (RFC 9110 §7.6.1) — forwarding
|
|
# transfer-encoding or content-length of a rewritten body breaks clients.
|
|
HOP_BY_HOP = {
|
|
"host", "content-length", "transfer-encoding", "connection",
|
|
"keep-alive", "proxy-authenticate", "proxy-authorization", "te",
|
|
"trailers", "upgrade",
|
|
}
|
|
|
|
_cycle = itertools.cycle(range(len(UPSTREAMS)))
|
|
|
|
|
|
async def health(session: ClientSession) -> web.Response:
|
|
results = await asyncio.gather(
|
|
*(session.get(u + "/health", timeout=ClientTimeout(total=5)) for u in UPSTREAMS),
|
|
return_exceptions=True,
|
|
)
|
|
lines, up = [], 0
|
|
for url, res in zip(UPSTREAMS, results):
|
|
if isinstance(res, Exception) or res.status != 200:
|
|
lines.append(f"{url} DOWN")
|
|
else:
|
|
up += 1
|
|
lines.append(f"{url} UP")
|
|
body = "\n".join(lines) + "\n"
|
|
return web.Response(status=200 if up else 503, text=body)
|
|
|
|
|
|
async def proxy(request: web.Request) -> web.StreamResponse:
|
|
session: ClientSession = request.app["session"]
|
|
if request.path == "/health":
|
|
return await health(session)
|
|
|
|
target = UPSTREAMS[next(_cycle)] + request.path_qs
|
|
headers = {k: v for k, v in request.headers.items() if k.lower() not in HOP_BY_HOP}
|
|
body = await request.read()
|
|
|
|
try:
|
|
upstream = await session.request(
|
|
request.method, target, headers=headers, data=body or None,
|
|
timeout=ClientTimeout(total=None, sock_connect=10, sock_read=3600),
|
|
)
|
|
except Exception as exc: # upstream refused/unreachable
|
|
return web.Response(status=502, text=f"proxy: {target} unreachable: {exc}\n")
|
|
|
|
out = web.StreamResponse(status=upstream.status, reason=upstream.reason)
|
|
for k, v in upstream.headers.items():
|
|
if k.lower() not in HOP_BY_HOP:
|
|
out.headers[k] = v
|
|
await out.prepare(request)
|
|
try:
|
|
async for chunk in upstream.content.iter_any():
|
|
await out.write(chunk)
|
|
finally:
|
|
upstream.release()
|
|
await out.write_eof()
|
|
return out
|
|
|
|
|
|
async def main() -> None:
|
|
app = web.Application(client_max_size=1024 ** 3)
|
|
app["session"] = ClientSession()
|
|
app.router.add_route("*", "/{tail:.*}", proxy)
|
|
runner = web.AppRunner(app)
|
|
await runner.setup()
|
|
site = web.TCPSite(runner, "0.0.0.0", PORT)
|
|
await site.start()
|
|
print(f"proxy: :{PORT} -> {', '.join(UPSTREAMS)}", flush=True)
|
|
await asyncio.Event().wait()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|