From 3946ce9a3a6645616caa24881e13ba10bc0382de Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Wed, 16 Sep 2026 05:29:08 -0700
Subject: [PATCH] frank-medium: fail over to the other replica instead of
 returning 502
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The proxy trusted its health table, which the watcher refreshes every 10 s. A
replica that restarts — and a deploy restarts both — therefore produced a
window in which requests routed to the dead card got a 502 back, even though
the other card was serving the whole time.

A failed connection now demotes that replica immediately and retries the
request on another live one, before anything is written to the client.
vLLM's OpenAI endpoints are stateless, so re-sending a request that never
reached an engine is safe; a failure after streaming has begun is still not
retried. The failover is logged, because a silent one hides a card that is
down.

Measured with card 1 stopped mid-flight: six requests, six 200s, one failover
line in the log, no client-visible error. Before this commit that burst
returned 502s.
---
 .../frank-medium/proxy/affinity-proxy.py      | 46 ++++++++++++++-----
 1 file changed, 35 insertions(+), 11 deletions(-)

diff --git a/dankserver/frank-medium/proxy/affinity-proxy.py b/dankserver/frank-medium/proxy/affinity-proxy.py
index c749ca1..e526e8c 100755
--- a/dankserver/frank-medium/proxy/affinity-proxy.py
+++ b/dankserver/frank-medium/proxy/affinity-proxy.py
@@ -80,19 +80,43 @@ async def proxy(request: web.Request) -> web.StreamResponse:
         return await health(session)
 
     body = await request.read()
-    idx = routing.route(body, request.remote, _UP)
-    if idx < 0:
-        return web.Response(status=503, text="proxy: no healthy frank-medium replica\n")
-    target = UPSTREAMS[idx] + request.path_qs
     headers = {k: v for k, v in request.headers.items() if k.lower() not in HOP_BY_HOP}
 
-    try:
-        upstream = await session.request(
-            request.method, target, headers=headers, data=body or None,
-            timeout=ClientTimeout(total=None, sock_connect=10, sock_read=3600),
-        )
-    except Exception as exc:  # upstream refused or unreachable
-        return web.Response(status=502, text=f"proxy: {target} unreachable: {exc}\n")
+    # Fail over before anything is written to the client. A replica that
+    # restarts is invisible to the health watcher for up to HEALTH_TTL, and a
+    # deploy restarts both replicas — without this retry a 10 s window of
+    # requests would 502 on a service that has a live card the whole time.
+    # vLLM's OpenAI endpoints are stateless per request, so re-sending a
+    # request that never reached an engine is safe; a request that failed
+    # mid-stream is not retried (that path is below).
+    tried: set[int] = set()
+    upstream = None
+    last_error = "no healthy frank-medium replica"
+    while len(tried) < len(UPSTREAMS):
+        idx = routing.route(body, request.remote, _UP)
+        if idx < 0 or idx in tried:
+            break
+        tried.add(idx)
+        target = UPSTREAMS[idx] + request.path_qs
+        try:
+            upstream = await session.request(
+                request.method, target, headers=headers, data=body or None,
+                timeout=ClientTimeout(total=None, sock_connect=10, sock_read=3600),
+            )
+            break
+        except Exception as exc:
+            # Demote at once: the watcher will confirm, but every request until
+            # then should skip this replica rather than pay the same timeout.
+            # Logged because a silent failover hides a card that is down.
+            _UP[idx] = False
+            last_error = f"{target} unreachable: {exc}"
+            print(f"proxy: failover — {target} unreachable ({exc}); retrying "
+                  f"another replica", flush=True)
+    if upstream is None:
+        # 503 when no replica was even reachable through the routing table,
+        # 502 when replicas were tried and all of them failed.
+        status = 502 if tried else 503
+        return web.Response(status=status, text=f"proxy: {last_error}\n")
 
     out = web.StreamResponse(status=upstream.status, reason=upstream.reason)
     for k, v in upstream.headers.items():