chore: stop tracking .AGENTTRASH — the purger deletes tracked files, dirtying every host
The deletion policy moves in-repo deletions into .AGENTTRASH/ beside the item, and .gitignore line 25 already declares it ignored. But 37 retired payloads were committed before that rule existed, and gitignore never applies to tracked paths. Consequence measured 2026-10-31: Dankserver's deploy checkout reported 37 tracked deletions — exactly every tracked trash file — because the approved purger had done its job. Any host that purges is permanently dirty, and a future pull that touches one of those paths aborts mid-deploy with 'local changes would be overwritten' while the tree is in fact clean of real edits. Untrack the four dirs; the bytes stay on disk and in history (they always were recoverable from git, which is what the trash policy's 'temporary hold, not an archive' means). tests/deployment_scoping.bats asserts the staged chat template exists on disk — it still does, so the seam is untouched.
This commit is contained in:
dank committed
1 parent
6a4291f224
commit
8ae4d1c710
37 files changed
-4680
No files matched your search
@@ -1,145 +0,0 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the frank-medium proxy's routing decisions.
|
||||
# The proxy is the only thing between the fleet's clients and the two
|
||||
# single-GPU replicas, and its routing rules carry the performance work: spread
|
||||
# single-turn requests (batch members, one-shot questions) so both cards prefill
|
||||
# in parallel, and pin continuations so an agent loop keeps hitting the replica
|
||||
# whose prefix cache holds its context. This seam drives the routing policy
|
||||
# directly; the first version of these rules shipped a bug that only appeared on
|
||||
# a continuation request, which is exactly the class of mistake a test this
|
||||
# cheap should catch.
|
||||
|
||||
load helpers
|
||||
|
||||
ROUTING="$REPO_ROOT/dankserver/frank-medium/proxy/routing.py"
|
||||
PROXY="$REPO_ROOT/dankserver/frank-medium/proxy/affinity-proxy.py"
|
||||
|
||||
setup() {
|
||||
DRIVER="$BATS_TEST_TMPDIR/route.py"
|
||||
touch "$DRIVER"
|
||||
}
|
||||
|
||||
@test "the proxy and its routing policy exist and compile" {
|
||||
[ -f "$ROUTING" ]
|
||||
[ -f "$PROXY" ]
|
||||
python3 -m py_compile "$ROUTING"
|
||||
python3 -m py_compile "$PROXY"
|
||||
# The policy must not drag the HTTP stack in: it is imported by tests and
|
||||
# reasoned about without aiohttp, a network or a GPU.
|
||||
! grep -q 'import aiohttp' "$ROUTING"
|
||||
}
|
||||
|
||||
@test "a batch of single-turn requests spreads across both cards" {
|
||||
cat > "$DRIVER" <<PY
|
||||
import importlib.util, json, os, sys
|
||||
os.environ["ROUTING"] = "adaptive"
|
||||
spec = importlib.util.spec_from_file_location("routing", "$ROUTING")
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules["routing"] = mod
|
||||
spec.loader.exec_module(mod)
|
||||
class R:
|
||||
remote = "127.0.0.1"
|
||||
shared = "shared document " + "y" * 8000
|
||||
picks = []
|
||||
for i in range(6):
|
||||
text = shared if i % 2 else f"distinct prompt {i} " + "x" * 8000
|
||||
body = json.dumps({"messages": [{"role": "user", "content": text}]}).encode()
|
||||
picks.append(mod.pick(*mod.affinity_key(body, R.remote), [True, True]))
|
||||
print(" ".join(str(p) for p in picks))
|
||||
PY
|
||||
run python3 "$DRIVER"
|
||||
[ "$status" -eq 0 ]
|
||||
read -r -a picks <<< "$output"
|
||||
used=$(printf '%s\n' "${picks[@]}" | sort -u | wc -l)
|
||||
# Both cards must work on a batch, or one card prefills an entire benchmark
|
||||
# while the other idles.
|
||||
[ "$used" -eq 2 ]
|
||||
# Consecutive members alternate (the counter is advanced only by requests
|
||||
# that are not continuations), including the two that share a document.
|
||||
for ((i = 0; i < 5; i++)); do
|
||||
[ "${picks[i]}" != "${picks[i + 1]}" ]
|
||||
done
|
||||
}
|
||||
|
||||
@test "a continuation is pinned to the replica that served its prefix" {
|
||||
cat > "$DRIVER" <<PY
|
||||
import importlib.util, json, os, sys
|
||||
os.environ["ROUTING"] = "adaptive"
|
||||
spec = importlib.util.spec_from_file_location("routing", "$ROUTING")
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules["routing"] = mod
|
||||
spec.loader.exec_module(mod)
|
||||
class R:
|
||||
remote = "127.0.0.1"
|
||||
prefix = "system prompt " + "z" * 8000
|
||||
first = json.dumps({"messages": [{"role": "user", "content": prefix + " turn one"}]}).encode()
|
||||
start = mod.pick(*mod.affinity_key(first, R.remote), [True, True])
|
||||
# Same leading bytes, longer history and a different tail each turn: an agent
|
||||
# loop resending its context.
|
||||
turns = []
|
||||
history = [{"role": "assistant", "content": "ack"}]
|
||||
for tail in ("turn two", "turn three", "turn four"):
|
||||
history.append({"role": "user", "content": "question " + tail})
|
||||
body = json.dumps({"messages": [{"role": "user", "content": prefix + " turn one"}] + history}).encode()
|
||||
turns.append(mod.pick(*mod.affinity_key(body, R.remote), [True, True]))
|
||||
history.append({"role": "assistant", "content": "ack again"})
|
||||
print(start, " ".join(str(t) for t in turns))
|
||||
PY
|
||||
run python3 "$DRIVER"
|
||||
[ "$status" -eq 0 ]
|
||||
read -r -a out <<< "$output"
|
||||
# Every continuation agrees with the first turn's replica: that is where
|
||||
# the prefix cache entry lives.
|
||||
[ "${out[1]}" = "${out[0]}" ]
|
||||
[ "${out[2]}" = "${out[0]}" ]
|
||||
[ "${out[3]}" = "${out[0]}" ]
|
||||
}
|
||||
|
||||
@test "no routing mode ever picks a replica that is down" {
|
||||
for mode in adaptive affinity roundrobin; do
|
||||
cat > "$DRIVER" <<PY
|
||||
import importlib.util, json, os, sys
|
||||
os.environ["ROUTING"] = "$mode"
|
||||
spec = importlib.util.spec_from_file_location("routing", "$ROUTING")
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules["routing"] = mod
|
||||
spec.loader.exec_module(mod)
|
||||
class R:
|
||||
remote = "127.0.0.1"
|
||||
body = json.dumps({"messages": [{"role": "user", "content": "x" * 400}]}).encode()
|
||||
one_down = mod.pick(*mod.affinity_key(body, R.remote), [False, True])
|
||||
both_down = mod.pick(*mod.affinity_key(body, R.remote), [False, False])
|
||||
print(one_down, both_down)
|
||||
PY
|
||||
run python3 "$DRIVER"
|
||||
[ "$status" -eq 0 ]
|
||||
# Replica 0 down -> replica 1; both down -> -1, and the proxy answers
|
||||
# 503 rather than forwarding into the void.
|
||||
[ "$output" = "1 -1" ]
|
||||
done
|
||||
}
|
||||
|
||||
@test "affinity is deterministic per prefix, roundrobin is not" {
|
||||
cat > "$DRIVER" <<PY
|
||||
import importlib.util, json, os, sys
|
||||
os.environ["ROUTING"] = "affinity"
|
||||
spec = importlib.util.spec_from_file_location("routing", "$ROUTING")
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules["routing"] = mod
|
||||
spec.loader.exec_module(mod)
|
||||
class R:
|
||||
remote = "127.0.0.1"
|
||||
def pick(text):
|
||||
body = json.dumps({"messages": [{"role": "user", "content": text}]}).encode()
|
||||
return mod.pick(*mod.affinity_key(body, R.remote), [True, True])
|
||||
doc = "d" * 8000
|
||||
# Same leading 4 KiB, different tails: affinity keys on the prefix, so both go
|
||||
# to one replica. Different documents may hash anywhere — that is the point of
|
||||
# hashing — so only the prefix property is asserted.
|
||||
print(pick(doc + " a"), pick(doc + " b"))
|
||||
PY
|
||||
run python3 "$DRIVER"
|
||||
[ "$status" -eq 0 ]
|
||||
read -r -a aff <<< "$output"
|
||||
[ "${aff[0]}" = "${aff[1]}" ]
|
||||
}
|
||||
@@ -1,164 +0,0 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the frank-medium service on dankserver.
|
||||
# frank-medium became a rootless Quadlet deployment on 2026-09-15 when the
|
||||
# vllm-radiance MXFP4 stack replaced the charlie12345/ROCmFPX llama.cpp
|
||||
# runtime on the 2x Radeon AI PRO R9700. The intended shape is one
|
||||
# tensor-parallel process across both cards (frank-medium.container, kept as
|
||||
# the target); it cannot run on this host because RCCL's kernels require PCIe
|
||||
# atomics and the second card's chipset path advertises AtomicOpsCap: Routing-.
|
||||
# The deployed shape is therefore two single-GPU replicas behind a round-robin
|
||||
# proxy, which keeps the fleet contract of one :7770 endpoint. This seam pins
|
||||
# the properties that make that work: GPU access on the fixed device indices,
|
||||
# the upstream launcher's podman posture, host networking with the fleet's
|
||||
# client port and alias, per-replica resource shape, boot persistence, and a
|
||||
# deploy handler that moves the unusable TP=2 unit into the trash.
|
||||
|
||||
load helpers
|
||||
|
||||
UNIT_A="$REPO_ROOT/dankserver/frank-medium/quadlet/frank-medium-a.container"
|
||||
UNIT_B="$REPO_ROOT/dankserver/frank-medium/quadlet/frank-medium-b.container"
|
||||
UNIT_PROXY="$REPO_ROOT/dankserver/frank-medium/quadlet/frank-medium-proxy.container"
|
||||
UNIT_TP2="$REPO_ROOT/dankserver/frank-medium/quadlet/frank-medium.container"
|
||||
ENTRYPOINT="$REPO_ROOT/dankserver/frank-medium/radiance-entrypoint.sh"
|
||||
HEALTHCHECK="$REPO_ROOT/dankserver/frank-medium/frank-medium-healthcheck.py"
|
||||
PROXY="$REPO_ROOT/dankserver/frank-medium/proxy/affinity-proxy.py"
|
||||
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
|
||||
@test "the frank-medium units live under dankserver/frank-medium/quadlet/" {
|
||||
[ -f "$UNIT_A" ]
|
||||
[ -f "$UNIT_B" ]
|
||||
[ -f "$UNIT_PROXY" ]
|
||||
}
|
||||
|
||||
@test "the TP=2 unit is kept as the documented target, not deleted" {
|
||||
# It becomes runnable again only on a host whose second card sits on an
|
||||
# atomic-capable root port, so the artifact stays in the repo.
|
||||
[ -f "$UNIT_TP2" ]
|
||||
grep -q '^Environment=TP=2$' "$UNIT_TP2"
|
||||
}
|
||||
|
||||
@test "every frank-medium unit is boot-persistent (WantedBy=default.target)" {
|
||||
# Generated units cannot be enabled on the podman 5.x v1 quadlet
|
||||
# generator; [Install] plus linger is the entire boot-persistence story.
|
||||
for unit in "$UNIT_A" "$UNIT_B" "$UNIT_PROXY"; do
|
||||
grep -q '^WantedBy=default.target$' "$unit"
|
||||
done
|
||||
}
|
||||
|
||||
@test "every frank-medium unit is self-healing" {
|
||||
for unit in "$UNIT_A" "$UNIT_B" "$UNIT_PROXY"; do
|
||||
grep -q '^Restart=on-failure$' "$unit"
|
||||
grep -q '^RestartSec=' "$unit"
|
||||
done
|
||||
}
|
||||
|
||||
@test "frank-medium units pin the image tag and never pull at boot" {
|
||||
for unit in "$UNIT_A" "$UNIT_B" "$UNIT_PROXY"; do
|
||||
grep -Eq '^Image=localhost/radiance-vllm:[0-9]+\.[0-9]+\.[0-9]+$' "$unit"
|
||||
grep -q '^Pull=never$' "$unit"
|
||||
done
|
||||
}
|
||||
|
||||
@test "the replicas each get exactly one GPU on the fixed device index" {
|
||||
grep -q '^AddDevice=/dev/kfd$' "$UNIT_A"
|
||||
grep -q '^AddDevice=/dev/dri$' "$UNIT_A"
|
||||
# One index per replica: ROCR filters first and HIP indexes into the
|
||||
# filtered list, so setting both to the same non-zero value leaves an
|
||||
# empty device set.
|
||||
grep -q '^Environment=HIP_VISIBLE_DEVICES=0$' "$UNIT_A"
|
||||
grep -q '^Environment=HIP_VISIBLE_DEVICES=1$' "$UNIT_B"
|
||||
! grep -q '^Environment=ROCR_VISIBLE_DEVICES=' "$UNIT_A"
|
||||
! grep -q '^Environment=ROCR_VISIBLE_DEVICES=' "$UNIT_B"
|
||||
}
|
||||
|
||||
@test "the replicas run with the upstream launcher's podman posture" {
|
||||
for unit in "$UNIT_A" "$UNIT_B"; do
|
||||
grep -q -- '--privileged' "$unit"
|
||||
grep -q -- '--ipc=host' "$unit"
|
||||
grep -q -- '--security-opt seccomp=unconfined' "$unit"
|
||||
grep -q -- '--network=host' "$unit"
|
||||
done
|
||||
}
|
||||
|
||||
@test "the replicas serve single-GPU shapes that fit one card" {
|
||||
for unit in "$UNIT_A" "$UNIT_B"; do
|
||||
grep -q '^Environment=TP=1$' "$unit"
|
||||
# 262144 does not fit one card (needs 9.61 GiB of KV, 5.14 available)
|
||||
# and 131072 does not either (5.58 needed vs 5.22 available); 114688
|
||||
# is the largest measured shape that boots.
|
||||
grep -q '^Environment=MAXLEN=114688$' "$unit"
|
||||
# KV_MEM=0 makes vLLM profile this shape: the 2x7551-32624 pin is a
|
||||
# TP=2/262144 measurement and invalid here.
|
||||
grep -q '^Environment=KV_MEM=0$' "$unit"
|
||||
grep -q '^Environment=SNAP=/models/Qwen3.8-27B-Uncensored-MXFP4-awq$' "$unit"
|
||||
grep -q '^Environment=SPEC_METHOD=dflash$' "$unit"
|
||||
grep -q '^Environment=SPEC=7$' "$unit"
|
||||
done
|
||||
}
|
||||
|
||||
@test "the proxy keeps the fleet endpoint: host-network port 7770, alias frank-medium" {
|
||||
grep -q '^Environment=PORT=7770$' "$UNIT_PROXY"
|
||||
grep -q '^Environment=UPSTREAMS=http://127.0.0.1:7771,http://127.0.0.1:7772$' "$UNIT_PROXY"
|
||||
grep -q -- '--network=host' "$UNIT_PROXY"
|
||||
[ "$(grep -c '^PublishPort=' "$UNIT_PROXY")" -eq 0 ]
|
||||
# The replicas answer as the same served model name, so a client cannot
|
||||
# tell which one served a request.
|
||||
grep -q '^Environment=SERVED_NAMES=frank-medium$' "$UNIT_A"
|
||||
grep -q '^Environment=SERVED_NAMES=frank-medium$' "$UNIT_B"
|
||||
grep -q '^Environment=PORT=7771$' "$UNIT_A"
|
||||
grep -q '^Environment=PORT=7772$' "$UNIT_B"
|
||||
}
|
||||
|
||||
@test "frank-medium entrypoint is a syntax-clean fork of the upstream launcher" {
|
||||
[ -f "$ENTRYPOINT" ]
|
||||
bash -n "$ENTRYPOINT"
|
||||
grep -q '9735329348ae1c9319f01cbe5f7004ec5f4dba63' "$ENTRYPOINT"
|
||||
}
|
||||
|
||||
@test "frank-medium healthcheck probes the serving endpoint on its own port" {
|
||||
[ -f "$HEALTHCHECK" ]
|
||||
python3 -m py_compile "$HEALTHCHECK"
|
||||
# One probe serves the replicas, the proxy and the TP=2 target because it
|
||||
# reads PORT instead of hardcoding 7770.
|
||||
grep -q 'os.environ.get("PORT"' "$HEALTHCHECK"
|
||||
for unit in "$UNIT_A" "$UNIT_B" "$UNIT_PROXY"; do
|
||||
grep -q 'frank-medium-healthcheck.py' "$unit"
|
||||
grep -q '^HealthStartPeriod=' "$unit"
|
||||
done
|
||||
}
|
||||
|
||||
@test "the proxy unit mounts the routing policy and the server" {
|
||||
# The policy lives in its own file so it can be tested without aiohttp;
|
||||
# the server imports it by name from its own directory, so both must be
|
||||
# mounted or the container dies on ModuleNotFoundError at start.
|
||||
grep -q 'proxy/routing.py:/usr/local/bin/routing.py:ro' "$UNIT_PROXY"
|
||||
grep -q 'proxy/affinity-proxy.py:/usr/local/bin/affinity-proxy.py:ro' "$UNIT_PROXY"
|
||||
# Routing behaviour itself is covered by tests/frank_medium_proxy.bats,
|
||||
# which drives the policy directly.
|
||||
[ -f "$REPO_ROOT/tests/frank_medium_proxy.bats" ]
|
||||
}
|
||||
@test "the deploy handler installs the trio and retires the TP=2 unit" {
|
||||
grep -q 'install_quadlet frank-medium-a dankserver/frank-medium/quadlet frank-medium-a' "$REMOTE_DEPLOY"
|
||||
grep -q 'install_quadlet frank-medium-b dankserver/frank-medium/quadlet frank-medium-b' "$REMOTE_DEPLOY"
|
||||
grep -q 'install_quadlet frank-medium-proxy dankserver/frank-medium/quadlet frank-medium-proxy' "$REMOTE_DEPLOY"
|
||||
grep -q 'systemctl --user restart frank-medium-a.service frank-medium-b.service frank-medium-proxy.service' "$REMOTE_DEPLOY"
|
||||
# The TP=2 unit cannot become healthy on this host; a leftover copy would
|
||||
# leave the generator restarting it forever. It moves to the trash.
|
||||
grep -q 'mv -- "$quadlet_unit_dir/frank-medium.container"' "$REMOTE_DEPLOY"
|
||||
! grep -qE '(^|[^[:alnum:]_])rm (-[a-zA-Z]+ )?"?\$quadlet_unit_dir/frank-medium' "$REMOTE_DEPLOY"
|
||||
}
|
||||
|
||||
@test "the deploy handler never tries to enable a generated unit" {
|
||||
! grep -q 'systemctl --user enable frank-medium' "$REMOTE_DEPLOY"
|
||||
}
|
||||
|
||||
@test "the deploy handler retires the old root unit without deleting it" {
|
||||
grep -q 'retire_root_llm_medium' "$REMOTE_DEPLOY"
|
||||
grep -q 'systemctl stop frank-medium.service' "$REMOTE_DEPLOY"
|
||||
grep -q 'mv -- /etc/systemd/system/frank-medium.service' "$REMOTE_DEPLOY"
|
||||
! grep -qE '(^|[^[:alnum:]_])rm (-[a-zA-Z]+ )?/etc/systemd/system/frank-medium' "$REMOTE_DEPLOY"
|
||||
}
|
||||
|
||||
@test "the deploy handler re-tags the registry image to the localhost alias" {
|
||||
grep -q 'podman tag docker.io/stilldeadcode/vllm-radiance:0.9.3 localhost/radiance-vllm:0.9.3' "$REMOTE_DEPLOY"
|
||||
}
|
||||
Reference in new issue
Block a user