#!/usr/bin/env bats
# Seam: the frank-medium service on dankserver.
# frank-medium became a rootless Quadlet deployment on 2026-09-15 when the
# vllm-radiance MXFP4 stack replaced the charlie12345/ROCmFPX llama.cpp
# runtime on the 2x Radeon AI PRO R9700. The intended shape is one
# tensor-parallel process across both cards (frank-medium.container, kept as
# the target); it cannot run on this host because RCCL's kernels require PCIe
# atomics and the second card's chipset path advertises AtomicOpsCap: Routing-.
# The deployed shape is therefore two single-GPU replicas behind a round-robin
# proxy, which keeps the fleet contract of one :7770 endpoint. This seam pins
# the properties that make that work: GPU access on the fixed device indices,
# the upstream launcher's podman posture, host networking with the fleet's
# client port and alias, per-replica resource shape, boot persistence, and a
# deploy handler that moves the unusable TP=2 unit into the trash.

load helpers

UNIT_A="$REPO_ROOT/dankserver/frank-medium/quadlet/frank-medium-a.container"
UNIT_B="$REPO_ROOT/dankserver/frank-medium/quadlet/frank-medium-b.container"
UNIT_PROXY="$REPO_ROOT/dankserver/frank-medium/quadlet/frank-medium-proxy.container"
UNIT_TP2="$REPO_ROOT/dankserver/frank-medium/quadlet/frank-medium.container"
ENTRYPOINT="$REPO_ROOT/dankserver/frank-medium/radiance-entrypoint.sh"
HEALTHCHECK="$REPO_ROOT/dankserver/frank-medium/frank-medium-healthcheck.py"
PROXY="$REPO_ROOT/dankserver/frank-medium/proxy/affinity-proxy.py"
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"

@test "the frank-medium units live under dankserver/frank-medium/quadlet/" {
    [ -f "$UNIT_A" ]
    [ -f "$UNIT_B" ]
    [ -f "$UNIT_PROXY" ]
}

@test "the TP=2 unit is kept as the documented target, not deleted" {
    # It becomes runnable again only on a host whose second card sits on an
    # atomic-capable root port, so the artifact stays in the repo.
    [ -f "$UNIT_TP2" ]
    grep -q '^Environment=TP=2$' "$UNIT_TP2"
}

@test "every frank-medium unit is boot-persistent (WantedBy=default.target)" {
    # Generated units cannot be enabled on the podman 5.x v1 quadlet
    # generator; [Install] plus linger is the entire boot-persistence story.
    for unit in "$UNIT_A" "$UNIT_B" "$UNIT_PROXY"; do
        grep -q '^WantedBy=default.target$' "$unit"
    done
}

@test "every frank-medium unit is self-healing" {
    for unit in "$UNIT_A" "$UNIT_B" "$UNIT_PROXY"; do
        grep -q '^Restart=on-failure$' "$unit"
        grep -q '^RestartSec=' "$unit"
    done
}

@test "frank-medium units pin the image tag and never pull at boot" {
    for unit in "$UNIT_A" "$UNIT_B" "$UNIT_PROXY"; do
        grep -Eq '^Image=localhost/radiance-vllm:[0-9]+\.[0-9]+\.[0-9]+$' "$unit"
        grep -q '^Pull=never$' "$unit"
    done
}

@test "the replicas each get exactly one GPU on the fixed device index" {
    grep -q '^AddDevice=/dev/kfd$' "$UNIT_A"
    grep -q '^AddDevice=/dev/dri$' "$UNIT_A"
    # One index per replica: ROCR filters first and HIP indexes into the
    # filtered list, so setting both to the same non-zero value leaves an
    # empty device set.
    grep -q '^Environment=HIP_VISIBLE_DEVICES=0$' "$UNIT_A"
    grep -q '^Environment=HIP_VISIBLE_DEVICES=1$' "$UNIT_B"
    ! grep -q '^Environment=ROCR_VISIBLE_DEVICES=' "$UNIT_A"
    ! grep -q '^Environment=ROCR_VISIBLE_DEVICES=' "$UNIT_B"
}

@test "the replicas run with the upstream launcher's podman posture" {
    for unit in "$UNIT_A" "$UNIT_B"; do
        grep -q -- '--privileged' "$unit"
        grep -q -- '--ipc=host' "$unit"
        grep -q -- '--security-opt seccomp=unconfined' "$unit"
        grep -q -- '--network=host' "$unit"
    done
}

@test "the replicas serve single-GPU shapes that fit one card" {
    for unit in "$UNIT_A" "$UNIT_B"; do
        grep -q '^Environment=TP=1$' "$unit"
        # 262144 does not fit one card (needs 9.61 GiB of KV, 5.14 available)
        # and 131072 does not either (5.58 needed vs 5.22 available); 114688
        # is the largest measured shape that boots.
        grep -q '^Environment=MAXLEN=114688$' "$unit"
        # KV_MEM=0 makes vLLM profile this shape: the 2x7551-32624 pin is a
        # TP=2/262144 measurement and invalid here.
        grep -q '^Environment=KV_MEM=0$' "$unit"
        grep -q '^Environment=SNAP=/models/Qwen3.8-27B-Uncensored-MXFP4-awq$' "$unit"
        grep -q '^Environment=SPEC_METHOD=dflash$' "$unit"
        grep -q '^Environment=SPEC=7$' "$unit"
    done
}

@test "the proxy keeps the fleet endpoint: host-network port 7770, alias frank-medium" {
    grep -q '^Environment=PORT=7770$' "$UNIT_PROXY"
    grep -q '^Environment=UPSTREAMS=http://127.0.0.1:7771,http://127.0.0.1:7772$' "$UNIT_PROXY"
    grep -q -- '--network=host' "$UNIT_PROXY"
    [ "$(grep -c '^PublishPort=' "$UNIT_PROXY")" -eq 0 ]
    # The replicas answer as the same served model name, so a client cannot
    # tell which one served a request.
    grep -q '^Environment=SERVED_NAMES=frank-medium$' "$UNIT_A"
    grep -q '^Environment=SERVED_NAMES=frank-medium$' "$UNIT_B"
    grep -q '^Environment=PORT=7771$' "$UNIT_A"
    grep -q '^Environment=PORT=7772$' "$UNIT_B"
}

@test "frank-medium entrypoint is a syntax-clean fork of the upstream launcher" {
    [ -f "$ENTRYPOINT" ]
    bash -n "$ENTRYPOINT"
    grep -q '9735329348ae1c9319f01cbe5f7004ec5f4dba63' "$ENTRYPOINT"
}

@test "frank-medium healthcheck probes the serving endpoint on its own port" {
    [ -f "$HEALTHCHECK" ]
    python3 -m py_compile "$HEALTHCHECK"
    # One probe serves the replicas, the proxy and the TP=2 target because it
    # reads PORT instead of hardcoding 7770.
    grep -q 'os.environ.get("PORT"' "$HEALTHCHECK"
    for unit in "$UNIT_A" "$UNIT_B" "$UNIT_PROXY"; do
        grep -q 'frank-medium-healthcheck.py' "$unit"
        grep -q '^HealthStartPeriod=' "$unit"
    done
}

@test "the proxy unit mounts the routing policy and the server" {
    # The policy lives in its own file so it can be tested without aiohttp;
    # the server imports it by name from its own directory, so both must be
    # mounted or the container dies on ModuleNotFoundError at start.
    grep -q 'proxy/routing.py:/usr/local/bin/routing.py:ro' "$UNIT_PROXY"
    grep -q 'proxy/affinity-proxy.py:/usr/local/bin/affinity-proxy.py:ro' "$UNIT_PROXY"
    # Routing behaviour itself is covered by tests/frank_medium_proxy.bats,
    # which drives the policy directly.
    [ -f "$REPO_ROOT/tests/frank_medium_proxy.bats" ]
}
@test "the deploy handler installs the trio and retires the TP=2 unit" {
    grep -q 'install_quadlet frank-medium-a dankserver/frank-medium/quadlet frank-medium-a' "$REMOTE_DEPLOY"
    grep -q 'install_quadlet frank-medium-b dankserver/frank-medium/quadlet frank-medium-b' "$REMOTE_DEPLOY"
    grep -q 'install_quadlet frank-medium-proxy dankserver/frank-medium/quadlet frank-medium-proxy' "$REMOTE_DEPLOY"
    grep -q 'systemctl --user restart frank-medium-a.service frank-medium-b.service frank-medium-proxy.service' "$REMOTE_DEPLOY"
    # The TP=2 unit cannot become healthy on this host; a leftover copy would
    # leave the generator restarting it forever. It moves to the trash.
    grep -q 'mv -- "$quadlet_unit_dir/frank-medium.container"' "$REMOTE_DEPLOY"
    ! grep -qE '(^|[^[:alnum:]_])rm (-[a-zA-Z]+ )?"?\$quadlet_unit_dir/frank-medium' "$REMOTE_DEPLOY"
}

@test "the deploy handler never tries to enable a generated unit" {
    ! grep -q 'systemctl --user enable frank-medium' "$REMOTE_DEPLOY"
}

@test "the deploy handler retires the old root unit without deleting it" {
    grep -q 'retire_root_llm_medium' "$REMOTE_DEPLOY"
    grep -q 'systemctl stop frank-medium.service' "$REMOTE_DEPLOY"
    grep -q 'mv -- /etc/systemd/system/frank-medium.service' "$REMOTE_DEPLOY"
    ! grep -qE '(^|[^[:alnum:]_])rm (-[a-zA-Z]+ )?/etc/systemd/system/frank-medium' "$REMOTE_DEPLOY"
}

@test "the deploy handler re-tags the registry image to the localhost alias" {
    grep -q 'podman tag docker.io/stilldeadcode/vllm-radiance:0.9.3 localhost/radiance-vllm:0.9.3' "$REMOTE_DEPLOY"
}