#!/usr/bin/env bats
# Seam: the frank-medium mainline-llama.cpp root unit on dankserver.
# frank-medium became a native llama.cpp service on 2026-09-16, replacing the
# vllm-radiance replica pair + routing proxy that served this endpoint for a
# day. What this seam pins is what the flags alone do not say: boot persistence
# for a NATIVE runtime (no generated quadlet is involved), that BOTH cards work
# on ONE model through a layer split with no collective in the path, that the
# DFlash2 drafter is wired with a depth its block size actually allows, that the
# runtime and payload are pinned rather than floated, and that one slot's KV
# still fits a 32 GiB card beside the weights it holds.

load helpers

UNIT="$REPO_ROOT/scripts/frank-medium.service"
SERVE="$REPO_ROOT/scripts/serve-frank-medium.sh"
ENV_FILE="$REPO_ROOT/scripts/frank-medium.env"
BUILD="$REPO_ROOT/scripts/build-llama-rdna4.sh"
FETCH="$REPO_ROOT/scripts/download-frank-medium-gguf.sh"
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"

# Measured constants (rocm-smi / HF blob sizes), used by the fit arithmetic.
CARD_BYTES=34208743424      # one R9700 as rocm-smi reports it
MODEL_BYTES=31457990112     # UD-Q8_K_XL target
DRAFT_BYTES=2056414816      # DFlash2 Q8_0 drafter

@test "frank-medium's unit, serve script, env and its build/fetch scripts exist" {
    [ -f "$UNIT" ]
    [ -f "$SERVE" ]
    [ -f "$ENV_FILE" ]
    [ -f "$BUILD" ]
    [ -f "$FETCH" ]
}

@test "frank-medium is boot-persistent, self-healing and unprivileged" {
    grep -q '^WantedBy=multi-user.target$' "$UNIT"
    grep -q '^Restart=on-failure$' "$UNIT"
    grep -q '^RestartSec=' "$UNIT"
    grep -q '^User=dank$' "$UNIT"
    grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
}

@test "frank-medium runs the repo's own serve script with the repo's env" {
    grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-medium.sh$' "$UNIT"
    grep -q 'FRANK_MEDIUM_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-medium.env' "$UNIT"
}

@test "both R9700s serve ONE model, layer-split inside one process" {
    grep -q '^SPLIT_MODE=layer$' "$ENV_FILE"
    grep -q '^DEVICES=ROCm0,ROCm1$' "$ENV_FILE"
    # The split is a layer PAIR: the shares add up to the model's 65 blocks (one
    # -ts unit per layer) and ROCm1 -- the card that also carries the drafter and
    # the output layers -- takes the FEWER of the two.  WHICH pair balances the
    # cards is the seam test's job, which recomputes it from this file; pinning a
    # literal here would only mean that re-measuring the balance breaks a test
    # about parallelism.
    local split_hi split_lo
    split_hi="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\1/p' "$ENV_FILE")"
    split_lo="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\2/p' "$ENV_FILE")"
    [ -n "$split_hi" ] && [ -n "$split_lo" ]
    [ $(( split_hi + split_lo )) -eq 65 ]
    [ "$split_lo" -lt "$split_hi" ]
    grep -q -- '--split-mode "$SPLIT_MODE"' "$SERVE"
    grep -q -- '--device "$DEVICES"' "$SERVE"
    grep -q -- '--tensor-split "$TENSOR_SPLIT"' "$SERVE"
    # Tensor split is the one shape this host cannot run: it needs a cross-card
    # collective, and the second card cannot dispatch one (see the env file).
    ! grep -q '^SPLIT_MODE=tensor$' "$ENV_FILE"
}

@test "the iGPU is hidden so no layer can land on system RAM" {
    grep -q '^HIP_VISIBLE_DEVICES=0,1$' "$ENV_FILE"
    grep -q '^export HIP_VISIBLE_DEVICES$' "$SERVE"
}

@test "the drafter is the model's own MTP head, at the measured depth" {
    # MTP's head lives in the target file (blk.<n>.nextn.*): no sidecar, no
    # second copy of the weights, and it is the only drafter that fits the
    # 393216 pool.  n_max 3 is the measured MTP winner -- n=7 accepts 33 % of
    # 147, n=3 accepts 58 % of 72 (the env's table; identical greedy text at
    # every depth, so this is pure speed).
    grep -qE '^SPEC_TYPE=draft-mtp$' "$ENV_FILE"
    grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE"
    local n_max
    n_max="$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")"
    [ -n "$n_max" ] && [ "$n_max" -eq 3 ]
    # The DFlash2 sidecar stays on disk and in the fetch script -- one
    # SPEC_TYPE change brings the faster chain back -- and ngram-mod's window
    # is retained with it, so neither may be dropped from the env.
    grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DFlash2-Q8_0\.gguf$' "$ENV_FILE"
    grep -q '^SPEC_NGRAM_N_MATCH=24$' "$ENV_FILE"
    # ...and neither is loaded: ngram-mod's flags only go out when the type
    # contains ngram, so a pure MTP configuration is not handed them.
    python3 -c "
lines = open('$SERVE').read().splitlines()
guard = next(i for i, l in enumerate(lines) if '*ngram*' in l)
indent = len(lines[guard]) - len(lines[guard].lstrip())
end = next(i for i in range(guard + 1, len(lines))
           if lines[i].strip() == 'fi' and len(lines[i]) - len(lines[i].lstrip()) == indent)
assert '--spec-ngram-mod-n-match' in chr(10).join(lines[guard:end]), 'ngram flags must sit in the ngram branch'
"
}

@test "a draft MODEL is passed only for DFlash types, never for MTP" {
    # MTP builds its context against the TARGET model (its head lives in that
    # GGUF). Passing --spec-draft-model alongside it points the draft context at
    # the drafter's file, which has no MTP layers, and the server dies at load:
    #   context type MTP requested but model doesn't contain MTP layers
    #   failed to create MTP context
    # So the flag must appear exactly once, inside the dflash branch.
    python3 -c "
lines = open('$SERVE').read().splitlines()
guard = next(i for i, l in enumerate(lines) if '*dflash*' in l)
indent = len(lines[guard]) - len(lines[guard].lstrip())
end = next(i for i in range(guard + 1, len(lines))
           if lines[i].strip() == 'fi' and len(lines[i]) - len(lines[i].lstrip()) == indent)
block = chr(10).join(lines[guard:end])
assert '--spec-draft-model' in block, 'the dflash branch must pass the draft model'
assert sum(l.count('--spec-draft-model') for l in lines) == 1, 'only that branch may pass it'
print('draft-model wiring ok')
"
}

@test "the runtime is pinned to a commit and to gfx1201 alone" {
    # A floating ref would silently move the service between engine versions,
    # and an unpinned arch would build for a sibling GPU that loads and then
    # segfaults on these cards.
    grep -Eq '^LLAMA_REF="\$\{LLAMA_REF:-[0-9a-f]{40}\}"$' "$BUILD"
    grep -Eq '^HIP_ARCH="\$\{HIP_ARCH:-gfx1201\}"$' "$BUILD"
    grep -q 'GGML_HIP=ON' "$BUILD"
    grep -q 'frank-medium.env' "$SERVE"
}

@test "the payload is fetched resumably and verified by size" {
    grep -q 'UD-Q8_K_XL\.gguf' "$FETCH"
    grep -q 'Qwen3\.8-27B-DFlash2-Q8_0\.gguf' "$FETCH"
    grep -q 'curl -fL -C -' "$FETCH"
    grep -q 'verifying sizes' "$FETCH"
}

@test "both cards stay balanced and keep the operator's 1.5 GiB floor" {
    local ctx parallel cache_k cache_v split_hi split_lo batch ubatch
    ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
    parallel="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")"
    cache_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")"
    cache_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")"
    split_hi="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\1/p' "$ENV_FILE")"
    split_lo="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\2/p' "$ENV_FILE")"
    batch="$(sed -nE 's/^BATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
    ubatch="$(sed -nE 's/^UBATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
    spec_type="$(sed -nE 's/^SPEC_TYPE=(.+)$/\1/p' "$ENV_FILE")"
    [ -n "$ctx" ] && [ -n "$parallel" ] && [ -n "$cache_k" ] && [ -n "$cache_v" ] && [ -n "$spec_type" ]
    [ -n "$split_hi" ] && [ -n "$split_lo" ] && [ -n "$batch" ] && [ -n "$ubatch" ]
    [ "$parallel" -ge 1 ]
    [ "$batch" -ge "$ubatch" ]
    [ "$cache_k" = "f16" ] && [ "$cache_v" = "f16" ]
    # Per card, calibrated against rocm-smi at the 35:30 / ctx 262144 / batch
    # 6144 x ubatch 512 / two slots / DFlash2 point, where the two cards
    # measured 6.44 / 6.90 GiB free.  The fit reproduces card 1 to 0.00 GiB
    # and card 0 to 0.86 GiB at that point; it is a GATE, not a meter -- the
    # deployed point is re-measured on the host after every split/ctx/batch
    # move (the current one is 35:30 / 393216 / MTP, moved from 34:31 on
    # 2026-09-17 to lean work off ROCm1's Gen4-x1 chipset path -- see the
    # TENSOR_SPLIT header in frank-medium.env).
    #
    #   card 0 = its share of the weights and of the KV (qwen35 spends KV on
    #            its 16 full-attention layers only: 2 (K,V) x 4 heads x 256 dim
    #            x 2 B = 64 KiB per token at f16) plus ~1.4e6 B per ubatch unit
    #            of compute buffers
    #   card 1 = the same for its own share, plus the drafter.  The drafter's
    #            charge depends on WHAT it is: DFlash2 is a sidecar file, so
    #            its weights (2.06e9 B) plus window-bounded KV (20480 B/pos x
    #            2048) plus ~0.89 GiB of constant buffers; MTP's head lives in
    #            the target file (already charged above), so its cost is one
    #            nextn layer's KV (4096 B/position) plus small buffers.  Its
    #            compute buffers are the heavier ones either way, because it
    #            holds the output layers and the drafter's graphs.
    #
    # BOTH ubatch coefficients are measured at the deployed shape, from the
    # 1536 -> 512 move: 1.30 GiB on card 0 and 2.41 GiB on card 1.  That
    # asymmetry is why shrinking the batch moved the balance toward card 1 and
    # cost card 0 a layer -- the split and the batch cannot be chosen apart.
    #
    # The layer it charges for is the AVERAGE (0.70 GiB: 31.46e9 of weights plus
    # 16 GiB of KV over 65 layers).  This stack is a hybrid, so real steps vary
    # around that -- 0.43 GiB for a DeltaNet layer, 1.43 GiB for a full-attention
    # one -- which makes the drift check below CONSERVATIVE: it overstates the gap
    # a one-layer move creates, and so trips on moves a measurement might bless.
    # That is the intent: moving the split means re-measuring, not re-arguing.
    #
    # The slack is the DEPLOYED gap, rounded up: at each shape the balance
    # cannot be tuned finer than one layer, so the deployed split's gap IS
    # what the check must accept, and a tighter slack would only reject it.
    #
    # What this model cannot see is CONTIGUITY -- the mechanism by which the
    # DSpark-era 8192/2048 failed, a 2020 MiB pp buffer that found no room while
    # every total looked fine.  Total bytes are not the whole story for one big
    # allocation.
    local kv_pos=65536 draft_kv_pos=20480 mtp_kv_pos=4096 floor slack
    local card_hi card_lo diff
    local mmproj=931145888   # mmproj-BF16.gguf, autoloaded onto the primary device
    # 1.5 GiB was the floor until the 2x196k shape: the 34:31 / 393216 /
    # MTP point measured 28.91 / 30.78 GiB, leaving 2.95 / 1.08 GiB, so the
    # floor is the measured 1.08 GiB rounded down.  The margin for one big
    # allocation is thinner than it used to be.  The layer-toward-ROCm0
    # lever has since been spent (35:30, 2026-09-17): if a large allocation
    # fails at load now, the next lever is a ctx cut, not another layer.
    floor=$(( 1024 * 1024 * 1024 ))     # 1.0 GiB per card
    # The deployed point (35:30 at ctx 393216 with MTP) has a fit gap of
    # 0.58 GiB -- closer to balanced than its 34:31 predecessor (1.06 GiB).
    # The slack sits at 1.483 GiB: it accepts the deployed split and still
    # rejects the next step toward ROCm0, 36:29, which fails BOTH gates
    # (gap 2.22 GiB, card 0 free 0.45 GiB under the floor).  (The 36:29 /
    # 327680 / DFlash2 configuration's slack was 3225 MiB -- its 3.15 GiB
    # gap was by construction, and is no longer the deployed shape.)
    slack=$(( 1483 * 1024 * 1024 ))
    card_hi=$(( MODEL_BYTES * split_hi / (split_hi + split_lo) \
              + kv_pos * ctx * split_hi / (split_hi + split_lo) \
              + 381898377 \
              + mmproj \
              + 1363 * 1024 * ubatch ))
    if [[ "$spec_type" == *dflash* || "$spec_type" == *dspark* ]]; then
        card_lo=$(( MODEL_BYTES * split_lo / (split_hi + split_lo) \
                  + DRAFT_BYTES \
                  + kv_pos * ctx * split_lo / (split_hi + split_lo) \
                  + draft_kv_pos * (ctx < 2048 ? ctx : 2048) \
                  + 955046413 \
                  + 2468 * 1024 * ubatch ))
    else
        # MTP: the head is in the target file (charged above); one nextn
        # layer's KV plus buffers/graphs.  The constant is CALIBRATED to the
        # deployed point (2026-09-17, 34:31 / 393216): card 1 measured 30.78
        # GiB, of which 4.20 GiB is MTP's total charge -- 1.61 GiB of nextn KV
        # plus 2.70 GiB of buffers and graphs, far more than the 0.2 GiB
        # estimate the DFlash2-era shape suggested.
        card_lo=$(( MODEL_BYTES * split_lo / (split_hi + split_lo) \
                  + kv_pos * ctx * split_lo / (split_hi + split_lo) \
                  + mtp_kv_pos * ctx \
                  + 2901231913 \
                  + 2468 * 1024 * ubatch ))
    fi
    [ "$card_hi" -le $(( CARD_BYTES - floor )) ]
    [ "$card_lo" -le $(( CARD_BYTES - floor )) ]
    diff=$(( card_hi - card_lo )); [ "$diff" -ge 0 ] || diff=$(( -diff ))
    [ "$diff" -le "$slack" ]
    # ...and the window may not wander far past what the checkpoint was
    # trained for (262144; the GGUF carries no rope scaling, so the top 131k
    # of the 393216 pool extrapolates on the base rope).
    [ "$ctx" -le 393216 ]
}

@test "the host-RAM prompt cache can hold a full slot's state" {
    # --cache-ram is the one cache tunable that is NOT VRAM: it caps the total
    # bytes of saved prompt states in host RAM (server_prompt_cache, in
    # tools/server/server-task.cpp).  alloc() REFUSES any single state larger
    # than the whole cap -- silently, with one WRN line -- so a budget below
    # one slot's full context makes every deep conversation uncacheable and
    # every turn past it re-prefill from scratch.  That was the deployed state
    # on 2026-09-18: 137 refusals of 11.0-11.4 GiB states against the 8192 MiB
    # cap, in two days of journal.
    #
    # The state law at this shape, per token of sequence: 16 full-attention
    # layers x 4 kv heads x 256 dim x 2 B x 2 (K,V) = 64 KiB, plus 4096 B for
    # the MTP draft context's own KV.  The DeltaNet state is constant-size and
    # is left out of the sum -- which only makes this gate more conservative.
    local ctx parallel cache_mib per_token slot_state host_mib
    ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
    parallel="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")"
    cache_mib="$(sed -nE 's/^CACHE_RAM=([0-9]+)$/\1/p' "$ENV_FILE")"
    [ -n "$ctx" ] && [ -n "$parallel" ] && [ -n "$cache_mib" ]
    per_token=$(( (16 * 4 * 256 * 2 * 2) + (4 * 256 * 2 * 2) ))
    slot_state=$(( per_token * (ctx / parallel) ))
    # one full slot must fit, or the cache is decorative
    [ "$cache_mib" -ge $(( slot_state / 1024 / 1024 )) ]
    # ...and this budget is host RAM: never more than half of dankserver's
    # 62220 MiB, which the media-stack co-tenants and the page cache share.
    host_mib=62220
    [ "$cache_mib" -le $(( host_mib / 2 )) ]
    # the arithmetic above is about nothing unless the flag goes out
    grep -q -- '--cache-ram "$CACHE_RAM"' "$SERVE"
    grep -q -- '--cache-prompt' "$SERVE"
}

@test "vision is enabled: the BF16 projector loads with the model" {
    grep -q '^NO_MMPROJ=0$' "$ENV_FILE"
    grep -q '^MM_PROJ_FILE=mmproj-BF16.gguf$' "$ENV_FILE"
    # Explicit flag, not autoload: this build starts text-only without it,
    # and refuses to start without the projector file, so a missing file is
    # a deploy error, not a silent text-only server.
    grep -q -- '--mmproj "$MM_PROJ"' "$SERVE"
    grep -q 'vision is enabled but the projector is missing' "$SERVE"
    # And the download script fetches and byte-verifies it.
    grep -q '^MM_PROJ_SIZE=931145888$' "$FETCH"
    grep -q -- 'check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE"' "$FETCH"
}

@test "the fleet endpoint and the predecessor's sampler defaults survive" {
    grep -q '^ALIAS=frank-medium$' "$ENV_FILE"
    grep -q '^LISTEN_PORT=7770$' "$ENV_FILE"
    grep -q -- '--alias "$ALIAS"' "$SERVE"
    grep -q -- '--port "$LISTEN_PORT"' "$SERVE"
    grep -q -- '--host "$BIND_ADDR"' "$SERVE"
    # Prometheus scrapes this endpoint for LLM fleet metrics.
    grep -q -- '--metrics' "$SERVE"
    # Requests that send no sampler fields get these instead of an error.
    grep -q '^TEMPERATURE=0.7$' "$ENV_FILE"
    grep -q '^TOP_P=0.95$' "$ENV_FILE"
}

@test "the deploy handler installs the root unit and refuses a missing payload" {
    grep -q 'deploy_llm_medium' "$REMOTE_DEPLOY"
    grep -q 'relink_unit frank-medium' "$REMOTE_DEPLOY"
    grep -q 'systemctl enable frank-medium.service' "$REMOTE_DEPLOY"
    grep -q 'systemctl restart frank-medium.service' "$REMOTE_DEPLOY"
    grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY"
    grep -q 'build-llama-rdna4.sh' "$REMOTE_DEPLOY"
    grep -q 'download-frank-medium-gguf.sh' "$REMOTE_DEPLOY"
    grep -q '\[\[ "$missing" -eq 0 \]\] || return 1' "$REMOTE_DEPLOY"
}

@test "the deploy handler retires the vLLM quadlets and frees the port they held" {
    grep -q 'retire_vllm_medium_quadlets' "$REMOTE_DEPLOY"
    grep -q 'systemctl --user stop frank-medium-a.service frank-medium-b.service' "$REMOTE_DEPLOY"
    grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
    # No image left to tag: the runtime is a native binary now.
    # ...and no image work or quadlet install of the predecessor survives.
    ! grep -q 'install_quadlet frank-medium-a' "$REMOTE_DEPLOY"
    ! grep -q 'podman tag docker.io/stilldeadcode' "$REMOTE_DEPLOY"
}

@test "the retired vLLM deployment is gone from the repo, not merely unmapped" {
    [ ! -d "$REPO_ROOT/dankserver/frank-medium" ]
    [ ! -f "$REPO_ROOT/tests/frank_medium_quadlet.bats" ]
    # ...and the archive is present, so the two assertions above cannot pass
    # because a path went missing rather than because it stayed missing.
    [ -f "$REPO_ROOT/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-a.container" ]
    [ -f "$REPO_ROOT/dankserver/.AGENTTRASH/20260916-vllm-radiance/radiance-entrypoint.sh" ]
}

@test "the serve script refuses to start on an incomplete host" {
    grep -q 'frank-medium: missing\|missing executable runtime\|missing or unreadable' "$SERVE"
    grep -q 'missing or unreadable target model' "$SERVE"
    grep -q 'missing or unreadable draft model' "$SERVE"
}