#!/usr/bin/env bash
# deploy-remote.sh — apply an explicit service plan on one target host.
#
# This script runs on the target checkout. It deliberately does not infer a
# plan, pull Git, or restart an unrelated service. deploy.sh computes the plan
# from the pushed revision and passes the selected service names here.
# Per-service revisions live in /var/lib/serverAppConfigs/deploy-state (or
# DEPLOY_STATE_DIR), outside the checkout, so an unselected change remains
# pending until its service is explicitly deployed.
set -euo pipefail

TARGET="${1:-}"
REVISION="${2:-}"
shift 2 || true
FROM_REVISION=""
PENDING=0
SERVICES=()

usage() {
    cat <<'EOF'
Usage: deploy-remote.sh <dankclaw|dankserver|frankbook-pro> <git-revision> [--pending] [service]...

The service list is explicit. An empty list is a successful no-op.
EOF
}

while [[ "$#" -gt 0 ]]; do
    case "$1" in
        --from-revision)
            [[ "$#" -ge 2 ]] || { echo "--from-revision requires a revision" >&2; exit 2; }
            FROM_REVISION="$2"
            shift 2
            ;;
        --pending)
            PENDING=1
            shift
            ;;
        --help|-h)
            usage
            exit 0
            ;;
        *)
            SERVICES+=("$1")
            shift
            ;;
    esac
done

REMOTE_DIR="${REMOTE_DIR:-/home/dank/serverAppConfigs}"
STATE_DIR="${DEPLOY_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/serverAppConfigs/deploy-state}"

run_privileged() {
    if [[ -n "${FRANK_SUDO_PW:-}" ]]; then
        printf '%s\n' "$FRANK_SUDO_PW" | sudo -S -- "$@"
    else
        sudo -- "$@"
    fi
}

if [[ "$TARGET" != dankclaw && "$TARGET" != dankserver && "$TARGET" != frankbook-pro ]]; then
    usage >&2
    exit 2
fi
if [[ -z "$REVISION" ]]; then
    usage >&2
    exit 2
fi
if [[ "$REVISION" == *[!0-9a-fA-F]* ]]; then
    echo "invalid Git revision: $REVISION" >&2
    exit 2
fi
if ! git -C "$REMOTE_DIR" cat-file -e "$REVISION^{commit}" 2>/dev/null; then
    echo "revision is not a commit in the target checkout: $REVISION" >&2
    exit 2
fi
cd "$REMOTE_DIR"

# Keep the state directory outside Git. The file is written only after a
# handler completes successfully; an interrupted handler remains pending.
mkdir -p "$STATE_DIR"

service_is_allowed() {
    local service="$1"
    case "$TARGET:$service" in
        dankclaw:gluetun|dankclaw:searxng-valkey|dankclaw:searxng|dankclaw:pihole|\
        dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
        dankclaw:comfyui|dankclaw:hermes|\
        dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
        dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:alertmanager|\
        dankclaw:authentik-postgres|dankclaw:authentik-server|\
        dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
        dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
        dankclaw:frank-large|dankclaw:frank-small|\
        dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
        dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:plex|dankserver:tautulli|\
        dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
        dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
        dankserver:frank-medium|dankserver:node-exporter|\
        frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
        frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
        frankbook-pro:readarr)
            return 0
            ;;
        *)
            return 1
            ;;
    esac
}

mark_status() {
    local service="$1" status="$2" tmp
    tmp="$(mktemp "$STATE_DIR/.${service}.XXXXXX")"
    printf 'revision=%s\nstatus=%s\n' "$REVISION" "$status" > "$tmp"
    mv -f -- "$tmp" "$STATE_DIR/$service"
}

mark_deployed() {
    mark_status "$1" deployed
}

state_revision() {
    local service="$1" line value
    if [[ -r "$STATE_DIR/$service" ]]; then
        while IFS= read -r line; do
            case "$line" in
                revision=*) value="${line#revision=}" ;;
            esac
        done < "$STATE_DIR/$service"
        printf '%s' "${value:-}"
    fi
}

state_status() {
    local service="$1" line value=pending
    if [[ -r "$STATE_DIR/$service" ]]; then
        while IFS= read -r line; do
            case "$line" in
                status=*) value="${line#status=}" ;;
            esac
        done < "$STATE_DIR/$service"
    fi
    printf '%s' "$value"
}

register_changed_services() {
    local base="$1" service recorded status
    [[ -n "$base" ]] || return 0
    while IFS= read -r service; do
        [[ -n "$service" ]] || continue
        recorded="$(state_revision "$service")"
        status="$(state_status "$service")"
        if [[ "$status" != deployed || "$recorded" != "$REVISION" ]]; then
            mark_status "$service" pending
        fi
    done < <(python3 scripts/deploy-plan.py --target "$TARGET" --repo "$REMOTE_DIR" \
        --base "$base" --head "$REVISION")
}

add_changed_services() {
    local base="$1"
    local service
    [[ -n "$base" ]] || return 0
    while IFS= read -r service; do
        [[ -n "$service" ]] || continue
        SERVICES+=("$service")
    done < <(python3 scripts/deploy-plan.py --target "$TARGET" --repo "$REMOTE_DIR" \
        --base "$base" --head "$REVISION")
}

add_pending_services() {
    local service recorded status changed
    for state_file in "$STATE_DIR"/*; do
        [[ -f "$state_file" ]] || continue
        service="${state_file##*/}"
        recorded="$(state_revision "$service")"
        status="$(state_status "$service")"
        if [[ "$status" != deployed ]]; then
            # A pending service was intentionally skipped or its handler
            # failed. Retry it even when the checkout has not advanced.
            # But a pending entry for a service RETIRED from this target can
            # never be deployable: retrying it fails the whole run before
            # the e2e gate (the frank-default-sightsound lockout, found
            # 2026-09-17). Drop such entries into the host trash instead.
            if ! service_is_allowed "$service"; then
                local trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-retired-deploy-state"
                mkdir -p "$trash"
                mv -- "$state_file" "$trash/$service"
                echo "WARN: $service is retired on $TARGET; dropped its stale $status state to $trash" >&2
                continue
            fi
            SERVICES+=("$service")
            continue
        fi
        [[ -n "$recorded" && "$recorded" != "$REVISION" ]] || continue
        if [[ "$TARGET" == dankserver && "$service" == frank-large ]]; then continue; fi
        if ! changed="$(python3 scripts/deploy-plan.py --target "$TARGET" \
            --repo "$REMOTE_DIR" --base "$recorded" --head "$REVISION")"; then
            echo "WARN: unable to compare state for $service; retaining it as pending" >&2
            SERVICES+=("$service")
            continue
        fi
        if grep -Fxq "$service" <<< "$changed"; then
            SERVICES+=("$service")
        fi
    done
}

copy_searxng_config() {
    run_privileged mkdir -p /opt/searxng/config
    run_privileged cp shared/searxng/settings.yml /opt/searxng/config/settings.yml
    run_privileged cp shared/searxng/limiter.toml /opt/searxng/config/limiter.toml
}

copy_prometheus_config() {
    run_privileged mkdir -p /opt/prometheus/config /opt/prometheus/config/rules
    run_privileged cp shared/prometheus/prometheus.yml /opt/prometheus/config/prometheus.yml
    # Rule files are read from the same mount (--config.file sits beside them),
    # so they deploy with the config that references them: a rule file without
    # its rule_files entry evaluates nothing, silently.
    run_privileged cp shared/prometheus/rules/llm-fleet.yml \
        /opt/prometheus/config/rules/llm-fleet.yml
}

# Alertmanager config ships beside the Prometheus config: the two halves of
# the alerting path change together, and a receiver with no route (or a
# route with no receiver) fails silently.
copy_alertmanager_config() {
    run_privileged mkdir -p /opt/alertmanager/config /opt/alertmanager/data
    run_privileged cp shared/prometheus/alertmanager.yml /opt/alertmanager/config/alertmanager.yml
}

# The Signal relay is Alertmanager's only webhook consumer and the only
# thing on this host that talks to signal-cli. It is a plain systemd USER
# unit (not a quadlet): it runs python3 from the checkout, so a deploy
# updates it without a copy step, and — unlike generated quadlet units,
# which this host's podman 5.x v1 generator rejects — a hand-written user
# unit can be enabled, and Linger=yes for dank starts it at boot.
install_signal_relay() {
    local unit_dir="$HOME/.config/systemd/user"
    mkdir -p "$unit_dir"
    cp scripts/signal-relay.service "$unit_dir/signal-relay.service"
    systemctl --user daemon-reload
    systemctl --user enable signal-relay.service
    systemctl --user restart signal-relay.service
    # The relay binds the host's LAN IP (the one address a bridge container
    # can reach as host.containers.internal), so keep the unauthenticated
    # endpoint off the rest of the LAN: admit loopback, the host itself
    # (its own health probe) and the podman bridge subnet only. Idempotent:
    # -C before -I.
    run_privileged iptables -C INPUT -p tcp --dport 18093 -s 127.0.0.1 -j ACCEPT 2>/dev/null \
        || run_privileged iptables -I INPUT -p tcp --dport 18093 -s 127.0.0.1 -j ACCEPT
    run_privileged iptables -C INPUT -p tcp --dport 18093 -s 192.168.0.143 -j ACCEPT 2>/dev/null \
        || run_privileged iptables -I INPUT -p tcp --dport 18093 -s 192.168.0.143 -j ACCEPT
    run_privileged iptables -C INPUT -p tcp --dport 18093 -s 10.89.0.0/16 -j ACCEPT 2>/dev/null \
        || run_privileged iptables -I INPUT -p tcp --dport 18093 -s 10.89.0.0/16 -j ACCEPT
    run_privileged iptables -C INPUT -p tcp --dport 18093 -j DROP 2>/dev/null \
        || run_privileged iptables -I INPUT -p tcp --dport 18093 -j DROP
    # The interpreter needs a moment to bind after restart; a one-shot curl
    # raced the socket on 2026-09-17 and failed a healthy deploy.
    local i
    for i in $(seq 1 15); do
        if curl -fsS -m 5 http://192.168.0.143:18093/health >/dev/null; then
            return 0
        fi
        sleep 1
    done
    echo "signal-relay is not answering on 192.168.0.143:18093" >&2
    return 1
}

copy_loki_config() {
    run_privileged mkdir -p /opt/loki/config
    run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
}

copy_alloy_config() {
    run_privileged mkdir -p /opt/alloy/config
    run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy
}

copy_grafana_provisioning() {
    # Datasources and dashboards are declared in git, never clicked into
    # grafana.db: this Grafana is an empty install, and a UI-configured
    # datasource would be invisible to review, lost with the data volume, and
    # unknown to anyone reading the repository. The unit already mounts
    # /opt/grafana/provisioning read-only, so this only has to populate it.
    run_privileged mkdir -p /opt/grafana/provisioning/datasources \
        /opt/grafana/provisioning/dashboards
    run_privileged cp shared/grafana/provisioning/datasources/prometheus.yml \
        /opt/grafana/provisioning/datasources/prometheus.yml
    run_privileged cp shared/grafana/provisioning/dashboards/provider.yml \
        /opt/grafana/provisioning/dashboards/provider.yml
    run_privileged cp shared/grafana/provisioning/dashboards/llm-fleet.json \
        /opt/grafana/provisioning/dashboards/llm-fleet.json
}

render_caddyfile() {
    local hash
    hash="$(grep -E '^CADDY_BASICAUTH_HASH=' .env 2>/dev/null | tail -1 | cut -d= -f2- || true)"
    if [[ -z "$hash" ]]; then
        echo "WARN: CADDY_BASICAUTH_HASH is not set; refusing to overwrite rendered Caddyfile" >&2
        return 1
    fi
    run_privileged mkdir -p /home/dank/Containers/caddy
    sed -e "s|REPLACE_WITH_HASH|$hash|g" dankclaw/Caddyfile > /tmp/serverAppConfigs.Caddyfile
    run_privileged cp /tmp/serverAppConfigs.Caddyfile /home/dank/Containers/caddy/Caddyfile
}

deploy_homepage() {
    if [[ "$TARGET" != dankclaw ]]; then
        echo "homepage is owned by dankclaw; refusing to deploy it on $TARGET" >&2
        return 2
    fi
    run_privileged mkdir -p /srv/homepage
    run_privileged cp shared/homepage/index.html shared/homepage/admin.html shared/homepage/app.js \
        shared/homepage/style.css shared/homepage/services.json shared/homepage/docs.html /srv/homepage/
    if [[ -d shared/homepage/fonts ]]; then
        run_privileged cp -r shared/homepage/fonts /srv/homepage/
    fi
    local deployed_at
    deployed_at="$(git log -1 --format=%cI)"
    printf 'window.DEPLOYED_AT = "%s";\n' "$deployed_at" > /tmp/deployed.js
    run_privileged cp /tmp/deployed.js /srv/homepage/deployed.js
    run_privileged chmod -R a+rX /srv/homepage
    run_privileged cp scripts/homepage-status.py /usr/local/bin/homepage-status.py
    run_privileged cp scripts/homepage-status.service scripts/homepage-status.timer /etc/systemd/system/
    run_privileged systemctl daemon-reload
    run_privileged systemctl enable homepage-status.timer
    run_privileged systemctl restart homepage-status.timer
    run_privileged python3 /usr/local/bin/homepage-status.py /srv/homepage/services.json /srv/homepage/status.json
}

stage_legacy_gitea_caddy_state() {
    local legacy_state="$STATE_DIR/gitea-caddy"
    [[ -e "$legacy_state" ]] || return 0
    local legacy_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-gitea-caddy"
    mkdir -p "$legacy_trash"
    mv -- "$legacy_state" "$legacy_trash/gitea-caddy.state"
}

retire_legacy_gitea_caddy() {
    # The old split proxy owns ports 80/443 on hosts upgraded in place. Stop
    # and disable its generated unit before the canonical Caddy is started.
    if systemctl --user is-active --quiet gitea-caddy.service 2>/dev/null; then
        systemctl --user stop gitea-caddy.service
    fi
    if systemctl --user is-enabled --quiet gitea-caddy.service 2>/dev/null; then
        # A Quadlet-generated unit may report "generated" without having an
        # enablement symlink. Disable is therefore best-effort; moving the
        # source out of the generator directory is the durable action.
        systemctl --user disable gitea-caddy.service 2>/dev/null || true
    fi
    local unit_dir="$HOME/.config/containers/systemd"
    local legacy_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-gitea-caddy"
    # Podman generators scan the unit directory recursively on this host, so
    # leaving the retired file in systemd/.AGENTTRASH would regenerate it.
    # Stage it outside the unit tree in the dated home trash instead.
    for unit_file in "$unit_dir/gitea-caddy.container" \
        "$unit_dir/.AGENTTRASH/gitea-caddy.container"; do
        if [[ -e "$unit_file" ]]; then
            mkdir -p "$legacy_trash"
            mv -- "$unit_file" "$legacy_trash/"
        fi
    done
    stage_legacy_gitea_caddy_state
    systemctl --user daemon-reload
    if podman container exists gitea-caddy; then
        podman rm --force gitea-caddy
    fi
}

deploy_caddy() {
    if [[ "$TARGET" != dankclaw ]]; then
        echo "caddy is owned by dankclaw; refusing to deploy it on $TARGET" >&2
        return 2
    fi
    render_caddyfile
    deploy_homepage
    local cfhash oldhash image
    cfhash="$(sha256sum dankclaw/caddy/Containerfile | cut -d' ' -f1)"
    image=franksdank/caddy-porkbun
    oldhash="$(podman image inspect "$image:latest" --format '{{index .Config.Labels "cfhash"}}' 2>/dev/null || echo none)"
    if [[ "$oldhash" != "$cfhash" ]]; then
        podman build --no-cache --label "cfhash=$cfhash" -t "$image:latest" \
            -f dankclaw/caddy/Containerfile .
    fi
    # Retire the old split proxy before claiming ports 80/443. This is
    # idempotent and leaves the old unit recoverable in the target trash.
    retire_legacy_gitea_caddy
    # Caddy's env: Porkbun DNS-01 credentials from the checkout's .env. The
    # live file holds secrets, so it exists only on the host; the quadlet
    # reads it via EnvironmentFile.
    mkdir -p /home/dank/Containers/caddy
    if [[ ! -f /home/dank/Containers/caddy/env ]]; then
        awk -F= '/^PORKBUN_API_KEY=|^PORKBUN_API_SECRET_KEY=/ { print }' \
            dankclaw/.env > /home/dank/Containers/caddy/env
    fi
    local var
    for var in PORKBUN_API_KEY PORKBUN_API_SECRET_KEY; do
        grep -q "^${var}=." /home/dank/Containers/caddy/env \
            || { echo "caddy env missing ${var}; set it in dankclaw/.env" >&2; return 1; }
    done
    # Retire the bare compose container: all state (Caddyfile, certs, config)
    # lives in host volumes, so removing the container loses nothing.
    podman stop caddy >/dev/null 2>&1 || true
    podman rm caddy >/dev/null 2>&1 || true
    # install_quadlet runs daemon-reload, which re-runs podman's quadlet
    # generator: it creates caddy.service AND the default.target.wants symlink
    # from the unit's [Install] section (is-enabled reports "generated" —
    # that is the healthy state for a quadlet; an explicit systemctl enable
    # fails on generated units). Boot persistence comes from the generator,
    # same as pihole/gitea/searxng/open-webui.
    install_quadlet caddy dankclaw/caddy/quadlet caddy
    systemctl --user restart caddy.service
    if podman container exists caddy; then
        local reloaded=0
        for _ in 1 2 3 4 5; do
            if podman exec caddy caddy reload --config /etc/caddy/Caddyfile \
                --adapter caddyfile >/dev/null 2>&1; then
                reloaded=1
                echo "Caddy reloaded."
                break
            fi
            sleep 3
        done
        if [[ "$reloaded" -ne 1 ]]; then
            echo "Caddy reload failed; restarting only caddy."
            podman restart caddy
        fi
    fi
}

deploy_compose_service() {
    local service="$1"
    if [[ "$TARGET" == dankserver ]]; then
        # Dankserver's compose file is the source-of-truth inventory, but its
        # production containers are individually systemd-managed/bare. Do not
        # use compose up here: it can create a second container with a wrong
        # name and can interfere with a running workload.
        echo "WARN: $service is recorded in dankserver/compose.yaml; production container management is host-specific and no restart was attempted."
        return 0
    fi
    if [[ "$TARGET" == dankclaw ]]; then
        # Dankclaw has no compose file since 2026-09-16: every service is a
        # Quadlet unit with its own handler above. Reaching this means the
        # dispatcher is missing a case, and the old fallback (`podman-compose
        # up -d`) would now be a silent no-op or a duplicate container.
        echo "no compose services on dankclaw; $service has no quadlet handler" >&2
        return 2
    fi
    case "$service" in
        *)
            echo "unknown compose service: $service" >&2
            return 2
            ;;
    esac
}

# grafana is the one observability service that holds a credential: the
# Authentik OAuth2 client secret. It is seeded into a per-service env file (the
# fleet convention: caddy, pihole, copyparty, romm, seerr) instead of being
# imported from the repo .env wholesale. The repo-root .env that the compose
# service used to pick up is a gluetun env file — it also carries HOME=/root
# and the WireGuard keys, and an EnvironmentFile of that file sets HOME for the
# service, which makes podman look for its config under /root and fail.
seed_grafana_env() {
    local dest=/home/dank/Containers/grafana/env
    mkdir -p /home/dank/Containers/grafana
    if [[ ! -f "$dest" ]]; then
        awk -F= '/^GRAFANA_OAUTH_CLIENT_SECRET=/ { print }' dankclaw/.env > "$dest"
        chmod 600 "$dest"
    fi
    grep -q '^GRAFANA_OAUTH_CLIENT_SECRET=.' "$dest" \
        || { echo "grafana env missing GRAFANA_OAUTH_CLIENT_SECRET; set it in dankclaw/.env" >&2; return 1; }
}

# The five observability containers left compose on 2026-09-11 (they were
# started by hand with podman-compose and never came back after a reboot).
# They are ordinary rootless Quadlet units now: same images, same mounts, same
# arguments, same container names, so nothing that addresses them by name —
# Caddy, Prometheus' own scrape config, Alloy's loki.write endpoint — changed.
# Only the published ports did: the three HTTP UIs bind 127.0.0.1, because the
# compose services published on 0.0.0.0 and none of them authenticates.
deploy_observability() {
    local service="$1"
    # Observability units live under <target>/observability/quadlet: Dankclaw
    # has six, Dankserver one (node-exporter, since 2026-09-16). The config
    # copies below are Dankclaw-owned state and run only there.
    local unit_dir="$TARGET/observability/quadlet"
    case "$service" in
        prometheus)
            copy_prometheus_config
            ;;
        alertmanager)
            copy_alertmanager_config
            install_signal_relay
            ;;
        loki)
            copy_loki_config
            build_compose_image loki
            ;;
        alloy)
            copy_alloy_config
            build_compose_image alloy
            ;;
        grafana)
            copy_grafana_provisioning
            seed_grafana_env || return 1
            ;;
        node-exporter)
            ;;
        *)
            echo "unknown observability service: $service" >&2
            return 2
            ;;
    esac
    install_quadlet observability "$unit_dir" "$service"
    systemctl --user restart "$service.service"
    case "$service" in
        prometheus) wait_for_health prometheus http://127.0.0.1:9090/-/healthy 60 ;;
        alertmanager) wait_for_health alertmanager http://127.0.0.1:9093/-/healthy 60 ;;
        grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;;
        loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
        alloy) wait_for_health alloy http://127.0.0.1:12345/-/healthy 60 ;;
        node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;;
    esac
}

build_compose_image() {
    local service="$1" image cfhash oldhash containerfile
    case "$service" in
        comfyui)
            image=comfyui-rocm:gfx1151
            containerfile=dankclaw/Containerfile.comfyui
            ;;
        loki|alloy)
            image="franksdank/$service:latest"
            containerfile="dankclaw/$service/Containerfile"
            ;;
        *)
            return 0
            ;;
    esac
    cfhash="$(sha256sum "$containerfile" | cut -d' ' -f1)"
    oldhash="$(podman image inspect "$image" --format '{{index .Config.Labels "cfhash"}}' 2>/dev/null || echo none)"
    if [[ "$oldhash" != "$cfhash" ]]; then
        # --format docker is required, not cosmetic: podman defaults to the OCI
        # image format, which has no HEALTHCHECK field at all, so a build in
        # the default format silently drops the loki image's healthcheck and
        # the container then reports no health instead of failing loudly.
        podman build --format docker --no-cache --label "cfhash=$cfhash" \
            -t "$image" -f "$containerfile" .
    fi
}

quadlet_unit_dir="$HOME/.config/containers/systemd"
# dankserver's user manager had no systemd container generator dir before the
# quadlet migration; every handler that installs a unit must find it present.
mkdir -p "$quadlet_unit_dir"
install_quadlet() {
    local service="$1"
    local source_dir="$2"
    local unit="$3"
    mkdir -p "$quadlet_unit_dir"
    cp "$source_dir/$unit.container" "$quadlet_unit_dir/"
    if [[ -f "$source_dir/$unit.network" ]]; then
        cp "$source_dir/$unit.network" "$quadlet_unit_dir/"
    fi
    systemctl --user daemon-reload
}

deploy_searxng() {
    copy_searxng_config
    install_quadlet searxng dankclaw/searxng/quadlet searxng
    install_quadlet searxng-valkey dankclaw/searxng/quadlet searxng-valkey
    install_quadlet gluetun dankclaw/searxng/quadlet gluetun
    # The network is shared by all three units; install it even when only the
    # SearXNG service was selected. These copies are user-owned Quadlet state.
    cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
    systemctl --user daemon-reload
    systemctl --user start searxng-network.service
    systemctl --user restart gluetun.service searxng-valkey.service searxng.service
}

deploy_gluetun() {
    install_quadlet gluetun dankclaw/searxng/quadlet gluetun
    cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
    systemctl --user daemon-reload
    systemctl --user restart gluetun.service searxng.service
}

deploy_searxng_valkey() {
    install_quadlet searxng-valkey dankclaw/searxng/quadlet searxng-valkey
    cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
    systemctl --user daemon-reload
    systemctl --user restart searxng-valkey.service searxng.service
}

# Marreta's secrets are seeded into a per-service env file under the deploy
# user's home (rootless-readable), following the grafana convention rather
# than the /opt/searxng one, so the rootless unit can read it without any
# ownership gymnastics. Only the three MARRETA_* values are copied; the
# quadlet maps them to the app's APP_KEY / ADMIN_EMAIL / ADMIN_PASSWORD.
seed_marreta_env() {
    local dest=/home/dank/Containers/marreta/env
    mkdir -p /home/dank/Containers/marreta
    if [[ ! -f "$dest" ]]; then
        awk -F= '/^MARRETA_APP_KEY=/ || /^MARRETA_ADMIN_EMAIL=/ || /^MARRETA_ADMIN_PASSWORD=/ { print }' \
            dankclaw/.env > "$dest"
        chmod 600 "$dest"
    fi
    local var
    for var in MARRETA_APP_KEY MARRETA_ADMIN_EMAIL MARRETA_ADMIN_PASSWORD; do
        grep -q "^${var}=." "$dest" \
            || { echo "marreta env missing ${var}; set it in dankclaw/.env" >&2; return 1; }
    done
}

deploy_marreta() {
    seed_marreta_env
    install_quadlet marreta dankclaw/marreta/quadlet marreta
    install_quadlet marreta-browser dankclaw/marreta/quadlet marreta-browser
    systemctl --user daemon-reload
    # Both containers share gluetun's netns; the tunnel must be up first.
    systemctl --user start gluetun.service
    # Browser before app: marreta drives the CDP endpoint on loopback.
    systemctl --user restart marreta-browser.service
    systemctl --user restart marreta.service
}

deploy_open_webui() {
    install_quadlet open-webui dankclaw/open-webui/quadlet open-webui
    mkdir -p /home/dank/Containers/open-webui/data
    systemctl --user daemon-reload
    systemctl --user restart open-webui.service
}

deploy_comfyui() {
    # Build first: the unit's Image= resolves against local storage, so the
    # image has to exist before the generator starts a container from it.
    build_compose_image comfyui
    install_quadlet comfyui dankclaw/comfyui/quadlet comfyui
    # Retire the compose-era unit. It owns the same unit name the Quadlet
    # generator produces, and it is what kept a stale, device-less container
    # alive through `restart: unless-stopped` (9,146 failures on
    # "No HIP GPUs are available"). Never `rm`: it moves into the home trash.
    local legacy="$HOME/.config/systemd/user/comfyui.service"
    if [[ -e "$legacy" ]]; then
        local trash="$HOME/.AGENTTRASH/20260916-comfyui-compose-unit"
        mkdir -p "$trash"
        systemctl --user stop comfyui.service 2>/dev/null || true
        systemctl --user disable comfyui.service 2>/dev/null || true
        mv -- "$legacy" "$trash/comfyui.service"
        echo "retired the compose-era comfyui.service -> $trash"
    fi
    # The stale container carries the old config (no /dev/kfd, no /dev/dri) and
    # would keep the quadlet's container from taking the name. It is replaced
    # by the unit in the next step, so nothing is lost.
    podman rm -f comfyui >/dev/null 2>&1 || true
    systemctl --user daemon-reload
    systemctl --user restart comfyui.service
    wait_for_health comfyui http://127.0.0.1:8188/ 300
}

deploy_pihole() {
    # Rootless port 53 needs net.ipv4.ip_unprivileged_port_start <= 53, and it
    # must hold at boot. The repo drop-in is the single source of truth.
    # systemd-sysctl applies /etc/sysctl.d/*.conf (lexicographic), then
    # /run/sysctl.d, /usr/lib/sysctl.d, and finally /etc/sysctl.conf — later
    # files override earlier ones. On 2026-08-26 a later-sorted drop-in (=80)
    # AND a stale line in /etc/sysctl.conf (=80) both reset the value after a
    # reboot and took LAN DNS down. Neutralize every conflicting source:
    # stage sysctl.d files that set a value above 53 in the home trash, and
    # comment out an active line in /etc/sysctl.conf (idempotent).
    run_privileged install -m 0644 dankclaw/pihole/sysctl/99-dnsmasq.conf \
        /etc/sysctl.d/99-dnsmasq.conf
    local sysctl_file sysctl_val sysctl_trash
    while IFS= read -r sysctl_file; do
        [[ "$sysctl_file" == */99-dnsmasq.conf ]] && continue
        sysctl_val="$(sed -nE 's/^[[:space:]]*net\.ipv4\.ip_unprivileged_port_start[[:space:]]*=[[:space:]]*([0-9]+).*/\1/p' "$sysctl_file" | head -1)"
        if [[ "$sysctl_val" =~ ^[0-9]+$ ]] && (( sysctl_val > 53 )); then
            sysctl_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-sysctl-unprivileged-port"
            mkdir -p "$sysctl_trash"
            run_privileged mv "$sysctl_file" "$sysctl_trash/"
        fi
    done < <(grep -lE '^[[:space:]]*net\.ipv4\.ip_unprivileged_port_start[[:space:]]*=' /etc/sysctl.d/*.conf 2>/dev/null || true)
    if [[ -f /etc/sysctl.conf ]] && grep -qE '^[[:space:]]*net\.ipv4\.ip_unprivileged_port_start[[:space:]]*=' /etc/sysctl.conf; then
        run_privileged sed -i -E 's/^[[:space:]]*(net\.ipv4\.ip_unprivileged_port_start[[:space:]]*=)/# \1 managed by dankclaw\/pihole\/sysctl\/99-dnsmasq.conf/' /etc/sysctl.conf
    fi
    run_privileged sysctl --system >/dev/null
    mkdir -p /home/dank/Containers/pihole/etc-pihole /home/dank/Containers/pihole/etc-dockers
    if [[ ! -f /home/dank/Containers/pihole/env ]]; then
        cp dankclaw/pihole/env.example /home/dank/Containers/pihole/env
        echo "WARN: created Pi-hole env from the example; set WEBPASSWORD before using its UI" >&2
    fi
    cp dankclaw/pihole/addn-hosts /home/dank/Containers/pihole/addn-hosts
    mkdir -p "$quadlet_unit_dir"
    cp dankclaw/pihole/quadlet/pihole.container "$quadlet_unit_dir/"
    systemctl --user daemon-reload
    systemctl --user restart pihole.service
}

deploy_gitea_unit() {
    local service="$1"
    local unit="$2"
    mkdir -p "$quadlet_unit_dir"
    cp "dankclaw/gitea/quadlet/$unit.container" "$quadlet_unit_dir/"
    [[ -f dankclaw/gitea/quadlet/gitea.network ]] && cp dankclaw/gitea/quadlet/gitea.network "$quadlet_unit_dir/"
    systemctl --user daemon-reload
    systemctl --user restart "$service.service"
}

relink_unit() {
    local service="$1"
    run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "/etc/systemd/system/$service.service"
}

# Authentik runs as three boot-persistent Quadlet units (postgres -> server ->
# worker). The trio deploys as one unit: unit files are installed, stale
# compose-era containers holding the claimed names are retired, then the chain
# starts in dependency order with health gates between stages. Re-running the
# handler on an already-active stack is a deliberate no-op (no DB bounce).
wait_container_healthy() {
    local name="$1" limit="$2" status="unknown"
    for ((i = 1; i <= limit; i += 5)); do
        status="$(podman inspect -f '{{.State.Health.Status}}' "$name" 2>/dev/null || echo missing)"
        [[ "$status" == "healthy" ]] && {
            echo "$name healthy after approximately ${i}s"
            return 0
        }
        sleep 5
    done
    echo "FAIL: $name did not become healthy in ${limit}s (status: $status)" >&2
    return 1
}

# --- dankserver media/library stack ----------------------------------------------
# These services migrated from compose-era bare containers to boot-persistent
# Quadlet units installed from this repo (podman 5.x v1 generator honors the
# WantedBy=default.target inside each .container file). The first deploy of a
# service retires the old container and migrates the data root before starting
# the unit; re-running the handler on an already-active unit restarts only the
# selected unit.

migrate_data_root() { # $1=old /opt path, $2=new data root, $3=chown|keep (default chown)
    local from="$1" to="$2" mode="${3:-chown}" entry sub base
    [[ -d "$from" ]] || return 0
    # Renaming a directory needs WRITE permission on the directory being
    # renamed itself. Compose-era data roots contain directories owned by
    # the containers' mapped host uids (mariadb/redis land in the subuid
    # range as 100998), which the deploy user cannot rename. Privileged mv
    # preserves numeric ownership — the rootless mapping contract.
    mkdir -p "$to"
    shopt -s dotglob
    for entry in "$from"/*; do
        [[ -e "$entry" || -L "$entry" ]] || continue
        base="${entry##*/}"
        if [[ -e "$to/$base" ]]; then
            # Collides with pre-seeded state (env file) or a rescued
            # anonymous-volume stub: merge one level deep, never overwriting
            # an existing entry.
            if [[ -d "$entry" && -d "$to/$base" ]]; then
                for sub in "$entry"/*; do
                    [[ -e "$sub" || -L "$sub" ]] || continue
                    run_privileged mv -n -- "$sub" "$to/$base/"
                done
                run_privileged rmdir -- "$entry" 2>/dev/null || \
                    echo "WARN: $entry not empty after merge; left in place" >&2
            else
                echo "WARN: collision on $to/$base kept existing; $entry left in place" >&2
            fi
            continue
        fi
        run_privileged mv -- "$entry" "$to/"
    done
    shopt -u dotglob
    run_privileged rmdir -- "$from" 2>/dev/null || \
        echo "WARN: $from not empty after migration; left in place" >&2
    if [[ "$mode" == chown ]]; then
        chown -R "$(id -u):$(id -g)" "$to"
    fi
}

deploy_romm_stack() {
    local service="$1" unit vol tmp
    case "$service" in
        mariadb) service=romm-mariadb ;;
        redis) service=romm-redis ;;
    esac
    for unit in romm-mariadb romm-redis romm; do
        cp "dankserver/romm/quadlet/$unit.container" "$quadlet_unit_dir/"
    done
    systemctl --user daemon-reload
    if ! systemctl --user is-active --quiet romm-mariadb.service; then
        # First cutover: the env file must exist before the first start, or
        # the units would boot with empty secrets. Fail before stopping anything.
        [[ -f /home/dank/Containers/romm/env ]] || {
            echo "FAIL: /home/dank/Containers/romm/env does not exist; seed it before the first romm deploy" >&2
            return 1
        }
        # Capture anonymous volume names while the containers are still inspectable,
        # then stop and rm the compose-era trio.
        local vols
        vols="$(podman inspect -f '{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}} {{end}}{{end}}' romm 2>/dev/null || true)"
        podman stop -t 60 romm-mariadb romm-redis romm || true
        podman rm romm romm-redis romm-mariadb || true
        for vol in $vols; do
            tmp="$(mktemp -d)"
            podman volume export --output "$tmp/v.tar" "$vol"
            # --skip-old-files keeps live bind-mounted content (config/, library/,
            # resources/, mariadb/, redis/) and rescues only unshadowed entries like assets/.
            tar --skip-old-files -C /home/dank/Containers/romm -xf "$tmp/v.tar"
            rm -rf -- "$tmp"
        done
        migrate_data_root /opt/romm /home/dank/Containers/romm keep
        systemctl --user start romm-mariadb.service
        wait_container_healthy romm-mariadb 120 || return 1
        systemctl --user start romm-redis.service
        wait_container_healthy romm-redis 60 || return 1
        if [[ "$service" == "romm" ]]; then
            systemctl --user start romm.service
        fi
        return 0
    fi
    systemctl --user restart "$service.service"
}

deploy_jellyfin() {
    cp "dankserver/jellyfin/quadlet/jellyfin.container" "$quadlet_unit_dir/"
    systemctl --user daemon-reload
    if ! systemctl --user is-active -q jellyfin.service; then
        podman stop -t 30 jellyfin || true
        podman rm jellyfin || true
        migrate_data_root /opt/jellyfin /home/dank/Containers/jellyfin
    fi
    systemctl --user restart jellyfin.service
}

deploy_calibre_web() {
    cp "dankserver/calibre-web/quadlet/calibre-web.container" "$quadlet_unit_dir/"
    systemctl --user daemon-reload
    if ! systemctl --user is-active -q calibre-web.service; then
        # The compose-era container ran under a podman-generate-systemd unit; stop
        # that unit first so its Restart=always cannot resurrect the old container
        # and collide with the new ContainerName.
        systemctl --user stop container-calibre-web.service 2>/dev/null || true
        podman stop calibre-web || true
        podman rm calibre-web || true
    fi
    systemctl --user restart calibre-web.service
}

deploy_copyparty() {
    # The admin account is created from ${COPAY_ACCOUNT}:${COPAY_PASSWORD} expanded
    # at ExecStart time; a missing env file must fail the unit, not boot copyparty
    # with a garbage admin password.
    [[ -f /home/dank/Containers/copyparty/env ]] || {
        echo "FAIL: /home/dank/Containers/copyparty/env does not exist; seed it before the first copyparty deploy" >&2
        return 1
    }
    cp "dankserver/copyparty/quadlet/copyparty.container" "$quadlet_unit_dir/"
    systemctl --user daemon-reload
    if ! systemctl --user is-active -q copyparty.service; then
        podman stop -t 15 copyparty || true
        podman rm copyparty || true
        migrate_data_root /opt/copyparty /home/dank/Containers/copyparty
    fi
    systemctl --user restart copyparty.service
}

deploy_plex() {
    cp "dankserver/plex/quadlet/plex.container" "$quadlet_unit_dir/"
    systemctl --user daemon-reload
    if ! systemctl --user is-active -q plex.service; then
        # First cutover: retire the native package unit BEFORE starting the
        # container — both cannot own 32400. Mask (not disable): on these hosts
        # `systemctl disable` deletes the unit file itself; mask keeps a
        # one-command rollback while nothing is signed off yet.
        run_privileged systemctl stop plexmediaserver.service
        run_privileged systemctl mask plexmediaserver.service
        # Rootless identity model: container root maps to the dank user
        # (uid 1000) — real uid 997 is not mappable rootless. Adopt the data
        # root into the fleet convention (jellyfin et al. run as 1000:1000).
        # Rollback chowns it back to 997:984 before restarting the native unit.
        run_privileged chown -R 1000:1000 /var/lib/plexmediaserver
    fi
    systemctl --user restart plex.service
    # Readiness gate: /identity answers MediaContainer XML once PMS is up.
    # Reality check from the 2026-09 cutover: the FIRST container boot of a
    # years-old library ran ~10 min (PMS pre-binds 32400 during boot and
    # empty-replies until the HTTP layer is ready), so the gate allows 600s.
    # Fail the deploy (pending state preserved) rather than mark it deployed blind.
    for ((i = 1; i <= 120; i++)); do
        curl -fsS --max-time 3 http://127.0.0.1:32400/identity >/dev/null && return 0
        sleep 5
    done
    echo "FAIL: plex did not answer /identity within 600s" >&2
    return 1
}

deploy_tautulli() {
    cp "dankserver/tautulli/quadlet/tautulli.container" "$quadlet_unit_dir/"
    systemctl --user daemon-reload
    if ! systemctl --user is-active -q tautulli.service; then
        run_privileged snap stop tautulli || true
        run_privileged snap disable tautulli || true
        mkdir -p /home/dank/Containers/tautulli/config
        if [[ ! -f /home/dank/Containers/tautulli/config/config.ini ]]; then
            printf '[General]\nPMS_Web_Host = plex\nPMS_Web_Port = 32400\nPMS_Secure = 0\n' \
                > /home/dank/Containers/tautulli/config/config.ini
        fi
    fi
    systemctl --user restart tautulli.service
}

deploy_authentik() {
    local unit
    mkdir -p "$quadlet_unit_dir"
    for unit in authentik-postgres authentik-server authentik-worker; do
        cp "dankclaw/authentik/quadlet/$unit.container" "$quadlet_unit_dir/"
    done
    systemctl --user daemon-reload
    # Retire compose-era containers only while the units are not yet active;
    # unit-managed containers self-heal via Restart=always.
    if ! systemctl --user is-active --quiet authentik-postgres.service; then
        podman stop authentik-postgres authentik-server authentik-worker 2>/dev/null || true
        podman rm authentik-postgres authentik-server authentik-worker 2>/dev/null || true
    fi
    # No `systemctl --user enable` here: this host's quadlet generator
    # (podman 5.x v1) generates units into /run/user/1000/systemd/generator/,
    # and generated units cannot be enabled — enable fails with "transient or
    # generated". Boot persistence comes from [Install] WantedBy=default.target
    # inside each .container file, which the generator honors at user-manager
    # start (linger must stay on for the deploy user).
    systemctl --user start authentik-postgres.service
    wait_container_healthy authentik-postgres 120 || return 1
    systemctl --user start authentik-server.service
    wait_container_healthy authentik-server 300 || return 1
    systemctl --user start authentik-worker.service
}

deploy_authentik_outpost() {
    # The outpost is the worker-role binary of the same server image; it dials
    # back to core over websocket and serves /outpost.goauthentik.io/* for
    # Caddy's forward_auth. Image already in storage — zero pulls. Restarting
    # only this unit cycles auth for guarded vhosts for a few seconds; the
    # rest of the stack is untouched.
    cp dankclaw/authentik/quadlet/authentik-outpost.container "$quadlet_unit_dir/"
    systemctl --user daemon-reload
    systemctl --user restart authentik-outpost.service
    sleep 5
    systemctl --user is-active --quiet authentik-outpost.service || \
        echo "WARN: authentik-outpost did not come up active" >&2
}

deploy_authentik_outpost_ldap() {
    # Dedicated LDAP outpost for Jellyfin's LDAP-Auth plugin. The type-specific
    # ghcr.io/goauthentik/ldap image serves 3389/6636 container-side, published
    # as host 389/636 for cross-host binds from dankserver. The outpost token
    # lives in the host .env, never in this repo. Restarting only this unit
    # cycles LDAP auth for a few seconds; the rest of the stack is untouched.
    cp dankclaw/authentik/quadlet/authentik-outpost-ldap.container "$quadlet_unit_dir/"
    systemctl --user daemon-reload
    systemctl --user restart authentik-outpost-ldap.service
    sleep 5
    systemctl --user is-active --quiet authentik-outpost-ldap.service || \
        echo "WARN: authentik-outpost-ldap did not come up active" >&2
}

deploy_hermes() {
    # The hermes trio (gateway, dashboard, webui) shares one image and the
    # /data volume and deploys as one unit: install the units, retire the
    # compose-era containers holding the claimed names while the units are not
    # yet active, then restart each unit. An explicit hermes deploy is a
    # deliberate action, so restarting an already-running trio is intended.
    # Boot persistence comes from [Install] in each unit file (generated units
    # cannot be enabled by this host's v1 quadlet generator).
    local unit
    mkdir -p "$quadlet_unit_dir"
    # The CDP browser daemon is a companion service to the trio: install its
    # unit and start it if inactive, but never restart it on deploy — a
    # restart would kill in-flight browser sessions. Consumers (gateway,
    # webui) reconnect with backoff.
    cp "dankclaw/hermes/quadlet/chrome-devtools-protocol.container" "$quadlet_unit_dir/"
    systemctl --user daemon-reload
    if ! systemctl --user is-active --quiet chrome-devtools-protocol.service; then
        systemctl --user start chrome-devtools-protocol.service
    fi
    # The units run the image under a localhost/-aliased tag: this host's
    # quadlet path can fail to resolve registry-prefixed refs from local
    # storage at create time. Tag first, then (re)create via deploy.
    podman tag docker.io/nousresearch/hermes-agent:latest localhost/hermes-agent:latest 2>/dev/null || true
    for unit in hermes hermes-dashboard hermes-webui; do
        cp "dankclaw/hermes/quadlet/$unit.container" "$quadlet_unit_dir/"
    done
    # The generator must see the freshly-copied unit files before the
    # restarts below; without this reload the first deploy after a unit edit
    # restarts with the stale generated ExecStart (statfs ENOENT on renamed
    # Volume= sources).
    systemctl --user daemon-reload
    if ! systemctl --user is-active --quiet hermes.service; then
        podman stop hermes hermes-dashboard hermes-webui 2>/dev/null || true
        podman rm hermes hermes-dashboard hermes-webui 2>/dev/null || true
    fi
    systemctl --user restart hermes.service
    systemctl --user restart hermes-dashboard.service
    systemctl --user restart hermes-webui.service
}

deploy_anubis() {
    # Anubis is the bot-challenge proxy in front of the guarded vhosts.
    # The image lives on ghcr.io; pull once, then the unit runs it from
    # local storage (Pull=never keeps boots off the registry).
    if ! podman image exists ghcr.io/techarohq/anubis:v1.27.0; then
        podman pull ghcr.io/techarohq/anubis:v1.27.0
    fi
    # The unit runs the localhost/ alias (see the unit file comment): this
    # host's podman quadlet path cannot resolve registry-prefixed refs from
    # local storage at create time. Tag first, then (re)create via deploy.
    podman tag ghcr.io/techarohq/anubis:v1.27.0 localhost/techarohq/anubis:v1.27.0 2>/dev/null || true
    cp "dankclaw/anubis/quadlet/anubis.container" "$quadlet_unit_dir/"
    systemctl --user daemon-reload
    # No enable: generated units cannot be enabled on this host (v1 quadlet
    # generator); boot persistence comes from [Install] in the unit file.
    # restart, not start: policy.json is parsed at process start only (no
    # reload signal exists), so every redeploy must cycle the process.
    systemctl --user restart anubis.service
    wait_container_healthy anubis 60 || return 1
}

# frank-large is the CIRU v4.4.1 runtime (QSA window hotfix) serving
# jcbtc/Qwen3.8-Flash-CIRU-STRIX-Orca on the Strix Halo iGPU, as a ROOT
# unit since the 2026-09-19 cutover: the runtime is a host build against
# the private ROCm 10 SDK, so there is no container image to install and no
# generated unit to dodge. relink_unit points
# /etc/systemd/system/frank-large.service at the repo file and boot
# persistence is that unit's [Install] section.
#
# It replaces the halogen-flash-server quadlet, which served
# peonist-ai/halogen-qwen3.8-flash-next on this same iGPU since 2026-09-16.
# The quadlet is retired HERE (the unit file moves into the home trash,
# never `rm`), and the port gate is the same as the 2026-09-16 cutover:
# nothing starts until 7770 is released.
LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/Qwen3.8-Flash-CIRU-STRIX-Orca.gguf}"
LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/mtp/Qwen3.8-Flash-CIRU-STRIX-Orca-MTP-Q8_0.gguf}"
LLM_LARGE_PLE="${LLM_LARGE_PLE:-/home/dank/models/Qwen3.8-Flash-CIRU-STRIX-Orca/ple/ple.payload.bin}"

retire_halogen_large_quadlet() {
    local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
    systemctl --user stop frank-large.service 2>/dev/null || true
    if [[ -e "$quadlet_unit_dir/frank-large.container" ]]; then
        mkdir -p "$trash"
        mv -- "$quadlet_unit_dir/frank-large.container" "$trash/frank-large.container"
        echo "retired quadlet frank-large.container -> $trash"
    fi
    systemctl --user daemon-reload
}

# Wait until a TCP port has no socket on it at all -- listening, accepted, or
# lingering in TIME_WAIT. Rootless podman publishes through pasta, which binds
# the host port at container CREATE, and a stopped LLM runtime keeps its
# listener (and the TIME_WAIT residue of the connections it served) for a
# moment after the process exits. `sport` is the port in question on both the
# listener and the server side of every accepted connection.
wait_port_free() {
    local port="$1" limit="${2:-90}" i
    for ((i = 1; i <= limit; i++)); do
        if [[ -z "$(ss -Htan "sport = :$port" 2>/dev/null)" ]]; then
            return 0
        fi
        sleep 1
    done
    echo "FAIL: port $port is still held after ${limit}s:" >&2
    ss -tan "sport = :$port" >&2 || true
    return 1
}

deploy_llm_large() {
    # Refuse rather than restart-loop. A unit whose runtime or payload is
    # missing never becomes healthy, and a green deploy would leave the
    # endpoint dark with nothing in the deploy log to say why.
    local missing=0 f
    for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT" "$LLM_LARGE_PLE"; do
        if [[ ! -f "$f" ]]; then
            echo "ERROR: payload absent at $f" >&2
            echo "frank-large: fetch it: bash scripts/download-frank-large-orca.sh" >&2
            missing=1
        fi
    done
    [[ "$missing" -eq 0 ]] || return 1

    retire_halogen_large_quadlet
    # On a re-deploy the predecessor is THIS unit: stop it before waiting on
    # the port, or the wait deadlocks on the server it just stopped serving.
    # Then sweep a stray manual run of the SAME model -- that is how a bench
    # harness leaves the port busy. Match the model path, not the binary name:
    # frank-small on this host is also a llama-server and must not be killed.
    # The pattern never appears in the deploying shell's own command line, so
    # -f is safe here (it is not in `bash deploy-remote.sh`).
    run_privileged systemctl stop frank-large.service 2>/dev/null || true
    pkill -f 'Qwen3.8-Flash-CIRU-STRIX-Orca' 2>/dev/null || true
    sleep 2
    # Nothing may start until the predecessor has released the port. On
    # 2026-09-16 the cutover failed exactly here: the old server logged
    # "Deactivated successfully", the next engine tried to bind 7770 a second
    # later and reported "Address already in use". Gate on the port, not on
    # hope.
    wait_port_free 7770 || return 1
    relink_unit frank-large
    run_privileged systemctl daemon-reload
    run_privileged systemctl enable frank-large.service
    run_privileged systemctl restart frank-large.service
    wait_for_health frank-large http://127.0.0.1:7770/health 1200
}

# deploy_llm_small — frank-small is mainline llama.cpp on the RTX 3090, as a
# ROOT unit: the runtime is a native CUDA binary, so there is no container
# image to install and no generated unit to avoid. relink_unit points
# /etc/systemd/system/frank-small.service at the repo file and boot persistence
# is that unit's [Install] section.
#
# It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
# not shared: frank-large's engine runs on the gfx1151 iGPU.
LLM_SMALL_MODEL="${LLM_SMALL_MODEL:-/home/dank/models/Gemma4-12B-QAT/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf}"

deploy_llm_small() {
    if [[ ! -f "$LLM_SMALL_MODEL" ]]; then
        echo "ERROR: model absent at $LLM_SMALL_MODEL" >&2
        echo "frank-small: fetch HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced (Q4_K_M, the mmproj and the MTP sidecar) first;" >&2
        echo "frank-small: refusing to start a unit that cannot load." >&2
        return 1
    fi
    relink_unit frank-small
    run_privileged systemctl daemon-reload
    run_privileged systemctl enable frank-small.service
    run_privileged systemctl restart frank-small.service
    wait_for_health frank-small http://127.0.0.1:7771/health 900
}

# deploy_llm_medium — frank-medium is the vllm-radiance MXFP4 stack as a
# rootless Quadlet since the 2026-09-20 cutover: one tensor-parallel (TP=2)
# engine across both R9700s, R4D one-shot all-reduce, DFlash2-FP8 drafter.
# It replaced mainline llama.cpp's layer split, which held the endpoint since
# 2026-09-16; the conversation battery measured radiance at 2.1x decode and a
# third of the time-to-first-token with 4x the concurrent throughput
# (scripts/bench-conversation.py, 2026-09-20).
#
# WHY THE PAIR IS BACK AND THE TRIO IS NOT: the 2026-09-16 cutover retired this
# unit because TP=2 could not dispatch a collective on the second card (B650
# chipset bridges, AtomicOpsCap: Routing-), and the two-replica + routing-proxy
# trio was the workaround. The board swap cleared the atomics wall, so one
# TP=2 process is both possible and the fastest shape; the replicas and their
# affinity proxy stay retired.
#
# HOST ASSETS. Nothing here installs the image's payloads: the checkpoints, the
# pinned radiance patch clone, the patched libr4d build and the Triton compile
# cache live outside the checkout (see the unit file's mount comments). The
# gates below refuse a deploy that would start a container guaranteed to die.
LLM_MEDIUM_IMAGE="${LLM_MEDIUM_IMAGE:-docker.io/stilldeadcode/vllm-radiance:0.9.3}"
LLM_MEDIUM_IMAGE_LOCAL="${LLM_MEDIUM_IMAGE_LOCAL:-localhost/radiance-vllm:0.9.3}"
LLM_MEDIUM_SNAP="${LLM_MEDIUM_SNAP:-/home/dank/models/Qwen3.8-27B-Uncensored-MXFP4-awq}"
LLM_MEDIUM_DRAFTER="${LLM_MEDIUM_DRAFTER:-/home/dank/models/Qwen3.8-27B-DFlash2-FP8}"
LLM_MEDIUM_PATCHES="${LLM_MEDIUM_PATCHES:-/home/dank/GitPackages/radiance-vllm-mxfp4}"
LLM_MEDIUM_R4D="${LLM_MEDIUM_R4D:-/home/dank/.cache/radiance-libr4d/b9e42ab-rx6}"
LLM_MEDIUM_CACHE="${LLM_MEDIUM_CACHE:-/home/dank/.radiance-cache-w4a8-093-gdnm-nqft-fp8s-gnq}"
 
# The three replica/proxy user units from the 2026-09-16 era are retired HERE,
# never left to rot: they must not exist alongside the TP=2 unit (the proxy
# published 7770 on the host network). Never `rm` — unit files move into the
# home trash; the images stay in local storage. `frank-medium` itself is NOT in
# the list: it is the unit this handler installs.
retire_vllm_replica_quadlets() {
    local trash="$HOME/.AGENTTRASH/20260916-vllm-radiance-retire" unit
    systemctl --user stop frank-medium-a.service frank-medium-b.service \
        frank-medium-proxy.service 2>/dev/null || true
    for unit in frank-medium-a frank-medium-b frank-medium-proxy; do
        if [[ -e "$quadlet_unit_dir/$unit.container" ]]; then
            mkdir -p "$trash"
            mv -- "$quadlet_unit_dir/$unit.container" "$trash/$unit.container"
            echo "retired quadlet $unit.container -> $trash"
        fi
    done
    systemctl --user daemon-reload
}

# The llama.cpp root unit is retired HERE, into the home trash, never `rm`:
# /etc/systemd/system/frank-medium.service is a relink_unit symlink into this
# checkout, and the checkout pull that precedes this handler already moved the
# repo file away, leaving the symlink dangling. Stop it while systemd still has
# it loaded (an active unit stays loaded without its file), then take both the
# symlink and its [Install] alias. Nothing can start it at boot afterwards.
retire_llama_medium_unit() {
    local trash="$HOME/.AGENTTRASH/20260920-llama-layersplit-frank-medium"
    run_privileged systemctl stop frank-medium.service 2>/dev/null || true
    if [[ -L /etc/systemd/system/frank-medium.service ||
          -L /etc/systemd/system/multi-user.target.wants/frank-medium.service ]]; then
        mkdir -p "$trash"
        [[ -L /etc/systemd/system/frank-medium.service ]] &&
            run_privileged mv -- /etc/systemd/system/frank-medium.service \
                "$trash/etc-frank-medium.service.symlink"
        [[ -L /etc/systemd/system/multi-user.target.wants/frank-medium.service ]] &&
            run_privileged mv -- /etc/systemd/system/multi-user.target.wants/frank-medium.service \
                "$trash/etc-frank-medium.wants.symlink"
        echo "retired llama root unit frank-medium.service -> $trash"
    fi
    run_privileged systemctl daemon-reload
}

deploy_llm_medium() {
    # Refuse rather than restart-loop. A container whose image or payload is
    # missing never becomes healthy, and a green deploy would leave the endpoint
    # dark with nothing in the deploy log to say why.
    local missing=0
    if ! podman image exists "$LLM_MEDIUM_IMAGE_LOCAL"; then
        if podman image exists "$LLM_MEDIUM_IMAGE"; then
            podman tag "$LLM_MEDIUM_IMAGE" "$LLM_MEDIUM_IMAGE_LOCAL"
        else
            echo "ERROR: radiance image absent ($LLM_MEDIUM_IMAGE / $LLM_MEDIUM_IMAGE_LOCAL)" >&2
            echo "frank-medium: podman pull $LLM_MEDIUM_IMAGE && podman tag $LLM_MEDIUM_IMAGE $LLM_MEDIUM_IMAGE_LOCAL" >&2
            missing=1
        fi
    fi
    if [[ ! -f "$LLM_MEDIUM_SNAP/config.json" ]]; then
        echo "ERROR: MXFP4 checkpoint absent at $LLM_MEDIUM_SNAP" >&2
        missing=1
    fi
    if [[ ! -f "$LLM_MEDIUM_DRAFTER/config.json" ]]; then
        echo "ERROR: DFlash2-FP8 drafter absent at $LLM_MEDIUM_DRAFTER" >&2
        missing=1
    fi
    # The patches clone is mounted and its scripts run at EVERY container start;
    # the chat template is the fleet's reasoning-format contract.
    if [[ ! -f "$LLM_MEDIUM_PATCHES/qwen-fixed-v22.3.jinja" ]]; then
        echo "ERROR: pinned patch clone / chat template absent at $LLM_MEDIUM_PATCHES" >&2
        echo "frank-medium: git clone https://github.com/ggz14/radiance-vllm-mxfp4 $LLM_MEDIUM_PATCHES" >&2
        missing=1
    fi
    # The image's shipped r4d.so predates the gated-delta-net overflow fix and
    # NaNs this model, so the patched build is not optional.
    if [[ ! -f "$LLM_MEDIUM_R4D/r4d.so" ]]; then
        echo "ERROR: patched libr4d absent at $LLM_MEDIUM_R4D" >&2
        missing=1
    fi
    [[ "$missing" -eq 0 ]] || return 1
    mkdir -p "$LLM_MEDIUM_CACHE"

    retire_vllm_replica_quadlets
    retire_llama_medium_unit
    # 7770 is a host-network bind: nothing new can be created until the old
    # holder released it (the same gate as the 2026-09-16 cutover, where the
    # retired proxy held the port and the new engine failed at create). Sweep a
    # stray manual llama-server too — that is how a bench harness leaves the
    # port busy. `pkill -x` matches the process name exactly and can never
    # match the deploying shell; never `pkill -f` here.
    pkill -x llama-server 2>/dev/null || true
    pkill -x VLLM::EngineCor 2>/dev/null || true
    sleep 2
    wait_port_free 7770 || return 1

    install_quadlet frank-medium dankserver/frank-medium/quadlet frank-medium
    # No enable: generated units cannot be enabled on this host (v1 quadlet
    # generator); boot persistence is the unit's [Install] WantedBy=default.target.
    systemctl --user restart frank-medium.service
    # Triton/inductor compile on a cold cache runs minutes before the engine
    # loads ~9.5 GiB per card; HealthStartPeriod in the unit is 1800s to match.
    wait_container_healthy frank-medium 1800 || {
        podman logs --tail 40 frank-medium 2>&1 || true
        return 1
    }
}

wait_for_health() {
    local service="$1" url="$2" limit="$3"
    local i
    for ((i=1; i<=limit; i++)); do
        if curl -fsS "$url" >/dev/null 2>&1; then
            echo "$service healthy after approximately $((i * 2))s"
            return 0
        fi
        sleep 2
    done
    echo "FAIL: $service did not become healthy in $((limit * 2))s" >&2
    return 1
}

# ---------------------------------------------------------------------------
# FrankBook-Pro (macOS): quadlet units live INSIDE the podman machine VM
# (/home/core/.config/containers/systemd), not in the macOS user session.
# This handler pipes unit files and seed logic into the VM over
# `podman machine ssh`; it never touches systemctl on the Mac host and never
# uses run_privileged (the Mac account is not passwordless sudo). Unit files
# are re-piped on every call (idempotent, cheap) before the selected unit is
# restarted; first boot also seeds the qBittorrent config once.
FRANKBOOK_SVCS=(gluetun qbittorrent prowlarr sonarr radarr lidarr readarr)
deploy_frankbook_stack() {
    local service="$1"
    export PATH="/usr/local/bin:/opt/homebrew/bin:$PATH"
    if ! podman machine inspect --format '{{.State}}' 2>/dev/null | grep -qx running; then
        echo "FAIL: podman machine is not running on frankbook-pro" >&2
        return 1
    fi
    local vm_dir="/home/core/.config/containers/systemd" svc
    podman machine ssh "mkdir -p '$vm_dir' /home/core/Containers/gluetun /home/core/Containers/qbittorrent"
    for svc in "${FRANKBOOK_SVCS[@]}"; do
        podman machine ssh "cat > '$vm_dir/$svc.container'" \
            < "frankbook-pro/$svc/quadlet/$svc.container"
    done
    local tmp
    tmp="$(mktemp /tmp/frankbook-vm.XXXXXX)"
    cat > "$tmp" <<'VM_SEED'
set -euo pipefail
# qBittorrent first-boot seed: conf + categories + machine-local PBKDF2
# password. qB >= 4.2 ignores plaintext passwords, so the conf carries the
# @ByteArray(salt:hash) form (PBKDF2-HMAC-SHA512, 100k iterations, 64 bytes).
if [ ! -e /home/core/Containers/qbittorrent/qbit-seeded ]; then
    mkdir -p /Volumes/COMPLETE/qbittorrent-data/qBittorrent
    QBIT_PW="$(openssl rand -base64 18 | tr -d '/+=' | head -c 20)"
    QBIT_HASH="$(python3 - "$QBIT_PW" <<'PYH'
import hashlib, base64, os, sys
pw = sys.argv[1].encode()
salt = os.urandom(16)
h = hashlib.pbkdf2_hmac("sha512", pw, salt, 100000, dklen=64)
print("@ByteArray(%s:%s)" % (base64.b64encode(salt).decode(), base64.b64encode(h).decode()))
PYH
)"
    umask 077
    printf 'QBIT_WEBUI_PASSWORD=%s\n' "$QBIT_PW" > /home/core/Containers/qbittorrent/env
    umask 022
    cat > /Volumes/COMPLETE/qbittorrent-data/qBittorrent/qBittorrent.conf <<QCONF
[BitTorrent]
Session\\DefaultSavePath=/Volumes/COMPLETE
Session\\Port=6881
Session\\QueueingSystemEnabled=true
Session\\TempPath=/Volumes/INCOMPLETE/incomplete
Session\\TempPathEnabled=true

[LegalNotice]
Accepted=true

[Meta]
MigrationVersion=8

[Network]
Cookies\\Enabled=true

[Preferences]
WebUI\\Address=*
WebUI\\ServerDomains=*
WebUI\\HostHeaderValidation=false
WebUI\\Username=admin
WebUI\\Password_PBKDF2=$QBIT_HASH
QCONF
    cat > /Volumes/COMPLETE/qbittorrent-data/qBittorrent/categories.json <<'QCAT'
{
    "Finished Torrents": {"save_path": "/Volumes/COMPLETE/Finished Torrents"},
    "lidarr": {"save_path": "/Volumes/COMPLETE/Finished Torrents"},
    "radarr": {"save_path": "/Volumes/COMPLETE/Finished Torrents"},
    "tv-sonarr": {"save_path": "/Volumes/COMPLETE/Finished Torrents"}
}
QCAT
    touch /home/core/Containers/qbittorrent/qbit-seeded
    echo "  seeded qBittorrent config (WebUI password: /home/core/Containers/qbittorrent/env inside the VM)"
fi
VM_SEED
    podman machine ssh bash -s < "$tmp"
    rm -f "$tmp"
    # Credential gate: without a WireGuard env the tunnel stays parked, and
    # qBittorrent (which downloads through it) waits with it. The arrs run
    # without the tunnel, so only these two services are gated.
    case "$service" in
        gluetun|qbittorrent)
            if ! podman machine ssh '[ -s /home/core/Containers/gluetun/env ]' >/dev/null 2>&1; then
                echo "SKIP: $service waits for gluetun credentials."
                echo "      Write the WireGuard env into /home/core/Containers/gluetun/env inside the VM"
                echo "      (podman machine ssh), then re-run: ./scripts/deploy.sh frankbook-pro --service gluetun --service qbittorrent"
                return 0
            fi
            ;;
    esac
    tmp="$(mktemp /tmp/frankbook-vm.XXXXXX)"
    cat > "$tmp" <<'VM_START'
set -euo pipefail
SVC="${1:?service required}"
systemctl --user daemon-reload
unit="$(systemctl --user list-units --all --no-legend "${SVC}.service" "container-${SVC}.service" 2>/dev/null | sed 's/^● //' | head -n1 | awk '{print $1}')"
[ -n "$unit" ] || { echo "FAIL: no generated systemd unit for $SVC" >&2; exit 1; }
systemctl --user restart "$unit"
sleep 3
state="$(systemctl --user is-active "$unit" || true)"
if [ "$state" != active ] && [ "$state" != activating ]; then
    echo "FAIL: $unit is $state" >&2
    systemctl --user status "$unit" --no-pager -l || true
    exit 1
fi
VM_START
    podman machine ssh bash -s "$service" < "$tmp"
    rm -f "$tmp"
    local port path
    case "$service" in
        qbittorrent) port=8080; path=/ ;;
        prowlarr) port=9696; path=/ping ;;
        sonarr) port=8989; path=/ping ;;
        radarr) port=7878; path=/ping ;;
        lidarr) port=8686; path=/ping ;;
        readarr) port=8787; path=/ping ;;
        *) return 0 ;;  # gluetun publishes only qBittorrent's ports
    esac
    printf '  waiting for http://127.0.0.1:%s%s on the Mac...\n' "$port" "$path"
    local ok=0 i
    for i in $(seq 1 60); do
        if curl -fsS -m 4 "http://127.0.0.1:${port}${path}" >/dev/null 2>&1; then
            ok=1
            break
        fi
        sleep 5
    done
    [[ "$ok" -eq 1 ]] || { echo "FAIL: $service did not answer on 127.0.0.1:$port" >&2; return 1; }
    echo "  $service answers on 127.0.0.1:$port"
}

deploy_seerr() {
    install_quadlet seerr dankclaw/seerr/quadlet seerr
    systemctl --user restart seerr.service
}

deploy_service() {
    local service="$1"
    if [[ "$TARGET" == "frankbook-pro" ]]; then
        # All frankbook-pro services route through the Mac handler: the name
        # case below addresses the Linux hosts, and service names overlap.
        deploy_frankbook_stack "$service"
        return
    fi
    case "$service" in
        caddy) deploy_caddy ;;
        homepage) deploy_homepage ;;
        searxng) deploy_searxng ;;
        gluetun) deploy_gluetun ;;
        searxng-valkey) deploy_searxng_valkey ;;
        open-webui) deploy_open_webui ;;
        pihole) deploy_pihole ;;
        gitea-postgres) deploy_gitea_unit gitea-postgres gitea-postgres ;;
        gitea-server) deploy_gitea_unit gitea-server gitea-server ;;
        authentik-postgres|authentik-server|authentik-worker)
            deploy_authentik
            ;;
        authentik-outpost)
            deploy_authentik_outpost
            ;;
        authentik-outpost-ldap)
            deploy_authentik_outpost_ldap
            ;;
        anubis) deploy_anubis ;;
        seerr) deploy_seerr ;;
        marreta) deploy_marreta ;;
        frank-large) deploy_llm_large ;;
        frank-small) deploy_llm_small ;;
        frank-medium) deploy_llm_medium ;;
        hermes|hermes-dashboard|hermes-webui) deploy_hermes ;;
        romm|romm-mariadb|romm-redis|mariadb|redis) deploy_romm_stack "$service" ;;
        jellyfin) deploy_jellyfin ;;
        calibre-web) deploy_calibre_web ;;
        copyparty) deploy_copyparty ;;
        plex) deploy_plex ;;
        tautulli) deploy_tautulli ;;
        prometheus|grafana|loki|alloy|node-exporter|alertmanager) deploy_observability "$service" ;;
        comfyui) deploy_comfyui ;;
        *) deploy_compose_service "$service" ;;
    esac
}

if [[ -n "$FROM_REVISION" ]]; then
    # Register every changed service before applying the requested scope. This
    # preserves an explicit --service selection as pending work for the next
    # ordinary deployment.
    register_changed_services "$FROM_REVISION"
fi
if [[ "$PENDING" -eq 1 ]]; then
    add_changed_services "$FROM_REVISION"
    add_pending_services
fi

# Remove duplicate service names while preserving caller/planner order.
UNIQUE_SERVICES=()
declare -A SEEN_SERVICES=()
for service in "${SERVICES[@]}"; do
    [[ -n "$service" ]] || continue
    if [[ -z "${SEEN_SERVICES[$service]:-}" ]]; then
        UNIQUE_SERVICES+=("$service")
        SEEN_SERVICES[$service]=1
    fi
done
SERVICES=("${UNIQUE_SERVICES[@]}")

if [[ "${#SERVICES[@]}" -eq 0 ]]; then
    echo "No services selected; nothing to deploy."
    exit 0
fi

for service in "${SERVICES[@]}"; do
    [[ "$service" =~ ^[a-z0-9][a-z0-9-]*$ ]] || { echo "invalid service: $service" >&2; exit 2; }
    service_is_allowed "$service" || { echo "service $service is not deployed on $TARGET" >&2; exit 2; }
    echo "=== Deploying $service at $REVISION ==="
    mark_status "$service" pending
    deploy_service "$service"
    mark_deployed "$service"
    echo "  STATE[$service] = $REVISION (status=deployed)"
done