From 5ae6b7b928585ca07d259fe344d73007d41dae68 Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 13 Oct 2026 22:35:12 -0700 Subject: [PATCH] refact(deploy-remote): shared root-unit start, config copy, retire helpers rank 5. The three LLM handlers duplicated the same tail (relink, daemon-reload, enable, restart, /health gate, MainPID port proof) -> start_root_unit SERVICE PORT LIMIT; frank-archon's bespoke 900-iteration loop was exactly wait_for_health at limit 900. copy_config DEST SRC... replaces five hand-rolled mkdir+cp pairs (loki, alloy, grafana x2); retire_unit_file PATH TRASH [UNIT] replaces the duplicated compose-era unit retirements (halogen dragoon quadlet, comfyui.service). The health grep seam now pins the per-endpoint call (start_root_unit frank-dragoon 7770). --- scripts/deploy-remote.sh | 189 +++++++++++++++---------------------- tests/deployment_flow.bats | 5 +- 2 files changed, 78 insertions(+), 116 deletions(-) diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index c148dde..85e85ee 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -87,31 +87,10 @@ cd "$REMOTE_DIR" mkdir -p "$STATE_DIR" service_is_allowed() { - local service="$1" - case "$TARGET:$service" in - dankclaw:gluetun|dankclaw:searxng-valkey|dankclaw:searxng|dankclaw:pihole|\ - dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\ - dankclaw:comfyui|dankclaw:hermes|\ - dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\ - dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:alertmanager|\ - dankclaw:authentik-postgres|dankclaw:authentik-server|\ - dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\ - dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\ - dankclaw:frank-dragoon|dankclaw:frank-zealot|\ - dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\ - dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\ - dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\ - dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\ - dankserver:frank-archon|dankserver:node-exporter|\ - frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\ - frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\ - frankbook-pro:readarr) - return 0 - ;; - *) - return 1 - ;; - esac + # The inventory (shared/services.yml via scripts/service-map.py) is the + # only allow-list: a service is deployable on a target iff the inventory + # places it there. Retiring a service from a host is a data edit. + python3 scripts/service-map.py allows --host "$TARGET" --service "$1" } mark_status() { @@ -281,13 +260,11 @@ install_signal_relay() { } copy_loki_config() { - run_privileged mkdir -p /opt/loki/config - run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml + copy_config /opt/loki/config shared/loki/loki-config.yaml } copy_alloy_config() { - run_privileged mkdir -p /opt/alloy/config - run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy + copy_config /opt/alloy/config shared/alloy/config.alloy } copy_grafana_provisioning() { @@ -296,14 +273,11 @@ copy_grafana_provisioning() { # datasource would be invisible to review, lost with the data volume, and # unknown to anyone reading the repository. The unit already mounts # /opt/grafana/provisioning read-only, so this only has to populate it. - run_privileged mkdir -p /opt/grafana/provisioning/datasources \ - /opt/grafana/provisioning/dashboards - run_privileged cp shared/grafana/provisioning/datasources/prometheus.yml \ - /opt/grafana/provisioning/datasources/prometheus.yml - run_privileged cp shared/grafana/provisioning/dashboards/provider.yml \ - /opt/grafana/provisioning/dashboards/provider.yml - run_privileged cp shared/grafana/provisioning/dashboards/llm-fleet.json \ - /opt/grafana/provisioning/dashboards/llm-fleet.json + copy_config /opt/grafana/provisioning/datasources \ + shared/grafana/provisioning/datasources/prometheus.yml + copy_config /opt/grafana/provisioning/dashboards \ + shared/grafana/provisioning/dashboards/provider.yml \ + shared/grafana/provisioning/dashboards/llm-fleet.json } render_caddyfile() { @@ -448,31 +422,6 @@ deploy_caddy() { fi } -deploy_compose_service() { - local service="$1" - if [[ "$TARGET" == dankserver ]]; then - # Dankserver's compose file is the source-of-truth inventory, but its - # production containers are individually systemd-managed/bare. Do not - # use compose up here: it can create a second container with a wrong - # name and can interfere with a running workload. - echo "WARN: $service is recorded in dankserver/compose.yaml; production container management is host-specific and no restart was attempted." - return 0 - fi - if [[ "$TARGET" == dankclaw ]]; then - # Dankclaw has no compose file since 2026-09-16: every service is a - # Quadlet unit with its own handler above. Reaching this means the - # dispatcher is missing a case, and the old fallback (`podman-compose - # up -d`) would now be a silent no-op or a duplicate container. - echo "no compose services on dankclaw; $service has no quadlet handler" >&2 - return 2 - fi - case "$service" in - *) - echo "unknown compose service: $service" >&2 - return 2 - ;; - esac -} # grafana is the one observability service that holds a credential: the # Authentik OAuth2 client secret. It is seeded into a per-service env file (the @@ -575,6 +524,35 @@ quadlet_unit_dir="$HOME/.config/containers/systemd" # dankserver's user manager had no systemd container generator dir before the # quadlet migration; every handler that installs a unit must find it present. mkdir -p "$quadlet_unit_dir" +# copy_config DEST SRC... — the privileged config copy every service uses: +# ensure DEST, copy each tracked SRC into it. The deployer owns /opt state; +# the files in git are the source of truth, so a service config that only +# exists on the host is a gap by policy. +copy_config() { + local dest="$1"; shift + run_privileged mkdir -p "$dest" + local src + for src in "$@"; do + run_privileged cp "$src" "$dest/" + done +} + +# retire_unit_file PATH TRASH [UNIT] — the repo Deletion & Trash policy, in +# code: stop and disable the named unit, move the unit FILE into the home +# trash (never rm), report. The dragoon quadlet retirement and the +# compose-era comfyui.service retirement were the same six lines twice. +retire_unit_file() { + local path="$1" trash="$2" unit="${3:-}" + [[ -e "$path" ]] || return 0 + if [[ -n "$unit" ]]; then + systemctl --user stop "$unit" 2>/dev/null || true + systemctl --user disable "$unit" 2>/dev/null || true + fi + mkdir -p "$trash" + mv -- "$path" "$trash/$(basename "$path")" + echo "retired $(basename "$path") -> $trash" +} + install_quadlet() { local service="$1" local source_dir="$2" @@ -662,15 +640,8 @@ deploy_comfyui() { # generator produces, and it is what kept a stale, device-less container # alive through `restart: unless-stopped` (9,146 failures on # "No HIP GPUs are available"). Never `rm`: it moves into the home trash. - local legacy="$HOME/.config/systemd/user/comfyui.service" - if [[ -e "$legacy" ]]; then - local trash="$HOME/.AGENTTRASH/20260916-comfyui-compose-unit" - mkdir -p "$trash" - systemctl --user stop comfyui.service 2>/dev/null || true - systemctl --user disable comfyui.service 2>/dev/null || true - mv -- "$legacy" "$trash/comfyui.service" - echo "retired the compose-era comfyui.service -> $trash" - fi + retire_unit_file "$HOME/.config/systemd/user/comfyui.service" \ + "$HOME/.AGENTTRASH/20260916-comfyui-compose-unit" comfyui.service # The stale container carries the old config (no /dev/kfd, no /dev/dri) and # would keep the quadlet's container from taking the name. It is replaced # by the unit in the next step, so nothing is lost. @@ -1138,13 +1109,8 @@ LLM_DRAGOON_TOK_DIR="${LLM_DRAGOON_TOK_DIR:-/home/dank/models/halogen-flashnext- LLM_DRAGOON_TOK_FILES="${LLM_DRAGOON_TOK_FILES:-tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json}" retire_halogen_dragoon_quadlet() { - local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet" - systemctl --user stop frank-dragoon.service 2>/dev/null || true - if [[ -e "$quadlet_unit_dir/frank-dragoon.container" ]]; then - mkdir -p "$trash" - mv -- "$quadlet_unit_dir/frank-dragoon.container" "$trash/frank-dragoon.container" - echo "retired quadlet frank-dragoon.container -> $trash" - fi + retire_unit_file "$quadlet_unit_dir/frank-dragoon.container" \ + "$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet" frank-dragoon.service systemctl --user daemon-reload } @@ -1181,6 +1147,20 @@ assert_unit_owns_port() { echo "$service: health answered by unit MainPID $main_pid" } +# start_root_unit — the tail every LLM root unit deploys: relink the unit +# file from the checkout into /etc/systemd/system, (re)enable, restart, +# wait for /health, and prove the answer came from the unit's own process. +# The three endpoints differ in gates, ports, and budgets, never in this. +start_root_unit() { + local service="$1" port="$2" health_limit="$3" + relink_unit "$service" + run_privileged systemctl daemon-reload + run_privileged systemctl enable "$service.service" + run_privileged systemctl restart "$service.service" + wait_for_health "$service" "http://127.0.0.1:${port}/health" "$health_limit" || return 1 + assert_unit_owns_port "$service" "$port" +} + deploy_llm_dragoon() { # Refuse rather than restart-loop; every gate here is the SAME library # call the serve script makes, so the two twins cannot drift (the @@ -1215,12 +1195,7 @@ deploy_llm_dragoon() { # Nothing may start until the predecessor has released the port # (2026-09-16: "Deactivated successfully" was not "port free"). wait_port_free 7770 || return 1 - relink_unit frank-dragoon - run_privileged systemctl daemon-reload - run_privileged systemctl enable frank-dragoon.service - run_privileged systemctl restart frank-dragoon.service - wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200 || return 1 - assert_unit_owns_port frank-dragoon 7770 + start_root_unit frank-dragoon 7770 1200 } # deploy_llm_zealot — frank-zealot is llama.cpp (master build, CUDA sm_86) on @@ -1255,12 +1230,7 @@ deploy_llm_zealot() { require_file "$LLM_ZEALOT_MMPROJ" mmproj \ "fetch it: bash scripts/download-frank-zealot.sh" || return 1 - relink_unit frank-zealot - run_privileged systemctl daemon-reload - run_privileged systemctl enable frank-zealot.service - run_privileged systemctl restart frank-zealot.service - wait_for_health frank-zealot http://127.0.0.1:7771/health 900 || return 1 - assert_unit_owns_port frank-zealot 7771 + start_root_unit frank-zealot 7771 900 } # deploy_llm_archon — frank-archon is the radiance engine (StillDeadcode, @@ -1332,31 +1302,15 @@ deploy_llm_archon() { # (2026-09-26) and the frank-dragoon cutover (2026-09-16) both failed # exactly here when the next engine bound 7770 a second too early. wait_port_free 7770 || return 1 - relink_unit frank-archon - run_privileged systemctl daemon-reload - run_privileged systemctl enable frank-archon.service - run_privileged systemctl restart frank-archon.service # Cold start: .rad load + PLE pin + expert host-pool upload (the - # publisher's compose healthcheck allows a 15-minute start_period). - # Parity with every predecessor on this port. The HTTP listener binds - # AFTER the full load (main.cpp: engine.start() then engine.serve()), - # so during load the port REFUSES connections — a bare curl is honest - # here, unlike FreeToken's 200-while-loading body. (The MainPID check - # below still guards the stray-port case.) - local i ok=0 - for ((i=1; i<=900; i++)); do - if curl -fsS http://127.0.0.1:7770/health 2>/dev/null >/dev/null; then - echo "frank-archon engine healthy after approximately $((i * 2))s" - ok=1 - break - fi - sleep 2 - done - if [[ "$ok" != 1 ]]; then - echo "FAIL: frank-archon /health never answered 200 in 1800s" >&2 - return 1 - fi - assert_unit_owns_port frank-archon 7770 + # publisher's compose healthcheck allows a 15-minute start_period) — + # the 900-iteration budget is 1800s, parity with every predecessor on + # this port. The HTTP listener binds AFTER the full load (main.cpp: + # engine.start() then engine.serve()), so during load the port REFUSES + # connections — a bare curl is honest here, unlike FreeToken's + # 200-while-loading body. (start_root_unit's MainPID check still + # guards the stray-port case.) + start_root_unit frank-archon 7770 900 } wait_for_health() { @@ -1556,7 +1510,12 @@ deploy_service() { tautulli) deploy_tautulli ;; prometheus|grafana|loki|alloy|node-exporter|alertmanager) deploy_observability "$service" ;; comfyui) deploy_comfyui ;; - *) deploy_compose_service "$service" ;; + *) + # Every deployable service on every target has a handler above + # (inventory = allow-list, so nothing unreachable arrives). + echo "no handler for service: $service" >&2 + exit 2 + ;; esac } diff --git a/tests/deployment_flow.bats b/tests/deployment_flow.bats index c529dd5..6b41fc0 100644 --- a/tests/deployment_flow.bats +++ b/tests/deployment_flow.bats @@ -78,7 +78,10 @@ load helpers } @test "remote runner health-checks an inference endpoint" { - grep -q '127.0.0.1:7770/health' "$REPO_ROOT/scripts/deploy-remote.sh" + # start_root_unit carries the health gate for all three LLM units; the + # contract is that each endpoint's start names its port, and 7770 is + # frank-dragoon's. + grep -q 'start_root_unit frank-dragoon 7770' "$REPO_ROOT/scripts/deploy-remote.sh" } @test "deploy.sh pulls the merged checkout on the target" {