refact(deploy-remote): shared root-unit start, config copy, retire helpers
rank 5. The three LLM handlers duplicated the same tail (relink, daemon-reload, enable, restart, /health gate, MainPID port proof) -> start_root_unit SERVICE PORT LIMIT; frank-archon's bespoke 900-iteration loop was exactly wait_for_health at limit 900. copy_config DEST SRC... replaces five hand-rolled mkdir+cp pairs (loki, alloy, grafana x2); retire_unit_file PATH TRASH [UNIT] replaces the duplicated compose-era unit retirements (halogen dragoon quadlet, comfyui.service). The health grep seam now pins the per-endpoint call (start_root_unit frank-dragoon 7770).
This commit is contained in:
Frank
+74
-115
@@ -87,31 +87,10 @@ cd "$REMOTE_DIR"
|
||||
mkdir -p "$STATE_DIR"
|
||||
|
||||
service_is_allowed() {
|
||||
local service="$1"
|
||||
case "$TARGET:$service" in
|
||||
dankclaw:gluetun|dankclaw:searxng-valkey|dankclaw:searxng|dankclaw:pihole|\
|
||||
dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
|
||||
dankclaw:comfyui|dankclaw:hermes|\
|
||||
dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
|
||||
dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:alertmanager|\
|
||||
dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
||||
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
|
||||
dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
|
||||
dankclaw:frank-dragoon|dankclaw:frank-zealot|\
|
||||
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
|
||||
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
|
||||
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
|
||||
dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
|
||||
dankserver:frank-archon|dankserver:node-exporter|\
|
||||
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
|
||||
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
|
||||
frankbook-pro:readarr)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
# The inventory (shared/services.yml via scripts/service-map.py) is the
|
||||
# only allow-list: a service is deployable on a target iff the inventory
|
||||
# places it there. Retiring a service from a host is a data edit.
|
||||
python3 scripts/service-map.py allows --host "$TARGET" --service "$1"
|
||||
}
|
||||
|
||||
mark_status() {
|
||||
@@ -281,13 +260,11 @@ install_signal_relay() {
|
||||
}
|
||||
|
||||
copy_loki_config() {
|
||||
run_privileged mkdir -p /opt/loki/config
|
||||
run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
|
||||
copy_config /opt/loki/config shared/loki/loki-config.yaml
|
||||
}
|
||||
|
||||
copy_alloy_config() {
|
||||
run_privileged mkdir -p /opt/alloy/config
|
||||
run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy
|
||||
copy_config /opt/alloy/config shared/alloy/config.alloy
|
||||
}
|
||||
|
||||
copy_grafana_provisioning() {
|
||||
@@ -296,14 +273,11 @@ copy_grafana_provisioning() {
|
||||
# datasource would be invisible to review, lost with the data volume, and
|
||||
# unknown to anyone reading the repository. The unit already mounts
|
||||
# /opt/grafana/provisioning read-only, so this only has to populate it.
|
||||
run_privileged mkdir -p /opt/grafana/provisioning/datasources \
|
||||
/opt/grafana/provisioning/dashboards
|
||||
run_privileged cp shared/grafana/provisioning/datasources/prometheus.yml \
|
||||
/opt/grafana/provisioning/datasources/prometheus.yml
|
||||
run_privileged cp shared/grafana/provisioning/dashboards/provider.yml \
|
||||
/opt/grafana/provisioning/dashboards/provider.yml
|
||||
run_privileged cp shared/grafana/provisioning/dashboards/llm-fleet.json \
|
||||
/opt/grafana/provisioning/dashboards/llm-fleet.json
|
||||
copy_config /opt/grafana/provisioning/datasources \
|
||||
shared/grafana/provisioning/datasources/prometheus.yml
|
||||
copy_config /opt/grafana/provisioning/dashboards \
|
||||
shared/grafana/provisioning/dashboards/provider.yml \
|
||||
shared/grafana/provisioning/dashboards/llm-fleet.json
|
||||
}
|
||||
|
||||
render_caddyfile() {
|
||||
@@ -448,31 +422,6 @@ deploy_caddy() {
|
||||
fi
|
||||
}
|
||||
|
||||
deploy_compose_service() {
|
||||
local service="$1"
|
||||
if [[ "$TARGET" == dankserver ]]; then
|
||||
# Dankserver's compose file is the source-of-truth inventory, but its
|
||||
# production containers are individually systemd-managed/bare. Do not
|
||||
# use compose up here: it can create a second container with a wrong
|
||||
# name and can interfere with a running workload.
|
||||
echo "WARN: $service is recorded in dankserver/compose.yaml; production container management is host-specific and no restart was attempted."
|
||||
return 0
|
||||
fi
|
||||
if [[ "$TARGET" == dankclaw ]]; then
|
||||
# Dankclaw has no compose file since 2026-09-16: every service is a
|
||||
# Quadlet unit with its own handler above. Reaching this means the
|
||||
# dispatcher is missing a case, and the old fallback (`podman-compose
|
||||
# up -d`) would now be a silent no-op or a duplicate container.
|
||||
echo "no compose services on dankclaw; $service has no quadlet handler" >&2
|
||||
return 2
|
||||
fi
|
||||
case "$service" in
|
||||
*)
|
||||
echo "unknown compose service: $service" >&2
|
||||
return 2
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# grafana is the one observability service that holds a credential: the
|
||||
# Authentik OAuth2 client secret. It is seeded into a per-service env file (the
|
||||
@@ -575,6 +524,35 @@ quadlet_unit_dir="$HOME/.config/containers/systemd"
|
||||
# dankserver's user manager had no systemd container generator dir before the
|
||||
# quadlet migration; every handler that installs a unit must find it present.
|
||||
mkdir -p "$quadlet_unit_dir"
|
||||
# copy_config DEST SRC... — the privileged config copy every service uses:
|
||||
# ensure DEST, copy each tracked SRC into it. The deployer owns /opt state;
|
||||
# the files in git are the source of truth, so a service config that only
|
||||
# exists on the host is a gap by policy.
|
||||
copy_config() {
|
||||
local dest="$1"; shift
|
||||
run_privileged mkdir -p "$dest"
|
||||
local src
|
||||
for src in "$@"; do
|
||||
run_privileged cp "$src" "$dest/"
|
||||
done
|
||||
}
|
||||
|
||||
# retire_unit_file PATH TRASH [UNIT] — the repo Deletion & Trash policy, in
|
||||
# code: stop and disable the named unit, move the unit FILE into the home
|
||||
# trash (never rm), report. The dragoon quadlet retirement and the
|
||||
# compose-era comfyui.service retirement were the same six lines twice.
|
||||
retire_unit_file() {
|
||||
local path="$1" trash="$2" unit="${3:-}"
|
||||
[[ -e "$path" ]] || return 0
|
||||
if [[ -n "$unit" ]]; then
|
||||
systemctl --user stop "$unit" 2>/dev/null || true
|
||||
systemctl --user disable "$unit" 2>/dev/null || true
|
||||
fi
|
||||
mkdir -p "$trash"
|
||||
mv -- "$path" "$trash/$(basename "$path")"
|
||||
echo "retired $(basename "$path") -> $trash"
|
||||
}
|
||||
|
||||
install_quadlet() {
|
||||
local service="$1"
|
||||
local source_dir="$2"
|
||||
@@ -662,15 +640,8 @@ deploy_comfyui() {
|
||||
# generator produces, and it is what kept a stale, device-less container
|
||||
# alive through `restart: unless-stopped` (9,146 failures on
|
||||
# "No HIP GPUs are available"). Never `rm`: it moves into the home trash.
|
||||
local legacy="$HOME/.config/systemd/user/comfyui.service"
|
||||
if [[ -e "$legacy" ]]; then
|
||||
local trash="$HOME/.AGENTTRASH/20260916-comfyui-compose-unit"
|
||||
mkdir -p "$trash"
|
||||
systemctl --user stop comfyui.service 2>/dev/null || true
|
||||
systemctl --user disable comfyui.service 2>/dev/null || true
|
||||
mv -- "$legacy" "$trash/comfyui.service"
|
||||
echo "retired the compose-era comfyui.service -> $trash"
|
||||
fi
|
||||
retire_unit_file "$HOME/.config/systemd/user/comfyui.service" \
|
||||
"$HOME/.AGENTTRASH/20260916-comfyui-compose-unit" comfyui.service
|
||||
# The stale container carries the old config (no /dev/kfd, no /dev/dri) and
|
||||
# would keep the quadlet's container from taking the name. It is replaced
|
||||
# by the unit in the next step, so nothing is lost.
|
||||
@@ -1138,13 +1109,8 @@ LLM_DRAGOON_TOK_DIR="${LLM_DRAGOON_TOK_DIR:-/home/dank/models/halogen-flashnext-
|
||||
LLM_DRAGOON_TOK_FILES="${LLM_DRAGOON_TOK_FILES:-tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json}"
|
||||
|
||||
retire_halogen_dragoon_quadlet() {
|
||||
local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
|
||||
systemctl --user stop frank-dragoon.service 2>/dev/null || true
|
||||
if [[ -e "$quadlet_unit_dir/frank-dragoon.container" ]]; then
|
||||
mkdir -p "$trash"
|
||||
mv -- "$quadlet_unit_dir/frank-dragoon.container" "$trash/frank-dragoon.container"
|
||||
echo "retired quadlet frank-dragoon.container -> $trash"
|
||||
fi
|
||||
retire_unit_file "$quadlet_unit_dir/frank-dragoon.container" \
|
||||
"$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet" frank-dragoon.service
|
||||
systemctl --user daemon-reload
|
||||
}
|
||||
|
||||
@@ -1181,6 +1147,20 @@ assert_unit_owns_port() {
|
||||
echo "$service: health answered by unit MainPID $main_pid"
|
||||
}
|
||||
|
||||
# start_root_unit — the tail every LLM root unit deploys: relink the unit
|
||||
# file from the checkout into /etc/systemd/system, (re)enable, restart,
|
||||
# wait for /health, and prove the answer came from the unit's own process.
|
||||
# The three endpoints differ in gates, ports, and budgets, never in this.
|
||||
start_root_unit() {
|
||||
local service="$1" port="$2" health_limit="$3"
|
||||
relink_unit "$service"
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable "$service.service"
|
||||
run_privileged systemctl restart "$service.service"
|
||||
wait_for_health "$service" "http://127.0.0.1:${port}/health" "$health_limit" || return 1
|
||||
assert_unit_owns_port "$service" "$port"
|
||||
}
|
||||
|
||||
deploy_llm_dragoon() {
|
||||
# Refuse rather than restart-loop; every gate here is the SAME library
|
||||
# call the serve script makes, so the two twins cannot drift (the
|
||||
@@ -1215,12 +1195,7 @@ deploy_llm_dragoon() {
|
||||
# Nothing may start until the predecessor has released the port
|
||||
# (2026-09-16: "Deactivated successfully" was not "port free").
|
||||
wait_port_free 7770 || return 1
|
||||
relink_unit frank-dragoon
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-dragoon.service
|
||||
run_privileged systemctl restart frank-dragoon.service
|
||||
wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200 || return 1
|
||||
assert_unit_owns_port frank-dragoon 7770
|
||||
start_root_unit frank-dragoon 7770 1200
|
||||
}
|
||||
|
||||
# deploy_llm_zealot — frank-zealot is llama.cpp (master build, CUDA sm_86) on
|
||||
@@ -1255,12 +1230,7 @@ deploy_llm_zealot() {
|
||||
require_file "$LLM_ZEALOT_MMPROJ" mmproj \
|
||||
"fetch it: bash scripts/download-frank-zealot.sh" || return 1
|
||||
|
||||
relink_unit frank-zealot
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-zealot.service
|
||||
run_privileged systemctl restart frank-zealot.service
|
||||
wait_for_health frank-zealot http://127.0.0.1:7771/health 900 || return 1
|
||||
assert_unit_owns_port frank-zealot 7771
|
||||
start_root_unit frank-zealot 7771 900
|
||||
}
|
||||
|
||||
# deploy_llm_archon — frank-archon is the radiance engine (StillDeadcode,
|
||||
@@ -1332,31 +1302,15 @@ deploy_llm_archon() {
|
||||
# (2026-09-26) and the frank-dragoon cutover (2026-09-16) both failed
|
||||
# exactly here when the next engine bound 7770 a second too early.
|
||||
wait_port_free 7770 || return 1
|
||||
relink_unit frank-archon
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-archon.service
|
||||
run_privileged systemctl restart frank-archon.service
|
||||
# Cold start: .rad load + PLE pin + expert host-pool upload (the
|
||||
# publisher's compose healthcheck allows a 15-minute start_period).
|
||||
# Parity with every predecessor on this port. The HTTP listener binds
|
||||
# AFTER the full load (main.cpp: engine.start() then engine.serve()),
|
||||
# so during load the port REFUSES connections — a bare curl is honest
|
||||
# here, unlike FreeToken's 200-while-loading body. (The MainPID check
|
||||
# below still guards the stray-port case.)
|
||||
local i ok=0
|
||||
for ((i=1; i<=900; i++)); do
|
||||
if curl -fsS http://127.0.0.1:7770/health 2>/dev/null >/dev/null; then
|
||||
echo "frank-archon engine healthy after approximately $((i * 2))s"
|
||||
ok=1
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
if [[ "$ok" != 1 ]]; then
|
||||
echo "FAIL: frank-archon /health never answered 200 in 1800s" >&2
|
||||
return 1
|
||||
fi
|
||||
assert_unit_owns_port frank-archon 7770
|
||||
# publisher's compose healthcheck allows a 15-minute start_period) —
|
||||
# the 900-iteration budget is 1800s, parity with every predecessor on
|
||||
# this port. The HTTP listener binds AFTER the full load (main.cpp:
|
||||
# engine.start() then engine.serve()), so during load the port REFUSES
|
||||
# connections — a bare curl is honest here, unlike FreeToken's
|
||||
# 200-while-loading body. (start_root_unit's MainPID check still
|
||||
# guards the stray-port case.)
|
||||
start_root_unit frank-archon 7770 900
|
||||
}
|
||||
|
||||
wait_for_health() {
|
||||
@@ -1556,7 +1510,12 @@ deploy_service() {
|
||||
tautulli) deploy_tautulli ;;
|
||||
prometheus|grafana|loki|alloy|node-exporter|alertmanager) deploy_observability "$service" ;;
|
||||
comfyui) deploy_comfyui ;;
|
||||
*) deploy_compose_service "$service" ;;
|
||||
*)
|
||||
# Every deployable service on every target has a handler above
|
||||
# (inventory = allow-list, so nothing unreachable arrives).
|
||||
echo "no handler for service: $service" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
|
||||
@@ -78,7 +78,10 @@ load helpers
|
||||
}
|
||||
|
||||
@test "remote runner health-checks an inference endpoint" {
|
||||
grep -q '127.0.0.1:7770/health' "$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
# start_root_unit carries the health gate for all three LLM units; the
|
||||
# contract is that each endpoint's start names its port, and 7770 is
|
||||
# frank-dragoon's.
|
||||
grep -q 'start_root_unit frank-dragoon 7770' "$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
}
|
||||
|
||||
@test "deploy.sh pulls the merged checkout on the target" {
|
||||
|
||||
Reference in New Issue
Block a user