refact(deploy-remote): shared root-unit start, config copy, retire helpers
rank 5. The three LLM handlers duplicated the same tail (relink, daemon-reload, enable, restart, /health gate, MainPID port proof) -> start_root_unit SERVICE PORT LIMIT; frank-archon's bespoke 900-iteration loop was exactly wait_for_health at limit 900. copy_config DEST SRC... replaces five hand-rolled mkdir+cp pairs (loki, alloy, grafana x2); retire_unit_file PATH TRASH [UNIT] replaces the duplicated compose-era unit retirements (halogen dragoon quadlet, comfyui.service). The health grep seam now pins the per-endpoint call (start_root_unit frank-dragoon 7770).
This commit is contained in:
1 parent
cc0bc06288
commit
5ae6b7b928
2 files changed
+78
-116
No files matched your search
+74
-115
@@ -87,31 +87,10 @@ cd "$REMOTE_DIR"
|
|||||||
mkdir -p "$STATE_DIR"
|
mkdir -p "$STATE_DIR"
|
||||||
|
|
||||||
service_is_allowed() {
|
service_is_allowed() {
|
||||||
local service="$1"
|
# The inventory (shared/services.yml via scripts/service-map.py) is the
|
||||||
case "$TARGET:$service" in
|
# only allow-list: a service is deployable on a target iff the inventory
|
||||||
dankclaw:gluetun|dankclaw:searxng-valkey|dankclaw:searxng|dankclaw:pihole|\
|
# places it there. Retiring a service from a host is a data edit.
|
||||||
dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
|
python3 scripts/service-map.py allows --host "$TARGET" --service "$1"
|
||||||
dankclaw:comfyui|dankclaw:hermes|\
|
|
||||||
dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
|
|
||||||
dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:alertmanager|\
|
|
||||||
dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
|
||||||
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
|
|
||||||
dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
|
|
||||||
dankclaw:frank-dragoon|dankclaw:frank-zealot|\
|
|
||||||
dankserver:romm-mariadb|dankserver:romm-redis|dankserver:romm|\
|
|
||||||
dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:calibre-server|dankserver:plex|dankserver:tautulli|\
|
|
||||||
dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
|
|
||||||
dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
|
|
||||||
dankserver:frank-archon|dankserver:node-exporter|\
|
|
||||||
frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
|
|
||||||
frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
|
|
||||||
frankbook-pro:readarr)
|
|
||||||
return 0
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
return 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
}
|
}
|
||||||
|
|
||||||
mark_status() {
|
mark_status() {
|
||||||
@@ -281,13 +260,11 @@ install_signal_relay() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
copy_loki_config() {
|
copy_loki_config() {
|
||||||
run_privileged mkdir -p /opt/loki/config
|
copy_config /opt/loki/config shared/loki/loki-config.yaml
|
||||||
run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
|
|
||||||
}
|
}
|
||||||
|
|
||||||
copy_alloy_config() {
|
copy_alloy_config() {
|
||||||
run_privileged mkdir -p /opt/alloy/config
|
copy_config /opt/alloy/config shared/alloy/config.alloy
|
||||||
run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy
|
|
||||||
}
|
}
|
||||||
|
|
||||||
copy_grafana_provisioning() {
|
copy_grafana_provisioning() {
|
||||||
@@ -296,14 +273,11 @@ copy_grafana_provisioning() {
|
|||||||
# datasource would be invisible to review, lost with the data volume, and
|
# datasource would be invisible to review, lost with the data volume, and
|
||||||
# unknown to anyone reading the repository. The unit already mounts
|
# unknown to anyone reading the repository. The unit already mounts
|
||||||
# /opt/grafana/provisioning read-only, so this only has to populate it.
|
# /opt/grafana/provisioning read-only, so this only has to populate it.
|
||||||
run_privileged mkdir -p /opt/grafana/provisioning/datasources \
|
copy_config /opt/grafana/provisioning/datasources \
|
||||||
/opt/grafana/provisioning/dashboards
|
shared/grafana/provisioning/datasources/prometheus.yml
|
||||||
run_privileged cp shared/grafana/provisioning/datasources/prometheus.yml \
|
copy_config /opt/grafana/provisioning/dashboards \
|
||||||
/opt/grafana/provisioning/datasources/prometheus.yml
|
shared/grafana/provisioning/dashboards/provider.yml \
|
||||||
run_privileged cp shared/grafana/provisioning/dashboards/provider.yml \
|
shared/grafana/provisioning/dashboards/llm-fleet.json
|
||||||
/opt/grafana/provisioning/dashboards/provider.yml
|
|
||||||
run_privileged cp shared/grafana/provisioning/dashboards/llm-fleet.json \
|
|
||||||
/opt/grafana/provisioning/dashboards/llm-fleet.json
|
|
||||||
}
|
}
|
||||||
|
|
||||||
render_caddyfile() {
|
render_caddyfile() {
|
||||||
@@ -448,31 +422,6 @@ deploy_caddy() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
deploy_compose_service() {
|
|
||||||
local service="$1"
|
|
||||||
if [[ "$TARGET" == dankserver ]]; then
|
|
||||||
# Dankserver's compose file is the source-of-truth inventory, but its
|
|
||||||
# production containers are individually systemd-managed/bare. Do not
|
|
||||||
# use compose up here: it can create a second container with a wrong
|
|
||||||
# name and can interfere with a running workload.
|
|
||||||
echo "WARN: $service is recorded in dankserver/compose.yaml; production container management is host-specific and no restart was attempted."
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
if [[ "$TARGET" == dankclaw ]]; then
|
|
||||||
# Dankclaw has no compose file since 2026-09-16: every service is a
|
|
||||||
# Quadlet unit with its own handler above. Reaching this means the
|
|
||||||
# dispatcher is missing a case, and the old fallback (`podman-compose
|
|
||||||
# up -d`) would now be a silent no-op or a duplicate container.
|
|
||||||
echo "no compose services on dankclaw; $service has no quadlet handler" >&2
|
|
||||||
return 2
|
|
||||||
fi
|
|
||||||
case "$service" in
|
|
||||||
*)
|
|
||||||
echo "unknown compose service: $service" >&2
|
|
||||||
return 2
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
# grafana is the one observability service that holds a credential: the
|
# grafana is the one observability service that holds a credential: the
|
||||||
# Authentik OAuth2 client secret. It is seeded into a per-service env file (the
|
# Authentik OAuth2 client secret. It is seeded into a per-service env file (the
|
||||||
@@ -575,6 +524,35 @@ quadlet_unit_dir="$HOME/.config/containers/systemd"
|
|||||||
# dankserver's user manager had no systemd container generator dir before the
|
# dankserver's user manager had no systemd container generator dir before the
|
||||||
# quadlet migration; every handler that installs a unit must find it present.
|
# quadlet migration; every handler that installs a unit must find it present.
|
||||||
mkdir -p "$quadlet_unit_dir"
|
mkdir -p "$quadlet_unit_dir"
|
||||||
|
# copy_config DEST SRC... — the privileged config copy every service uses:
|
||||||
|
# ensure DEST, copy each tracked SRC into it. The deployer owns /opt state;
|
||||||
|
# the files in git are the source of truth, so a service config that only
|
||||||
|
# exists on the host is a gap by policy.
|
||||||
|
copy_config() {
|
||||||
|
local dest="$1"; shift
|
||||||
|
run_privileged mkdir -p "$dest"
|
||||||
|
local src
|
||||||
|
for src in "$@"; do
|
||||||
|
run_privileged cp "$src" "$dest/"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# retire_unit_file PATH TRASH [UNIT] — the repo Deletion & Trash policy, in
|
||||||
|
# code: stop and disable the named unit, move the unit FILE into the home
|
||||||
|
# trash (never rm), report. The dragoon quadlet retirement and the
|
||||||
|
# compose-era comfyui.service retirement were the same six lines twice.
|
||||||
|
retire_unit_file() {
|
||||||
|
local path="$1" trash="$2" unit="${3:-}"
|
||||||
|
[[ -e "$path" ]] || return 0
|
||||||
|
if [[ -n "$unit" ]]; then
|
||||||
|
systemctl --user stop "$unit" 2>/dev/null || true
|
||||||
|
systemctl --user disable "$unit" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
mkdir -p "$trash"
|
||||||
|
mv -- "$path" "$trash/$(basename "$path")"
|
||||||
|
echo "retired $(basename "$path") -> $trash"
|
||||||
|
}
|
||||||
|
|
||||||
install_quadlet() {
|
install_quadlet() {
|
||||||
local service="$1"
|
local service="$1"
|
||||||
local source_dir="$2"
|
local source_dir="$2"
|
||||||
@@ -662,15 +640,8 @@ deploy_comfyui() {
|
|||||||
# generator produces, and it is what kept a stale, device-less container
|
# generator produces, and it is what kept a stale, device-less container
|
||||||
# alive through `restart: unless-stopped` (9,146 failures on
|
# alive through `restart: unless-stopped` (9,146 failures on
|
||||||
# "No HIP GPUs are available"). Never `rm`: it moves into the home trash.
|
# "No HIP GPUs are available"). Never `rm`: it moves into the home trash.
|
||||||
local legacy="$HOME/.config/systemd/user/comfyui.service"
|
retire_unit_file "$HOME/.config/systemd/user/comfyui.service" \
|
||||||
if [[ -e "$legacy" ]]; then
|
"$HOME/.AGENTTRASH/20260916-comfyui-compose-unit" comfyui.service
|
||||||
local trash="$HOME/.AGENTTRASH/20260916-comfyui-compose-unit"
|
|
||||||
mkdir -p "$trash"
|
|
||||||
systemctl --user stop comfyui.service 2>/dev/null || true
|
|
||||||
systemctl --user disable comfyui.service 2>/dev/null || true
|
|
||||||
mv -- "$legacy" "$trash/comfyui.service"
|
|
||||||
echo "retired the compose-era comfyui.service -> $trash"
|
|
||||||
fi
|
|
||||||
# The stale container carries the old config (no /dev/kfd, no /dev/dri) and
|
# The stale container carries the old config (no /dev/kfd, no /dev/dri) and
|
||||||
# would keep the quadlet's container from taking the name. It is replaced
|
# would keep the quadlet's container from taking the name. It is replaced
|
||||||
# by the unit in the next step, so nothing is lost.
|
# by the unit in the next step, so nothing is lost.
|
||||||
@@ -1138,13 +1109,8 @@ LLM_DRAGOON_TOK_DIR="${LLM_DRAGOON_TOK_DIR:-/home/dank/models/halogen-flashnext-
|
|||||||
LLM_DRAGOON_TOK_FILES="${LLM_DRAGOON_TOK_FILES:-tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json}"
|
LLM_DRAGOON_TOK_FILES="${LLM_DRAGOON_TOK_FILES:-tokenizer.json chat_template.jinja tokenizer_config.json merges.txt vocab.json generation_config.json}"
|
||||||
|
|
||||||
retire_halogen_dragoon_quadlet() {
|
retire_halogen_dragoon_quadlet() {
|
||||||
local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
|
retire_unit_file "$quadlet_unit_dir/frank-dragoon.container" \
|
||||||
systemctl --user stop frank-dragoon.service 2>/dev/null || true
|
"$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet" frank-dragoon.service
|
||||||
if [[ -e "$quadlet_unit_dir/frank-dragoon.container" ]]; then
|
|
||||||
mkdir -p "$trash"
|
|
||||||
mv -- "$quadlet_unit_dir/frank-dragoon.container" "$trash/frank-dragoon.container"
|
|
||||||
echo "retired quadlet frank-dragoon.container -> $trash"
|
|
||||||
fi
|
|
||||||
systemctl --user daemon-reload
|
systemctl --user daemon-reload
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1181,6 +1147,20 @@ assert_unit_owns_port() {
|
|||||||
echo "$service: health answered by unit MainPID $main_pid"
|
echo "$service: health answered by unit MainPID $main_pid"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# start_root_unit — the tail every LLM root unit deploys: relink the unit
|
||||||
|
# file from the checkout into /etc/systemd/system, (re)enable, restart,
|
||||||
|
# wait for /health, and prove the answer came from the unit's own process.
|
||||||
|
# The three endpoints differ in gates, ports, and budgets, never in this.
|
||||||
|
start_root_unit() {
|
||||||
|
local service="$1" port="$2" health_limit="$3"
|
||||||
|
relink_unit "$service"
|
||||||
|
run_privileged systemctl daemon-reload
|
||||||
|
run_privileged systemctl enable "$service.service"
|
||||||
|
run_privileged systemctl restart "$service.service"
|
||||||
|
wait_for_health "$service" "http://127.0.0.1:${port}/health" "$health_limit" || return 1
|
||||||
|
assert_unit_owns_port "$service" "$port"
|
||||||
|
}
|
||||||
|
|
||||||
deploy_llm_dragoon() {
|
deploy_llm_dragoon() {
|
||||||
# Refuse rather than restart-loop; every gate here is the SAME library
|
# Refuse rather than restart-loop; every gate here is the SAME library
|
||||||
# call the serve script makes, so the two twins cannot drift (the
|
# call the serve script makes, so the two twins cannot drift (the
|
||||||
@@ -1215,12 +1195,7 @@ deploy_llm_dragoon() {
|
|||||||
# Nothing may start until the predecessor has released the port
|
# Nothing may start until the predecessor has released the port
|
||||||
# (2026-09-16: "Deactivated successfully" was not "port free").
|
# (2026-09-16: "Deactivated successfully" was not "port free").
|
||||||
wait_port_free 7770 || return 1
|
wait_port_free 7770 || return 1
|
||||||
relink_unit frank-dragoon
|
start_root_unit frank-dragoon 7770 1200
|
||||||
run_privileged systemctl daemon-reload
|
|
||||||
run_privileged systemctl enable frank-dragoon.service
|
|
||||||
run_privileged systemctl restart frank-dragoon.service
|
|
||||||
wait_for_health frank-dragoon http://127.0.0.1:7770/health 1200 || return 1
|
|
||||||
assert_unit_owns_port frank-dragoon 7770
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# deploy_llm_zealot — frank-zealot is llama.cpp (master build, CUDA sm_86) on
|
# deploy_llm_zealot — frank-zealot is llama.cpp (master build, CUDA sm_86) on
|
||||||
@@ -1255,12 +1230,7 @@ deploy_llm_zealot() {
|
|||||||
require_file "$LLM_ZEALOT_MMPROJ" mmproj \
|
require_file "$LLM_ZEALOT_MMPROJ" mmproj \
|
||||||
"fetch it: bash scripts/download-frank-zealot.sh" || return 1
|
"fetch it: bash scripts/download-frank-zealot.sh" || return 1
|
||||||
|
|
||||||
relink_unit frank-zealot
|
start_root_unit frank-zealot 7771 900
|
||||||
run_privileged systemctl daemon-reload
|
|
||||||
run_privileged systemctl enable frank-zealot.service
|
|
||||||
run_privileged systemctl restart frank-zealot.service
|
|
||||||
wait_for_health frank-zealot http://127.0.0.1:7771/health 900 || return 1
|
|
||||||
assert_unit_owns_port frank-zealot 7771
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# deploy_llm_archon — frank-archon is the radiance engine (StillDeadcode,
|
# deploy_llm_archon — frank-archon is the radiance engine (StillDeadcode,
|
||||||
@@ -1332,31 +1302,15 @@ deploy_llm_archon() {
|
|||||||
# (2026-09-26) and the frank-dragoon cutover (2026-09-16) both failed
|
# (2026-09-26) and the frank-dragoon cutover (2026-09-16) both failed
|
||||||
# exactly here when the next engine bound 7770 a second too early.
|
# exactly here when the next engine bound 7770 a second too early.
|
||||||
wait_port_free 7770 || return 1
|
wait_port_free 7770 || return 1
|
||||||
relink_unit frank-archon
|
|
||||||
run_privileged systemctl daemon-reload
|
|
||||||
run_privileged systemctl enable frank-archon.service
|
|
||||||
run_privileged systemctl restart frank-archon.service
|
|
||||||
# Cold start: .rad load + PLE pin + expert host-pool upload (the
|
# Cold start: .rad load + PLE pin + expert host-pool upload (the
|
||||||
# publisher's compose healthcheck allows a 15-minute start_period).
|
# publisher's compose healthcheck allows a 15-minute start_period) —
|
||||||
# Parity with every predecessor on this port. The HTTP listener binds
|
# the 900-iteration budget is 1800s, parity with every predecessor on
|
||||||
# AFTER the full load (main.cpp: engine.start() then engine.serve()),
|
# this port. The HTTP listener binds AFTER the full load (main.cpp:
|
||||||
# so during load the port REFUSES connections — a bare curl is honest
|
# engine.start() then engine.serve()), so during load the port REFUSES
|
||||||
# here, unlike FreeToken's 200-while-loading body. (The MainPID check
|
# connections — a bare curl is honest here, unlike FreeToken's
|
||||||
# below still guards the stray-port case.)
|
# 200-while-loading body. (start_root_unit's MainPID check still
|
||||||
local i ok=0
|
# guards the stray-port case.)
|
||||||
for ((i=1; i<=900; i++)); do
|
start_root_unit frank-archon 7770 900
|
||||||
if curl -fsS http://127.0.0.1:7770/health 2>/dev/null >/dev/null; then
|
|
||||||
echo "frank-archon engine healthy after approximately $((i * 2))s"
|
|
||||||
ok=1
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
sleep 2
|
|
||||||
done
|
|
||||||
if [[ "$ok" != 1 ]]; then
|
|
||||||
echo "FAIL: frank-archon /health never answered 200 in 1800s" >&2
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
assert_unit_owns_port frank-archon 7770
|
|
||||||
}
|
}
|
||||||
|
|
||||||
wait_for_health() {
|
wait_for_health() {
|
||||||
@@ -1556,7 +1510,12 @@ deploy_service() {
|
|||||||
tautulli) deploy_tautulli ;;
|
tautulli) deploy_tautulli ;;
|
||||||
prometheus|grafana|loki|alloy|node-exporter|alertmanager) deploy_observability "$service" ;;
|
prometheus|grafana|loki|alloy|node-exporter|alertmanager) deploy_observability "$service" ;;
|
||||||
comfyui) deploy_comfyui ;;
|
comfyui) deploy_comfyui ;;
|
||||||
*) deploy_compose_service "$service" ;;
|
*)
|
||||||
|
# Every deployable service on every target has a handler above
|
||||||
|
# (inventory = allow-list, so nothing unreachable arrives).
|
||||||
|
echo "no handler for service: $service" >&2
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
esac
|
esac
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -78,7 +78,10 @@ load helpers
|
|||||||
}
|
}
|
||||||
|
|
||||||
@test "remote runner health-checks an inference endpoint" {
|
@test "remote runner health-checks an inference endpoint" {
|
||||||
grep -q '127.0.0.1:7770/health' "$REPO_ROOT/scripts/deploy-remote.sh"
|
# start_root_unit carries the health gate for all three LLM units; the
|
||||||
|
# contract is that each endpoint's start names its port, and 7770 is
|
||||||
|
# frank-dragoon's.
|
||||||
|
grep -q 'start_root_unit frank-dragoon 7770' "$REPO_ROOT/scripts/deploy-remote.sh"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "deploy.sh pulls the merged checkout on the target" {
|
@test "deploy.sh pulls the merged checkout on the target" {
|
||||||
|
|||||||
Reference in new issue
Block a user