Frank
6a191430ca
Grafana has been running with zero datasources and zero dashboards, so the metrics scraped last commit were only reachable through the Prometheus API. This wires the plumbing and one dashboard, both declared in shared/grafana/provisioning/ and installed to /opt/grafana/provisioning by the deployer (the unit already mounted that directory read-only, so nothing about the container changes): - datasources/prometheus.yml — Prometheus at http://prometheus:9090, the container name, not the loopback publish, with a fixed uid so dashboards reference it without a lookup. - dashboards/llm-fleet.json — KV pool fill, queue depth, prefix-cache reuse rate, e2e latency p95, and decode/prefill throughput, with an endpoint variable covering both frank-medium and frank-large. Every series name was checked against the live /metrics endpoint: this fleet already shipped one comment naming a metric that does not exist, and a dashboard full of those would be worse. Dashboards stay editable in the UI (allowUiUpdates) because the panels are a starting point, but the file is the source of truth and the deployer overwrites Grafana's copy on every deploy. Planner and seams follow the existing pattern: shared/grafana/** selects the grafana service, and the tests pin the container-name address plus the provisioning contract so a rename cannot silently empty the dashboard.
443 lines
16 KiB
Python
Executable File
443 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Plan runtime services affected by repository changes.
|
|
|
|
The planner is intentionally side-effect free. It accepts either an explicit
|
|
path list or two Git revisions. The latter lets compose-file changes resolve
|
|
to only the service definitions that actually changed instead of restarting an
|
|
entire compose project.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import subprocess
|
|
import sys
|
|
from pathlib import PurePosixPath
|
|
from typing import Any
|
|
|
|
import yaml
|
|
|
|
# Dependency-sensitive services come before their consumers. Independent
|
|
# compose services follow. LLM services are deliberately last and are never
|
|
# implied by a generic host, stack, documentation, or config change.
|
|
SERVICE_ORDER = (
|
|
"gluetun",
|
|
"searxng-valkey",
|
|
"searxng",
|
|
"marreta",
|
|
"pihole",
|
|
"open-webui",
|
|
"gitea-postgres",
|
|
"gitea-server",
|
|
"comfyui",
|
|
"seerr",
|
|
"hermes",
|
|
"hermes-dashboard",
|
|
"hermes-webui",
|
|
"prometheus",
|
|
"grafana",
|
|
"loki",
|
|
"alloy",
|
|
"node-exporter",
|
|
"authentik-postgres",
|
|
"authentik-server",
|
|
"authentik-worker",
|
|
"authentik-outpost",
|
|
"authentik-outpost-ldap",
|
|
"caddy",
|
|
"homepage",
|
|
"romm-mariadb",
|
|
"romm-redis",
|
|
"romm",
|
|
"copyparty",
|
|
"jellyfin",
|
|
"calibre-web",
|
|
"plex",
|
|
"tautulli",
|
|
"prowlarr",
|
|
"qbittorrent",
|
|
"radarr",
|
|
"sonarr",
|
|
"lidarr",
|
|
"readarr",
|
|
"frank-large",
|
|
"frank-medium",
|
|
)
|
|
|
|
DANKCLAW_SERVICES = {
|
|
"hermes-webui",
|
|
"gluetun",
|
|
"searxng-valkey",
|
|
"searxng",
|
|
"marreta",
|
|
"pihole",
|
|
"open-webui",
|
|
"gitea-postgres",
|
|
"gitea-server",
|
|
"comfyui",
|
|
"seerr",
|
|
"hermes",
|
|
"hermes-dashboard",
|
|
"prometheus",
|
|
"grafana",
|
|
"loki",
|
|
"alloy",
|
|
"node-exporter",
|
|
"authentik-postgres",
|
|
"authentik-server",
|
|
"authentik-worker",
|
|
"authentik-outpost",
|
|
"authentik-outpost-ldap",
|
|
"caddy",
|
|
"homepage",
|
|
"frank-large",
|
|
}
|
|
|
|
DANKSERVER_SERVICES = {
|
|
"romm-mariadb",
|
|
"romm-redis",
|
|
"romm",
|
|
"copyparty",
|
|
"jellyfin",
|
|
"calibre-web",
|
|
"plex",
|
|
"tautulli",
|
|
"prowlarr",
|
|
"qbittorrent",
|
|
"radarr",
|
|
"sonarr",
|
|
"lidarr",
|
|
"readarr",
|
|
"frank-medium",
|
|
}
|
|
|
|
# FrankBook-Pro runs the *arr stack + qBittorrent inside a podman machine on
|
|
# macOS. Service names overlap the Linux hosts (gluetun, qbittorrent, ...);
|
|
# the per-target sets and the frankbook-pro intercept in deploy-remote.sh
|
|
# keep dispatch unambiguous.
|
|
FRANKBOOK_SERVICES = {
|
|
"gluetun",
|
|
"qbittorrent",
|
|
"prowlarr",
|
|
"radarr",
|
|
"sonarr",
|
|
"lidarr",
|
|
"readarr",
|
|
}
|
|
|
|
DANKCLAW_COMPOSE_SERVICES = {
|
|
"comfyui",
|
|
"hermes",
|
|
"hermes-dashboard",
|
|
"hermes-webui",
|
|
"prometheus",
|
|
"grafana",
|
|
"loki",
|
|
"alloy",
|
|
"node-exporter",
|
|
"authentik-postgres",
|
|
"authentik-server",
|
|
"authentik-worker",
|
|
"caddy",
|
|
}
|
|
|
|
DANKSERVER_COMPOSE_SERVICES = {
|
|
"mariadb",
|
|
"redis",
|
|
"romm",
|
|
"copyparty",
|
|
"jellyfin",
|
|
"calibre-web",
|
|
}
|
|
COMPOSE_NAME_MAP = {"mariadb": "romm-mariadb", "redis": "romm-redis"}
|
|
|
|
|
|
|
|
def _add(services: set[str], *names: str) -> None:
|
|
services.update(names)
|
|
|
|
|
|
def target_services(target: str) -> set[str]:
|
|
if target == "dankclaw":
|
|
return DANKCLAW_SERVICES
|
|
if target == "dankserver":
|
|
return DANKSERVER_SERVICES
|
|
if target == "frankbook-pro":
|
|
return FRANKBOOK_SERVICES
|
|
raise ValueError(f"unknown target: {target}")
|
|
|
|
|
|
def _compose_services(target: str) -> set[str]:
|
|
return DANKCLAW_COMPOSE_SERVICES if target == "dankclaw" else DANKSERVER_COMPOSE_SERVICES
|
|
|
|
|
|
def _git_show(repo: str, revision: str, path: str) -> str | None:
|
|
result = subprocess.run(
|
|
["git", "-C", repo, "show", f"{revision}:{path}"],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
check=False,
|
|
)
|
|
return result.stdout if result.returncode == 0 else None
|
|
|
|
|
|
def _yaml_document(contents: str | None) -> dict[str, Any]:
|
|
if contents is None:
|
|
return {}
|
|
try:
|
|
data = yaml.safe_load(contents) or {}
|
|
except yaml.YAMLError:
|
|
return {}
|
|
return data if isinstance(data, dict) else {}
|
|
|
|
|
|
def _yaml_services(contents: str | None) -> dict[str, Any]:
|
|
services = _yaml_document(contents).get("services", {})
|
|
return services if isinstance(services, dict) else {}
|
|
|
|
|
|
def compose_changes(repo: str, target: str, base: str, head: str) -> set[str]:
|
|
"""Return only compose services whose YAML definitions or dependencies changed."""
|
|
path = f"{target}/compose.yaml"
|
|
old_doc = _yaml_document(_git_show(repo, base, path))
|
|
new_doc = _yaml_document(_git_show(repo, head, path))
|
|
old = old_doc.get("services", {}) if isinstance(old_doc.get("services", {}), dict) else {}
|
|
new = new_doc.get("services", {}) if isinstance(new_doc.get("services", {}), dict) else {}
|
|
names = set(old) | set(new)
|
|
allowed = _compose_services(target)
|
|
|
|
# An unavailable or malformed revision is safer as a broad compose plan
|
|
# than as a silent no-op. It still cannot include LLM units.
|
|
if (old_doc and not new_doc) or (new_doc and not old_doc and names):
|
|
return {COMPOSE_NAME_MAP.get(n, n) for n in allowed}
|
|
|
|
# Top-level networks/configuration can affect every compose service even
|
|
# when no individual service mapping changed. Ignore the services key while
|
|
# comparing the rest of the Compose document.
|
|
old_globals = {key: value for key, value in old_doc.items() if key != "services"}
|
|
new_globals = {key: value for key, value in new_doc.items() if key != "services"}
|
|
if old_globals != new_globals:
|
|
return {COMPOSE_NAME_MAP.get(n, n) for n in allowed}
|
|
|
|
changed = {name for name in names if name in allowed and old.get(name) != new.get(name)}
|
|
return {COMPOSE_NAME_MAP.get(n, n) for n in changed}
|
|
|
|
|
|
def services_for_path(path: str, target: str) -> set[str]:
|
|
"""Return services affected by one repository-relative path."""
|
|
path = path.replace("\\", "/")
|
|
p = PurePosixPath(path)
|
|
parts = p.parts
|
|
services: set[str] = set()
|
|
if not parts:
|
|
return services
|
|
|
|
# Non-runtime material is deliberately a no-op.
|
|
if parts[0] in {"docs", "tests", "old", ".githooks"}:
|
|
return services
|
|
if path in {"README.md", "AGENTS.md"} or parts[0] == ".obsidian":
|
|
return services
|
|
|
|
if parts[0] == "shared":
|
|
if len(parts) < 2:
|
|
return services
|
|
group = parts[1]
|
|
if group == "searxng":
|
|
_add(services, "searxng")
|
|
elif group == "prometheus":
|
|
_add(services, "prometheus")
|
|
elif group == "loki":
|
|
_add(services, "loki")
|
|
elif group == "alloy":
|
|
_add(services, "alloy")
|
|
elif group == "grafana":
|
|
# Provisioned datasources and dashboards: git is the source of
|
|
# truth, so a change here must restart the service that reads them.
|
|
_add(services, "grafana")
|
|
elif group == "homepage":
|
|
# Homepage is a Dankclaw-owned deployment. Shared homepage files
|
|
# are intentionally not copied during a Dankserver-only run.
|
|
if target == "dankclaw":
|
|
_add(services, "homepage")
|
|
return services
|
|
|
|
if parts[0] == "dankclaw":
|
|
if len(parts) >= 2 and parts[1] == "compose.yaml":
|
|
# Revision-aware callers replace this broad fallback with the
|
|
# service-level YAML comparison in compose_changes().
|
|
_add(services, *DANKCLAW_COMPOSE_SERVICES)
|
|
return services
|
|
if len(parts) >= 2 and parts[1] == "Caddyfile":
|
|
_add(services, "caddy")
|
|
return services
|
|
if len(parts) >= 2 and parts[1] == "Containerfile.comfyui":
|
|
_add(services, "comfyui")
|
|
return services
|
|
if len(parts) >= 2 and parts[1] in {"caddy", "comfyui", "loki", "alloy", "hermes", "anubis", "seerr"}:
|
|
_add(services, parts[1])
|
|
return services
|
|
if len(parts) >= 4 and parts[1] == "authentik" and parts[2] == "quadlet":
|
|
name = parts[3].removesuffix(".container")
|
|
if name in {"authentik-postgres", "authentik-server", "authentik-worker", "authentik-outpost", "authentik-outpost-ldap"}:
|
|
_add(services, name)
|
|
return services
|
|
if len(parts) >= 2 and parts[1] == "searxng":
|
|
if len(parts) >= 4 and parts[2] == "quadlet":
|
|
name = parts[3]
|
|
if name == "gluetun.container":
|
|
# SearXNG and Marreta (+ its browser companion) share
|
|
# Gluetun's network namespace. A Gluetun recreation
|
|
# tears the netns down, so every sharer must restart.
|
|
_add(services, "gluetun", "searxng", "marreta")
|
|
elif name == "searxng.container":
|
|
_add(services, "searxng")
|
|
elif name == "searxng-valkey.container":
|
|
_add(services, "searxng-valkey", "searxng")
|
|
elif name == "searxng.network":
|
|
_add(services, "gluetun", "searxng-valkey", "searxng", "marreta")
|
|
return services
|
|
if len(parts) >= 2 and parts[1] == "marreta":
|
|
# One deployable owns both the app and its browser companion;
|
|
# both live in gluetun's netns. A change to either unit selects
|
|
# the marreta service, which restarts browser-then-app.
|
|
_add(services, "marreta")
|
|
return services
|
|
if len(parts) >= 2 and parts[1] == "pihole":
|
|
_add(services, "pihole")
|
|
return services
|
|
if len(parts) >= 2 and parts[1] == "open-webui":
|
|
_add(services, "open-webui")
|
|
return services
|
|
if len(parts) >= 2 and parts[1] == "frank-large":
|
|
# The service is a rootless Quadlet container since 2026-09-11;
|
|
# its unit and fleet settings live in one .container file.
|
|
_add(services, "frank-large")
|
|
return services
|
|
if len(parts) >= 2 and parts[1] == "observability":
|
|
# The five observability containers became Quadlet units on
|
|
# 2026-09-11. A unit change selects exactly its own service;
|
|
# SERVICE_ORDER already puts each dependency ahead of its
|
|
# consumer (prometheus before grafana, loki before alloy).
|
|
if len(parts) >= 4 and parts[2] == "quadlet":
|
|
name = parts[3].removesuffix(".container")
|
|
if name in {"prometheus", "grafana", "loki", "alloy", "node-exporter"}:
|
|
_add(services, name)
|
|
return services
|
|
if len(parts) >= 2 and parts[1] == "gitea":
|
|
if len(parts) >= 4 and parts[2] == "quadlet":
|
|
name = parts[3]
|
|
if name == "gitea-postgres.container":
|
|
_add(services, "gitea-postgres")
|
|
elif name == "gitea-server.container":
|
|
_add(services, "gitea-server")
|
|
elif name == "gitea-caddy.container":
|
|
# Removing the old split proxy requires the canonical
|
|
# Caddy handler to retire its installed unit/container
|
|
# before it claims ports 80/443.
|
|
_add(services, "caddy")
|
|
elif name == "gitea.network":
|
|
_add(services, "gitea-postgres", "gitea-server")
|
|
return services
|
|
return services
|
|
|
|
if parts[0] == "dankserver":
|
|
if len(parts) >= 2 and parts[1] == "compose.yaml":
|
|
# Compose service keys map onto the deployed unit names.
|
|
_add(services, *(COMPOSE_NAME_MAP.get(n, n) for n in DANKSERVER_COMPOSE_SERVICES))
|
|
return services
|
|
if len(parts) >= 3 and parts[2] == "quadlet":
|
|
quadlet_services = {
|
|
"romm": {"romm", "romm-mariadb", "romm-redis"},
|
|
"jellyfin": {"jellyfin"},
|
|
"calibre-web": {"calibre-web"},
|
|
"copyparty": {"copyparty"},
|
|
"plex": {"plex"},
|
|
"tautulli": {"tautulli"},
|
|
"frank-medium": {"frank-medium"},
|
|
}
|
|
_add(services, *quadlet_services.get(parts[1], set()))
|
|
if len(parts) >= 2 and parts[1] == "frank-medium":
|
|
# The vllm-radiance runtime since 2026-09-15: the quadlet file
|
|
# above, plus the entrypoint and healthcheck it mounts straight
|
|
# out of the checkout. Every file in the service dir is runtime
|
|
# state.
|
|
_add(services, "frank-medium")
|
|
return services
|
|
return services
|
|
|
|
if parts[0] == "frankbook-pro":
|
|
if len(parts) >= 3 and parts[2] == "quadlet":
|
|
name = parts[3].removesuffix(".container")
|
|
if name == "gluetun":
|
|
# Every *arr + qBittorrent container shares gluetun's
|
|
# network namespace; a gluetun recreation kills all of
|
|
# them, so the plan restarts the whole netns family.
|
|
_add(services, "gluetun", "qbittorrent",
|
|
"prowlarr", "sonarr", "radarr", "lidarr", "readarr")
|
|
elif name in FRANKBOOK_SERVICES:
|
|
_add(services, name)
|
|
return services
|
|
|
|
if parts[0] == "scripts":
|
|
name = p.name
|
|
if name in {"homepage-status.py", "homepage-status.service", "homepage-status.timer"}:
|
|
_add(services, "homepage")
|
|
if name in {"frank-large.env", "frank-large.service", "serve-frank-large.sh"}:
|
|
# The ik_llama.cpp runtime (2026-09-15 cutover) keeps its unit,
|
|
# env and serve files here; each one selects frank-large only.
|
|
_add(services, "frank-large")
|
|
if name == "chat-template.jinja":
|
|
# The shared chat template is read by frank-large's serve script
|
|
# again; the vllm-radiance runtime on Dankserver ships its own.
|
|
_add(services, "frank-large")
|
|
# Deployment/test/benchmark scripts and other repo tooling do not
|
|
# restart a runtime service.
|
|
return services
|
|
|
|
# Examples and env templates are not deployed runtime state.
|
|
if p.name in {".env.example", "env.example"}:
|
|
return services
|
|
return services
|
|
|
|
|
|
def changed_paths(repo: str, base: str, head: str) -> list[str]:
|
|
result = subprocess.run(
|
|
["git", "-C", repo, "diff", "--name-only", "--diff-filter=ACDMRTUXB", base, head],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
check=True,
|
|
)
|
|
return [line for line in result.stdout.splitlines() if line]
|
|
|
|
|
|
def plan(paths: list[str], target: str, repo: str | None = None,
|
|
base: str | None = None, head: str | None = None) -> list[str]:
|
|
services: set[str] = set()
|
|
for path in paths:
|
|
if path == f"{target}/compose.yaml" and repo and base and head:
|
|
services.update(compose_changes(repo, target, base, head))
|
|
else:
|
|
services.update(services_for_path(path, target))
|
|
allowed = target_services(target)
|
|
return [name for name in SERVICE_ORDER if name in services and name in allowed]
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--target", choices=("dankclaw", "dankserver", "frankbook-pro"), required=True)
|
|
parser.add_argument("--paths", nargs="+")
|
|
parser.add_argument("--repo", help="repository path for revision-aware planning")
|
|
parser.add_argument("--base", help="base Git revision")
|
|
parser.add_argument("--head", help="head Git revision")
|
|
args = parser.parse_args()
|
|
|
|
if not args.paths and not (args.repo and args.base and args.head):
|
|
parser.error("provide --paths or all of --repo, --base, and --head")
|
|
paths = args.paths or changed_paths(args.repo, args.base, args.head)
|
|
print("\n".join(plan(paths, args.target, args.repo, args.base, args.head)))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|