diff --git a/dankclaw/comfyui/comfyui-healthcheck.py b/dankclaw/comfyui/comfyui-healthcheck.py new file mode 100755 index 0000000..477f733 --- /dev/null +++ b/dankclaw/comfyui/comfyui-healthcheck.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +"""ComfyUI container healthcheck: the UI must answer on :8188. + +Runs INSIDE the container, where the UI listens (the unit publishes that port on +the host loopback for the homepage status card). ComfyUI only answers once +torch, the custom nodes and the graph are loaded, so a 200 here means the +service is usable rather than merely a live process — the distinction that +matters, because the failure this replaced was a container that stayed "up" +while dying on every start. + +HealthCmd in the quadlet splits on spaces into an argv array (podman 5.x v1 +generator), which is why this is a file and not an inline `python3 -c`. +""" +import sys +import urllib.request + +try: + with urllib.request.urlopen("http://127.0.0.1:8188/", timeout=10) as resp: + sys.exit(0 if resp.status == 200 else 1) +except Exception: + sys.exit(1) diff --git a/dankclaw/comfyui/quadlet/comfyui.container b/dankclaw/comfyui/quadlet/comfyui.container new file mode 100644 index 0000000..5c0dbc7 --- /dev/null +++ b/dankclaw/comfyui/quadlet/comfyui.container @@ -0,0 +1,68 @@ +# Quadlet unit: ComfyUI (rootless, boot-persistent). +# +# Install path on dankclaw: ~/.config/containers/systemd/comfyui.container +# Unit name: comfyui.service (container name: comfyui) +# +# WHY THIS FILE EXISTS. ComfyUI ran from `dankclaw/compose.yaml` through a +# hand-written `~/.config/systemd/user/comfyui.service` that called +# `podman compose up -d`. Its container was created 2026-09-15 14:12, BEFORE the +# GPU declarations were added to the compose service, and `restart: +# unless-stopped` then kept restarting that stale container — so it started +# without /dev/kfd, without /dev/dri and without keep-groups, died on +# `RuntimeError: No HIP GPUs are available`, and had restarted 9,146 times when +# this was found. A recreate via compose would have fixed the container; a unit +# fixes the class: one owner per container name, repo-tracked, boot-persistent +# through the generator. +# +# GPU ACCESS. /dev/kfd + /dev/dri plus keep-groups. A literal `render` group +# entry cannot work under rootless Podman (the group name does not resolve +# inside the user namespace), and the companion udev rule +# (dankclaw/udev/99-comfyui-gpu-access.rules) keeps those nodes MODE 0666 so a +# user-unit spawn — which carries no supplementary groups — can still open them. +# Both halves are asserted by tests/comfyui_rocm.bats. +# +# EXPOSURE. Loopback only. ComfyUI has no authentication and executes arbitrary +# node graphs, and it has no Caddy vhost, so the repository's rule applies: +# an unguarded service is internal-only. The homepage status card probes +# 127.0.0.1:8188, which this satisfies. To reach the UI from another machine, +# guard it behind the auth chain and give it a vhost — do not widen the bind. +# +# DATA. Models, input, output and custom_nodes live under +# /home/dank/comfyui-data (host state, not in the repo). +[Unit] +Description=ComfyUI (gfx1151 ROCm image generation) +After=network-online.target +Wants=network-online.target + +[Container] +ContainerName=comfyui +Image=localhost/comfyui-rocm:gfx1151 +Pull=never +Network=dankclaw_default +PublishPort=127.0.0.1:8188:8188 +AddDevice=/dev/kfd +AddDevice=/dev/dri +GroupAdd=keep-groups +Volume=/home/dank/comfyui-data/models:/opt/ComfyUI/models +Volume=/home/dank/comfyui-data/input:/opt/ComfyUI/input +Volume=/home/dank/comfyui-data/output:/opt/ComfyUI/output +Volume=/home/dank/comfyui-data/custom_nodes:/opt/ComfyUI/custom_nodes +Volume=/home/dank/serverAppConfigs/dankclaw/comfyui/comfyui-healthcheck.py:/usr/local/bin/comfyui-healthcheck.py:ro +Exec=python main.py --listen 0.0.0.0 --port 8188 +# The probe lives in a file because the podman 5.x v1 generator splits +# HealthCmd on spaces into an argv array; an inline `python3 -c "..."` becomes +# a puzzle of quoting that fails silently. +HealthCmd=python3 /usr/local/bin/comfyui-healthcheck.py +HealthInterval=30s +HealthTimeout=10s +HealthRetries=3 +# First start loads torch and the node graph; give it room before the probe +# counts against it. +HealthStartPeriod=300s + +[Service] +Restart=always +RestartSec=10 + +[Install] +WantedBy=default.target diff --git a/dankclaw/compose.yaml b/dankclaw/compose.yaml index d3b7c2a..7d0e4b1 100644 --- a/dankclaw/compose.yaml +++ b/dankclaw/compose.yaml @@ -12,40 +12,14 @@ services: # See ../docs/Dankclaw/SearXNG.md, Gluetun.md, Valkey.md. # --- AI image generation (ROCm GPU backend, gfx1151) --------------------- - # Built from Containerfile.comfyui on AMD's gfx1151 torch base. The GPU is - # reached via the /dev/kfd + /dev/dri passthrough (checked host-side; the run - # user is already in the render group). Only the 'video' group is added: - # rootless podman cannot map the render gid into the container namespace. - # Models + data persist under /home/dank/comfyui-data. See ../docs/Dankclaw/ComfyUI.md. - comfyui: - build: - context: . - dockerfile: Containerfile.comfyui - image: comfyui-rocm:gfx1151 - container_name: comfyui - restart: unless-stopped - command: ["python", "main.py", "--listen", "0.0.0.0", "--port", "8188"] - ports: - - "8188:8188" - devices: - - /dev/kfd:/dev/kfd - - /dev/dri:/dev/dri - # keep-groups preserves the launching user's real supplementary groups; - # a literal `render` entry fails at container start under rootless podman - # because the group name cannot be resolved inside the user namespace. - group_add: - - keep-groups - volumes: - - /home/dank/comfyui-data/models:/opt/ComfyUI/models - - /home/dank/comfyui-data/input:/opt/ComfyUI/input - - /home/dank/comfyui-data/output:/opt/ComfyUI/output - - /home/dank/comfyui-data/custom_nodes:/opt/ComfyUI/custom_nodes - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8188/')"] - interval: 30s - timeout: 10s - retries: 5 - start_period: 180s + # ComfyUI is a boot-persistent Quadlet unit since 2026-09-16: + # comfyui/quadlet/comfyui.container, installed by deploy-remote.sh. It left + # compose after the container it had started went stale — created before the + # GPU declarations existed, then restarted by `restart: unless-stopped` + # without /dev/kfd or /dev/dri until it had failed 9,146 times on + # "No HIP GPUs are available". Built from Containerfile.comfyui (AMD's + # gfx1151 base), models and data under /home/dank/comfyui-data. + # See ../docs/Dankclaw/ComfyUI.md. # --- Chat UI (open-webui) ------------------------------------------------ # Now a boot-persistent Quadlet unit (quadlet, slirp4netns, port 3005), not diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 79f896a..b232396 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -383,12 +383,7 @@ deploy_compose_service() { return 0 fi case "$service" in - comfyui|hermes|hermes-dashboard|hermes-webui) - case "$service" in - comfyui) - build_compose_image "$service" - ;; - esac + hermes|hermes-dashboard|hermes-webui) podman-compose -f dankclaw/compose.yaml up -d --no-deps "$service" ;; *) @@ -572,6 +567,33 @@ deploy_open_webui() { systemctl --user restart open-webui.service } +deploy_comfyui() { + # Build first: the unit's Image= resolves against local storage, so the + # image has to exist before the generator starts a container from it. + build_compose_image comfyui + install_quadlet comfyui dankclaw/comfyui/quadlet comfyui + # Retire the compose-era unit. It owns the same unit name the Quadlet + # generator produces, and it is what kept a stale, device-less container + # alive through `restart: unless-stopped` (9,146 failures on + # "No HIP GPUs are available"). Never `rm`: it moves into the home trash. + local legacy="$HOME/.config/systemd/user/comfyui.service" + if [[ -e "$legacy" ]]; then + local trash="$HOME/.AGENTTRASH/20260916-comfyui-compose-unit" + mkdir -p "$trash" + systemctl --user stop comfyui.service 2>/dev/null || true + systemctl --user disable comfyui.service 2>/dev/null || true + mv -- "$legacy" "$trash/comfyui.service" + echo "retired the compose-era comfyui.service -> $trash" + fi + # The stale container carries the old config (no /dev/kfd, no /dev/dri) and + # would keep the quadlet's container from taking the name. It is replaced + # by the unit in the next step, so nothing is lost. + podman rm -f comfyui >/dev/null 2>&1 || true + systemctl --user daemon-reload + systemctl --user restart comfyui.service + wait_for_health comfyui http://127.0.0.1:8188/ 300 +} + deploy_pihole() { # Rootless port 53 needs net.ipv4.ip_unprivileged_port_start <= 53, and it # must hold at boot. The repo drop-in is the single source of truth. @@ -1237,6 +1259,7 @@ deploy_service() { plex) deploy_plex ;; tautulli) deploy_tautulli ;; prometheus|grafana|loki|alloy|node-exporter) deploy_observability "$service" ;; + comfyui) deploy_comfyui ;; *) deploy_compose_service "$service" ;; esac } diff --git a/tests/comfyui_rocm.bats b/tests/comfyui_rocm.bats index 783e3db..f74cfcd 100644 --- a/tests/comfyui_rocm.bats +++ b/tests/comfyui_rocm.bats @@ -9,93 +9,83 @@ # named-group membership, because systemd user-unit spawns carry NO # supplementary groups at spawn time # -# STATUS (2026-08-31): the GPU migration is applied. The compose builds -# comfyui-rocm:gfx1151 from Containerfile.comfyui (AMD's gfx1151 torch base), -# passes /dev/kfd + /dev/dri, and relies on the udev rule at -# dankclaw/udev/99-comfyui-gpu-access.rules (MODE 0666 on kfd + renderD*) for -# compute-node access; group_add is keep-groups only. The four compose-wiring -# tests below are active (un-skipped). See +# STATUS (2026-09-16): ComfyUI is a Quadlet unit, not a compose service. The +# build still comes from Containerfile.comfyui (AMD's gfx1151 torch base) via +# the deployer's build_compose_image, and the wiring now lives in +# dankclaw/comfyui/quadlet/comfyui.container: /dev/kfd + /dev/dri, keep-groups +# only, and the udev rule at dankclaw/udev/99-comfyui-gpu-access.rules (MODE +# 0666 on kfd + renderD*) for compute-node access, because systemd user-unit +# spawns carry no supplementary groups. The move happened after the compose +# container went stale — created before the GPU declarations existed, then +# restarted 9,146 times without them. See # serverAppConfigs-docs/Research/ComfyUI ROCm and Vulkan.md. load helpers @test "comfyui builds from the gfx1151 ROCm Containerfile" { - local compose="$REPO_ROOT/dankclaw/compose.yaml" - python3 - "$compose" <<'PYEOF' -import sys, yaml -with open(sys.argv[1]) as fh: - data = yaml.safe_load(fh) -svc = data["services"]["comfyui"] -assert svc.get("build"), "comfyui must have a build: section (Containerfile)" -assert svc.get("image", "").startswith("comfyui-rocm"), \ - "comfyui image must be tagged comfyui-rocm:*" -print("comfyui build OK:", svc["build"], svc.get("image")) -PYEOF + # The image is built by the deployer (build_compose_image comfyui, which + # resolves dankclaw/Containerfile.comfyui) and referenced by the unit. + grep -q "comfyui)" "$REPO_ROOT/scripts/deploy-remote.sh" + grep -q 'image=comfyui-rocm:gfx1151' "$REPO_ROOT/scripts/deploy-remote.sh" + grep -q 'containerfile=dankclaw/Containerfile.comfyui' "$REPO_ROOT/scripts/deploy-remote.sh" + local unit="$REPO_ROOT/dankclaw/comfyui/quadlet/comfyui.container" + grep -q '^Image=localhost/comfyui-rocm:gfx1151$' "$unit" \ + || { echo "unit must run the locally built image"; return 1; } + # One owner per container name: a surviving compose service would fight the + # unit for the container, which is how the stale one stayed alive. + ! grep -qE '^\s+comfyui:' "$REPO_ROOT/dankclaw/compose.yaml" } @test "comfyui Containerfile is based on the gfx1151 torch image" { - # Resolve the Containerfile the way the build does — build.context joined - # with build.dockerfile — rather than asking the filesystem. A - # `find … | head -1` here returned whichever Containerfile the directory - # listing happened to put first (dankclaw/alloy/Containerfile), so the test - # failed for sessions while the real ComfyUI Containerfile was correct — and - # it would equally have passed on a wrong file. - local cf - cf=$(python3 - "$REPO_ROOT/dankclaw/compose.yaml" "$REPO_ROOT/dankclaw" <<'PYEOF' -import os, sys, yaml -with open(sys.argv[1]) as fh: - svc = yaml.safe_load(fh)["services"]["comfyui"] -build = svc.get("build") or {} -# Compose resolves dockerfile relative to context, context relative to the -# compose file's own directory. -print(os.path.normpath(os.path.join( - sys.argv[2], build.get("context", "."), build.get("dockerfile", "Containerfile")))) -PYEOF -) - [ -n "$cf" ] || { echo "could not resolve the comfyui Containerfile"; return 1; } - [ -f "$cf" ] || { echo "$cf (from compose build) does not exist"; return 1; } + # Resolve the Containerfile from the deployer's declaration — the file the + # build actually uses — not from the filesystem. A `find … | head -1` here + # once returned whichever Containerfile the directory listing happened to + # put first (dankclaw/alloy/Containerfile), so the test failed for sessions + # while the real ComfyUI Containerfile was correct, and it would equally + # have passed on a wrong file. Compose no longer names it: ComfyUI is a + # Quadlet unit and the image is built by deploy_comfyui. + local cf="$REPO_ROOT/dankclaw/Containerfile.comfyui" + grep -q 'containerfile=dankclaw/Containerfile.comfyui' "$REPO_ROOT/scripts/deploy-remote.sh" \ + || { echo "deployer no longer builds that Containerfile"; return 1; } + [ -f "$cf" ] || { echo "$cf does not exist"; return 1; } grep -q "gfx1151" "$cf" || { echo "$cf does not reference gfx1151"; return 1; } grep -q "FROM" "$cf" || { echo "$cf has no FROM"; return 1; } } @test "comfyui passes /dev/kfd and /dev/dri to the container" { - python3 - "$REPO_ROOT/dankclaw/compose.yaml" <<'PYEOF' -import sys, yaml -with open(sys.argv[1]) as fh: - data = yaml.safe_load(fh) -devices = data["services"]["comfyui"].get("devices", []) -joined = "\n".join(devices) -for need in ("/dev/kfd", "/dev/dri"): - assert need in joined, f"comfyui devices must include {need}; got: {devices}" -print("device passthrough OK:", devices) -PYEOF + # Read the Quadlet unit, not compose: ComfyUI left compose on 2026-09-16 + # after a stale, device-less container was restarted 9,146 times by + # `restart: unless-stopped` (see comfyui/quadlet/comfyui.container). + local unit="$REPO_ROOT/dankclaw/comfyui/quadlet/comfyui.container" + [ -f "$unit" ] + grep -q '^AddDevice=/dev/kfd$' "$unit" || { echo "no /dev/kfd"; return 1; } + grep -q '^AddDevice=/dev/dri$' "$unit" || { echo "no /dev/dri"; return 1; } } @test "comfyui does not run with --cpu" { - python3 - "$REPO_ROOT/dankclaw/compose.yaml" <<'PYEOF' -import sys, yaml -with open(sys.argv[1]) as fh: - data = yaml.safe_load(fh) -cmd = data["services"]["comfyui"].get("command", []) -joined = " ".join(cmd) if isinstance(cmd, list) else str(cmd) -assert "--cpu" not in joined, "comfyui must not pass --cpu (ROCm GPU build)" -assert "--listen" in joined, "comfyui must listen on 0.0.0.0" -print("command OK:", cmd) -PYEOF + local unit="$REPO_ROOT/dankclaw/comfyui/quadlet/comfyui.container" + local exec_line + exec_line=$(grep -E '^Exec=' "$unit") + case "$exec_line" in + *--cpu*) echo "comfyui must not pass --cpu (ROCm GPU build)"; return 1 ;; + esac + case "$exec_line" in + *--listen*) ;; + *) echo "comfyui must listen on 0.0.0.0"; return 1 ;; + esac } @test "comfyui GPU access uses keep-groups + udev MODE 0666, not named groups" { - python3 - "$REPO_ROOT/dankclaw/compose.yaml" "$REPO_ROOT/dankclaw/udev/99-comfyui-gpu-access.rules" <<'PYEOF' -import sys, yaml -with open(sys.argv[1]) as fh: - data = yaml.safe_load(fh) -groups = data["services"]["comfyui"].get("group_add", []) + python3 - "$REPO_ROOT/dankclaw/comfyui/quadlet/comfyui.container" "$REPO_ROOT/dankclaw/udev/99-comfyui-gpu-access.rules" <<'PYEOF' +import sys, re +unit = open(sys.argv[1]).read() +groups = re.findall(r"^GroupAdd=(.*)$", unit, re.M) # User-unit spawns carry NO supplementary groups, so named group_add entries # (video, render) can never be honored at spawn time. The deployed contract is # keep-groups plus world-writable compute nodes via the udev rule; a future # edit that drops either half must fail here. -assert list(groups) == ["keep-groups"], \ - f"comfyui group_add must be exactly ['keep-groups']; got: {groups}" +assert groups == ["keep-groups"], \ + f"comfyui GroupAdd must be exactly ['keep-groups']; got: {groups}" rule = open(sys.argv[2]).read() assert 'KERNEL=="kfd", MODE="0666"' in rule, \ "udev rule must grant MODE 0666 on kfd with NO SUBSYSTEM guard (misc-class devices have no subsystem to match)"