diff --git a/AGENTS.md b/AGENTS.md index 135e454..808019b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,8 +46,9 @@ Retired scripts live in `old/` (reference only) and `.AGENTTRASH/`. - `dankserver/` — compose.yaml + Caddyfile + .env.example for the media stack - `dankclaw/` — compose.yaml + Caddyfile + .env.example for the AI stack - `shared/` — homepage, Prometheus, Loki, Alloy, SearXNG configs -- `scripts/` — `deploy.sh` (unified deployer), LLM serve scripts, systemd - units, env tunables, bench harnesses +- `scripts/` — `deploy.sh`, `deploy-plan.py`, and `deploy-remote.sh` + (service-scoped deployer), LLM serve scripts, systemd units, env tunables, + and bench harnesses - `old/` — retired PP/NPU-era LLM scripts (reference only) - `docs/` — knowledge base (Obsidian vault). `docs/LLM/` is the LLM fleet - `tests/` — bats test suite @@ -92,20 +93,28 @@ All edits flow through git. Do not edit files on the remote machine directly. 4. **Run the tests** — `./tests/run.sh` must pass 5. **Commit and push** — push to origin (Gitea on Dankclaw) 6. **Pull on the target** — `git pull` on the remote machine -7. **Restart** — `scripts/deploy.sh` pulls, restarts the stack and the LLM - service, and checks health +7. **Deploy** — `scripts/deploy.sh` pulls, plans, and dispatches only the + changed service(s), then checks the selected service health ## Deployment -Use `scripts/deploy.sh` to deploy committed changes to a target. One script -covers both workloads per host: the podman stack and the LLM service. +Use `scripts/deploy.sh` to deploy committed changes to a target. The planner +maps changed paths to individual services; the remote runner restarts only the +selected service and records its successful revision outside the checkout. ``` -./scripts/deploy.sh dankclaw # AI stack + frank-default-large (:7770) -./scripts/deploy.sh dankserver # media stack + frank-default-small (:7770) -./scripts/deploy.sh dankclaw --dry-run # preview without changes +./scripts/deploy.sh dankclaw +./scripts/deploy.sh dankserver +./scripts/deploy.sh dankclaw --service searxng +./scripts/deploy.sh dankserver --service frank-default-small +./scripts/deploy.sh dankclaw --dry-run ``` +A documentation-only change is a no-op. The LLM services are selected only by +their own unit/env/serve files or the shared chat template, so unrelated +container changes cannot interrupt inference. `frank-default-npu` is relink-only +when explicitly selected. + Set `FRANK_SUDO_PW` for headless runs on a host without passwordless sudo (remote sudo reads it from stdin via `sudo -S`). The value is never stored in this repo. @@ -115,10 +124,10 @@ graph LR A[Edit locally] --> B[Tests pass] B --> C[Commit] C --> D[Push to origin] - D --> E[deploy.sh: pull on target] - E --> F[restart podman stack] - F --> G[relink + restart LLM unit] - G --> H[health: homepage + LLM] + D --> E[deploy.sh computes service plan] + E --> F[Target pulls with ff-only] + F --> G[deploy-remote.sh dispatches selected service] + G --> H[Per-service health check] ``` ## Testing @@ -133,8 +142,8 @@ development machine. Test seams: compose validity, env consistency (no leaked secrets), Caddyfile validity, quadlet units, docs completeness, script syntax (all of `scripts/`, including the LLM serve scripts), LLM unit integrity (units point at files in -this repo), LLM secret hygiene, deployment flow. See -[docs/Architecture/Testing Policy.md](docs/Architecture/Testing%20Policy.md). +this repo), LLM secret hygiene, deployment flow, and service-scoped deployment +planning/dispatch. See [docs/Architecture/Testing Policy.md](docs/Architecture/Testing%20Policy.md). ## Secret Rules @@ -387,7 +396,7 @@ Direct edits on the remote drift from version control. Future agents will not know what changed. Git is the single source of truth. The two workloads share two hosts, one Caddy, and one Gitea. One repo keeps them in one history. -> User request → Fetch from remote → Commit here → Edit → Test → Commit → Push → Pull on remote → Deploy + restart +> User request → Fetch from remote → Commit here → Edit → Test → Commit → Push → Pull on remote → Dispatch changed service(s) → Health ## Summary @@ -396,7 +405,7 @@ two hosts, one Caddy, and one Gitea. One repo keeps them in one history. | Editing a compose/Caddyfile/serve script | **Work through git** — fetch, commit, edit, test, push, pull, deploy. Never edit on the remote directly. | | Config exists only on remote | **Pull it into the repo first** — commit the fetched version (redacted) before making changes. | | Unstaged local changes | **Commit or discard** — check `git status` before starting work. Ask the user if unsure. | -| Deploying a committed change | **Run `scripts/deploy.sh `** — it pushes, pulls on the target, restarts the stack and the LLM service, and checks both health endpoints. See [docs/LLM/deployment.md](docs/LLM/deployment.md). | +| Deploying a committed change | **Run `scripts/deploy.sh `** — it pushes, pulls with fast-forward-only, plans changed paths, dispatches selected services, and checks their health. See [docs/LLM/deployment.md](docs/LLM/deployment.md). | | Benchmarking | **Stop the service first** — scratch port, restart, verify health. See the Benchmarking Policy above. | | Deleting a file | **Move it to trash** — `.AGENTTRASH/` (in-repo) or `~/.AGENTTRASH/YYYYMMDD-topic`. Never `rm`. | | Tests exist | **Run them** — verify tests pass after every change. Do not commit failing tests. | diff --git a/README.md b/README.md index fc49c07..c8da849 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ one deploy script. - `dankserver/` — compose.yaml + Caddyfile + .env.example for the Dankserver stack - `dankclaw/` — compose.yaml + Caddyfile + .env.example for the Dankclaw stack - `shared/` — homepage, prometheus config, deploy helpers -- `scripts/` — `deploy.sh` (unified: stack + LLM), LLM serve scripts, systemd units, env tunables, bench harnesses +- `scripts/` — `deploy.sh` and `deploy-remote.sh` (service-scoped deployment), the planner, LLM serve scripts, systemd units, env tunables, and bench harnesses - `old/` — retired PP/NPU-era LLM scripts (reference only) - `docs/` — project knowledge base (Obsidian vault, wikilinks). `docs/LLM/` is the LLM fleet - `tests/` — bats test suite (see [[Testing Policy]]) @@ -46,9 +46,10 @@ one deploy script. See [docs/Architecture/Deployment Flow.md](docs/Architecture/Deployment%20Flow.md) and [docs/LLM/deployment.md](docs/LLM/deployment.md). Run -`./scripts/deploy.sh ` after pushing committed changes. -It pushes, pulls on the target, restarts the podman stack and the LLM -service, and checks both health endpoints. +`./scripts/deploy.sh ` after committing changes. It +plans and deploys only the changed services. Use `--service NAME` for an +explicit single-service retry; inference is restarted only when its own files +(or the shared chat template) changed. ## Knowledge Base diff --git a/docs/Architecture/Deployment Flow.md b/docs/Architecture/Deployment Flow.md index 5744b10..0f02405 100644 --- a/docs/Architecture/Deployment Flow.md +++ b/docs/Architecture/Deployment Flow.md @@ -1,60 +1,129 @@ +--- +type: guide +title: Deployment Flow +description: Service-scoped Git deployment for the two homelab hosts. +tags: + - deployment + - operations + - git +timestamp: 2026-09-03 +--- + # Deployment Flow -Git is the single source of truth. A `deploy.sh` script moves committed -changes to the target hosts over SSH. One run per target covers both -workloads on that host: the podman stack and the LLM service. +Git is the single source of truth. `scripts/deploy.sh` pushes the committed +`main` branch, pulls it on one target, and applies only the runtime services +whose tracked inputs changed. A SearXNG change no longer recreates the +inference service; an LLM change no longer restarts unrelated containers. ## Workflow ```mermaid graph LR - A[Edit in this repo] --> B[Commit] - B --> C[Push to origin] - C --> D[deploy.sh: pull on target] - D --> E[restart stack] - E --> F[health check] + A[Edit in this repo] --> B[Run tests] + B --> C[Commit main] + C --> D[deploy.sh computes plan] + D --> E[Push main] + E --> F[Target pulls with ff-only] + F --> G[deploy-remote.sh dispatches selected services] + G --> H[Per-service health check] ``` 1. Edit files locally in this repository. -2. Commit and push. -3. Run `./scripts/deploy.sh `. -4. The script pulls the repo on the target and restarts the podman stack - and the LLM service. -5. It verifies service health after restart (homepage and LLM endpoint). +2. Run `./tests/run.sh`, then commit the complete change on `main`. +3. Run `./scripts/deploy.sh `. +4. The planner maps changed paths to services. Compose YAML is compared by + service definition so an edit to one service does not restart its peers. +5. The target pulls with `git pull --ff-only origin main` and runs the remote + dispatcher with the explicit plan. +6. The dispatcher updates only selected Quadlet, compose, homepage, Caddy, or + systemd services and records each successful service revision outside Git. -## Deploy Script +## Commands -``` -./scripts/deploy.sh dankclaw # pull + restart on Dankclaw -./scripts/deploy.sh dankserver # pull + restart on Dankserver +```bash +./scripts/deploy.sh dankclaw +./scripts/deploy.sh dankserver +./scripts/deploy.sh dankclaw --service searxng +./scripts/deploy.sh dankserver --service frank-default-small ./scripts/deploy.sh dankclaw --dry-run ``` -SSH uses the existing key pair (`dank@dankclaw`, `dank@dankserver`). No passwords are stored or documented in this repo. If sudo is needed, use the interactive sudo on the target — never commit a password. +`--service` is repeatable and is an explicit retry/override. Without it, the +planner compares the local `HEAD` to `origin/main`. A documentation-only +commit produces an empty plan. The remote state directory can also replay a +service whose earlier deployment failed, but it never expands a plan to a +service on the other host. + +## Service managers + +| Service family | Runtime manager | Scope behavior | +|---|---|---| +| Dankclaw compose services | `podman-compose -f dankclaw/compose.yaml up -d --no-deps NAME` | Recreates only `NAME`; Caddy is handled separately because it fronts both hosts. | +| SearXNG, Gluetun, Valkey, OpenWebUI, Pi-hole, Gitea | user Quadlet + `systemctl --user` | Installs the selected unit and restarts only the required dependency-coupled units. | +| `frank-default-large` | systemd + `systemctl` | Relinks and restarts only the large inference service; waits up to 20 minutes for `/health`. | +| `frank-default-small` | systemd + `systemctl` | Relinks and restarts only the small inference service; waits up to 20 minutes for `/health`. | +| Dankserver media inventory | host-managed bare containers | Compose is inventory only; the scoped deploy does not create duplicate containers. | + +The shared chat template maps to the inference service on the selected target. +The inactive `frank-default-npu` unit is relink-only when explicitly selected; +it is not restarted by an unrelated large-model deployment. + +## Caddy safety + +Caddy runs on Dankclaw and fronts both hosts. The dispatcher never stops +`gitea-caddy` implicitly. If the legacy `gitea-caddy` container owns ports +80/443 while the compose `caddy` container is absent or stopped, a Caddy +selection fails closed rather than replacing the live proxy. Resolve port +ownership explicitly before retrying. A Caddy reload failure falls back to a +restart of **only** the selected `caddy` container. + +Homepage files, generated `deployed.js`, the status checker, and the rendered +Caddyfile are updated only when their service is selected. A media-only +Dankserver deployment does not touch Dankclaw's proxy or homepage. + +## Per-service state + +The remote runner writes one revision file per successfully completed service +to: + +```text +${XDG_STATE_HOME:-~/.local/state}/serverAppConfigs/deploy-state/ +``` + +The directory is outside the checkout and is never committed. A failed handler +is not marked successful. On the next ordinary deployment, the runner compares +the recorded revision with the new checkout and retries only a still-changed +service. This prevents a failed Caddy or LLM deployment from being silently +forgotten while avoiding a host-wide restart. ## Targets -| Target | Host | Stack | -|--------|------|-------| -| dankclaw | dank@dankclaw (192.168.0.143) | `dankclaw/` | -| dankserver | dank@dankserver (192.168.0.235) | `dankserver/` | +| Target | Host | Repository checkout | +|---|---|---| +| dankclaw | `dank@dankclaw` (`192.168.0.143`) | `/home/dank/serverAppConfigs` | +| dankserver | `dank@dankserver` (`192.168.0.235`) | `/home/dank/serverAppConfigs` | -## Health Verification - -The deploy script polls the health endpoint of each service after restart. Service healthchecks are defined in each compose file. A service that fails its healthcheck is reported, not hidden. +SSH uses the existing key pair. No password is stored or documented in this +repository. If sudo is needed on a headless target, `FRANK_SUDO_PW` is read +from the environment and is never written to the repository. ## Tests -Run the test suite before pushing: +Run the full suite before pushing: -``` +```bash ./tests/run.sh ``` -The suite validates compose YAML, env consistency, Caddyfiles, docs completeness, and script syntax. See [[Testing Policy]]. +The deployment seams are in `tests/deployment_scoping.bats` and +`tests/deployment_flow.bats`. They cover path isolation, compose narrowing, +Caddy safety, inference isolation, per-service state, and legacy wrapper +routing. See [[Testing Policy]]. ## See Also -- [[Storage Policy]] - [[Testing Policy]] -- [LLM deployment docs](../LLM/deployment.md) — the LLM half of the same flow \ No newline at end of file +- [LLM deployment](../LLM/deployment.md) +- [[Caddy]] +- [[SearXNG]] diff --git a/docs/Architecture/Testing Policy.md b/docs/Architecture/Testing Policy.md index 5ce7826..8a68ef0 100644 --- a/docs/Architecture/Testing Policy.md +++ b/docs/Architecture/Testing Policy.md @@ -11,7 +11,7 @@ This repository tests its configuration at defined seams. The tests run with `ba | Caddyfile validity | `tests/caddyfile.bats` | per-host Caddyfile, `.local` convention, balanced braces, basic auth present | | Docs completeness | `tests/docs_completeness.bats` | every compose service has a docs page; index links every page | | Script syntax | `tests/scripts_syntax.bats` | `bash -n` clean, executable bits, shellcheck when available | -| Deployment flow | `tests/deployment_flow.bats` | deploy.sh exists, targets known, SSH reachable (skips offline) | +| Deployment flow | `tests/deployment_flow.bats`, `tests/deployment_scoping.bats` | scoped dispatcher exists, targets are known, service plans isolate inference, Caddy ownership is guarded, and SSH reachability is checked (skips offline) | ## Focused pre-commit testing diff --git a/docs/LLM/deployment.md b/docs/LLM/deployment.md index 64f618e..07e70c8 100644 --- a/docs/LLM/deployment.md +++ b/docs/LLM/deployment.md @@ -1,227 +1,124 @@ +--- +type: guide +title: LLM Deployment +description: Scoped deployment and update process for the two inference services. +tags: + - llm + - deployment + - systemd +timestamp: 2026-09-03 +--- + # Deployment and Update Process -This page describes how to deploy and update the LLM servers on Dankserver -and Dankclaw. The process uses git as the single source of truth. All script -edits flow through this repository. +This page describes how to deploy and update the LLM servers on Dankserver and +Dankclaw. The process uses Git as the single source of truth. A change to +SearXNG, Caddy, or another container service does **not** restart inference. -> 2026-08-22: the former `vllmScripts` repository is merged into this repo. -> Both LLM services now run from the `/home/dank/serverAppConfigs` checkout -> on each host. The retired checkouts (`/home/dank/vLLM` on Dankserver, -> `/home/dank/vllmScripts` on Dankclaw) are archived on the hosts. +## Fleet -## Overview +| Target | Host | Service | Port | +|---|---|---|---:| +| Dankserver | `dank@dankserver` | `frank-default-small` | 7770 | +| Dankclaw | `dank@dankclaw` | `frank-default-large` | 7770 | -Two servers run LLM inference services. Each server pulls its scripts from -this repository. The deployment process moves a committed change from the -repository to a running service. - -| Target | Host | Runtime | Service | Port | -|--------|------|---------|---------|------| -| Dankserver | dank@dankserver | upstream llama.cpp (Ornith-1.5-35B-A3B-CRACK Q6_K, native MTP draft, selected MoE experts offloaded via --override-tensor) | frank-default-small | 7770 | -| Dankclaw | dank@dankclaw | charlie12345/ROCmFPX llama.cpp (kingjones Qwen3.8 Uncensored Q6_0 ROCmFPX AGENT, standalone MTP, 262K ctx / 1 slot) | frank-default-large | 7770 | - -## Workflow - -The deployment workflow has seven steps. +## Scoped workflow ```mermaid graph LR - A[Edit scripts locally] --> B[Commit] - B --> C[Push to origin
Gitea on Dankclaw] - C --> D[Pull on target
git pull] - D --> E[Restart podman stack] - E --> F[Relink + restart LLM unit] - F --> G[Verify health endpoints] + A[Edit LLM files] --> B[Run tests] + B --> C[Commit main] + C --> D[deploy.sh selects matching service] + D --> E[Target pulls ff-only] + E --> F[Relink one unit] + F --> G[Restart one inference service] + G --> H[Wait for that service health] ``` -1. Fetch the current script from the target machine. -2. Commit the fetched version in this repository. -3. Edit the script locally in the working tree. -4. Run the tests (`./tests/run.sh` must pass). -5. Commit and push the change to origin. -6. Pull the change on the target machine. -7. Restart the services and verify that they are healthy. +Run the unified deployer from the repository root: -## Deploy Script - -The `scripts/deploy.sh` script is the unified deployer. One run per target -covers both workloads on that host: the podman stack and the LLM service. -Run it from the repository root. - -Run this command to deploy to Dankserver: - -``` -./scripts/deploy.sh dankserver +```bash +./scripts/deploy.sh dankserver --service frank-default-small +./scripts/deploy.sh dankclaw --service frank-default-large +./scripts/deploy.sh dankclaw --dry-run ``` -Run this command to deploy to Dankclaw: +Without `--service`, `scripts/deploy-plan.py` maps changed files to the service +on the selected host. The following files select the corresponding service: -``` -./scripts/deploy.sh dankclaw +- `scripts/frank-default-small.env`, `frank-default-small.service`, and + `serve-frank-small.sh` select `frank-default-small` on Dankserver. +- `scripts/frank-default-large.env`, `frank-default-large.service`, and + `serve-frank-large.sh` select `frank-default-large` on Dankclaw. +- `scripts/chat-template.jinja` selects the LLM on the selected host only. +- Documentation, tests, deployment tooling, and unrelated container changes + do not select either inference service. + +The target-side `scripts/deploy-remote.sh` relinks the selected unit into +`/etc/systemd/system/`, reloads systemd, enables it, restarts it, and waits for +`http://127.0.0.1:7770/health`. The wait budget is 20 minutes because cold model +loads can be slow. A failed handler does not advance its per-service state. + +## Unit and artifact paths + +Both units point into the stable target checkout: + +```text +/home/dank/serverAppConfigs/scripts/frank-default-small.service +/home/dank/serverAppConfigs/scripts/frank-default-large.service ``` -Add the `--dry-run` flag to preview without making changes: +The unit files and serve scripts validate their model/runtime artifacts before +starting. Keep `REMOTE_DIR` stable; moving the checkout requires updating both +the unit absolute paths and the deployment tests in the same change. -``` -./scripts/deploy.sh dankserver --dry-run +## NPU unit + +`frank-default-npu` is an inactive, historical Dankclaw endpoint. It is +relinked only when explicitly selected with `--service frank-default-npu`; the +large-model handler never restarts it. This is intentional: the NPU is not part +of the active inference path and should not be disturbed by a large-model +change. + +## Isolation guarantee + +The remote runner has one explicit service dispatch. It does not run a +host-wide LLM loop, `podman-compose up` for unrelated services, or a blanket +systemd restart. Each successful service is recorded under the target user's +state directory outside Git: + +```text +~/.local/state/serverAppConfigs/deploy-state/ ``` -The script does these actions: +On a later ordinary deployment, a service remains pending only while its +recorded revision differs from the checkout and the planner still finds a +change in that service's inputs. -1. Verifies that the local working tree has no uncommitted changes. -2. Pushes the `main` branch to origin. -3. Pulls the repository on the target machine. -4. Restarts the podman stack (Dankclaw only. The Dankserver media stack - runs as bare containers). -5. Relinks the LLM systemd units, reloads systemd, enables, and restarts - the active LLM service. -6. Waits for the Caddy homepage and the LLM health endpoint (over ssh). +## Manual recovery -Set `FRANK_SUDO_PW` when running headless on a host without passwordless -sudo, so that remote sudo reads the password from stdin (`sudo -S`). The -password is never stored in this repo. Both fleet hosts currently allow -passwordless sudo for the deploy user (verified 2026-08-22). The -passwordless restart helpers from the retired topology are obsolete and are -not used. +If the LLM handler fails after the target pull: -Health endpoints: +1. Inspect `journalctl -u frank-default-small -n 200` or + `journalctl -u frank-default-large -n 200`. +2. Confirm the model files, custom runtime binary, and chat template are + readable by `dank`. +3. Retry only the affected service: -- Dankserver: `http://127.0.0.1:7770/health` -- Dankclaw: `http://127.0.0.1:7770/health` + ```bash + ./scripts/deploy.sh dankserver --service frank-default-small + # or + ./scripts/deploy.sh dankclaw --service frank-default-large + ``` -## Manual Update +4. Verify the matching health endpoint before declaring recovery. -### Update Dankserver (frank-default-small) - -Run these commands on Dankserver: - -``` -cd /home/dank/serverAppConfigs -git pull origin main -sudo ln -sf /home/dank/serverAppConfigs/scripts/frank-default-small.service /etc/systemd/system/frank-default-small.service -sudo systemctl daemon-reload -sudo systemctl restart frank-default-small -curl -sf http://127.0.0.1:7770/health -``` - -### Update Dankclaw (frank-default-large) - -Run these commands on Dankclaw: - -``` -cd /home/dank/serverAppConfigs -git pull origin main -sudo ln -sf /home/dank/serverAppConfigs/scripts/frank-default-large.service /etc/systemd/system/frank-default-large.service -sudo systemctl daemon-reload -sudo systemctl restart frank-default-large -curl -sf http://127.0.0.1:7770/health -``` - -## Service Units - -### Dankserver — frank-default-small - -The active service on Dankserver is upstream llama.cpp (b10121) serving -`Ornith-1.5-35B-A3B-CRACK-Q6_K.gguf` under the alias `frank-default-small` -on port 7770. It runs the stock `/home/dank/llama.cpp/build/bin/llama-server` -binary with expert tensors selected by an explicit `--override-tensor` -regex. Expert tensors from every full-attention block and blocks 35 through 40 -stay on CUDA; other expert tensors are CPU-resident. The service uses 7 CPU -threads, ngram-mod speculative decoding only, f16 KV, context 491520 (3 slots -of 163840 each), batch -4096, ubatch 1024, and `--no-mmap`. The unit is -`scripts/frank-default-small.service`. It launches through -`scripts/serve-frank-small.sh`, which reads tunables from -`scripts/frank-default-small.env`. It uses `ubatch=1024` and the chat template -from `peculiar-ragdoll/Qwen-Sharp-Chat-Templates`. See [[dealign-ornith-crack]]. - -### Dankclaw — frank-default-large - -The active service on Dankclaw uses the ROCmFPX llama.cpp runtime to serve -[kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF](https://huggingface.co/kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF) -under the alias `frank-default-large` on port 7770. The main model is -`Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf`; the standalone MTP draft is -`mtp-Qwen3.8-27B-Q4_0.gguf`, and the Q8_0 vision projector is loaded alongside -it. The K-side KV cache remains `f16`; the V-side cache uses `turbo4`. The -service uses the 262144 context / 1-slot profile. The unit file is -`scripts/frank-default-large.service`; it launches through -`scripts/serve-frank-large.sh`, which reads tunables from -`scripts/frank-default-large.env`, applying the Qwen-Sharp chat template. - -> The Q8_0 AGENT experiment was retired on 2026-09-01 after repeated -> `set_rows` crashes and device-local allocation failures. The former caf-eemin -> ROCmI4 embedded-MTP model and the v3 PromptForge payload are historical -> deployments. The intermediate DFlash2 Vulkan deployment was retired on -> 2026-08-21. See [[kingjones-q8agent-rocmfpx]] and [[dflash2-large-vulkan]]. - -## Validation - -The deploy script checks the health endpoint after a restart. For deeper -checks, run a completion request against the endpoint: - -``` -curl -s http://127.0.0.1:7770/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model":"frank-default-small","messages":[{"role":"user","content":"Say OK"}]}' -``` - -## Troubleshooting - -If the health check fails after a deploy: - -1. Read the service logs with `journalctl -u -n 200`. -2. Confirm that the model files exist in the cache. -3. Confirm that the GPU has free memory (`nvidia-smi` on Dankserver, `rocm-smi` on Dankclaw). -4. Check the service status with `systemctl status `. - -## Incident: HIP Runtime Crash From the `TEMP` Env Variable (2026-08-13) - -On 2026-08-13 the old `frank-default-large` service crash-looped for hours -on Dankclaw. Root cause: the env files exported `TEMP=1.0` (a sampling -default). The ROCm HIP runtime reads `TEMP` as its JIT kernel-cache -directory, used the path `1.0`, failed, and segfaulted on a NULL object. -Fix: rename the sampling variable to `TEMPERATURE` (commit `227ce01`). - -If a service segfaults at load with no obvious cause, compare the exported -env against the HIP runtime's env reads. Do not name any tunable `TEMP`, -`HOST`, or `GPU_MAX_HEAP_SIZE`. - -## Incident: Jobs "Released Prematurely" Under Parallel Load (2026-08-13) - -Historical: on the retired topology, parallel jobs on the Dankserver 35B -service died with a cuBLAS crash under researcher load. The affected -service is retired. A client abort logs `W srv stop: cancel task, -id_task = N`. A natural stop logs only `slot release` with -`truncated = 0`. The server never preempts a running generation. - -## Incident: frank-default-small Crash Loop After Distro Upgrade (2026-08-21) - -On 2026-08-21 a release upgrade moved Dankserver from Ubuntu 24.04 to -Ubuntu 26.04 (Resolute Raccoon). The upgrade removed the CUDA 12 runtime -packages: `libcudart12`, `libcublas12`, `libcublaslt12`, and -`nvidia-cuda-toolkit`. The `llama-server` binary links -`libcudart.so.12` and `libcublas.so.12`. On every start the binary exited -with code 127. systemd restart-looped the service continuously. The health -endpoint on port 7770 stayed down. - -Fix: reinstall the runtime packages. The same versions remain in the 26.04 -repository. - - sudo apt-get install -y libcudart12 libcublas12 libcublaslt12 - -Verify that the binary resolves all libraries: - - ldd /home/dank/llama.cpp/build/bin/llama-server | grep "not found" - -Start the service and check the health endpoint on port 7770. - -After any release upgrade on Dankserver, run the `ldd` check above. A -missing CUDA 12 library means the packages need a reinstall. +Do not stop the other host's inference service during recovery. For benchmark +work, follow the separate stop/benchmark/restart procedure in [[Testing Policy]] +and the repository rules. ## See Also -- [AGENTS.md](../../AGENTS.md) — the git workflow rules -- [README.md](../../README.md) — repository overview -- [index.md](index.md) — LLM fleet knowledge base index -- [frank-redo-2026-08-17.md](frank-redo-2026-08-17.md) — the full redo runbook -- [docs/index.md](../index.md) — the project knowledge base (podman stacks) +- [Deployment Flow](../Architecture/Deployment%20Flow.md) +- [[index]] +- [Dankclaw deployment architecture](../Architecture/Network%20Architecture.md) diff --git a/docs/LLM/index.md b/docs/LLM/index.md index 836e177..6776ab0 100644 --- a/docs/LLM/index.md +++ b/docs/LLM/index.md @@ -120,5 +120,5 @@ MTP draft + Turbo4 V cache, 262K ctx / 1 slot] - [Project knowledge base](../index.md) — the Podman app stacks on the same two hosts. OpenWebUI consumes the `:7770` endpoint on Dankserver. The Gitea instance on Dankclaw hosts this repository. -- [Deployment Flow](../Architecture/Deployment%20Flow.md) — the unified - git-based deploy for both workloads \ No newline at end of file +- [Deployment Flow](../Architecture/Deployment%20Flow.md) — the service-scoped + git-based deploy with inference isolation \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 0537829..1377b00 100644 --- a/docs/index.md +++ b/docs/index.md @@ -27,7 +27,7 @@ covers both workloads). - [[Network Architecture]] — Caddy reverse proxy, hostname routing, Gluetun VPN path - [[Storage Policy]] — bind mounts, data locations, ownership -- [[Deployment Flow]] — git-based deploy, target map, health checks +- [[Deployment Flow]] — service-scoped git deploy, target map, and health checks - [[Testing Policy]] — the test seams and how to run them ## Dankserver Storage diff --git a/scripts/deploy-open-webui-quadlet.sh b/scripts/deploy-open-webui-quadlet.sh index d430f78..5db6c1c 100755 --- a/scripts/deploy-open-webui-quadlet.sh +++ b/scripts/deploy-open-webui-quadlet.sh @@ -1,104 +1,14 @@ #!/usr/bin/env bash -# deploy-open-webui-quadlet.sh — Install and (re)start open-webui on dankclaw. -# -# open-webui is boot-persistent via Quadlet (see AGENTS.md). This script: -# 1. Pushes main to origin -# 2. Pulls the repo on the target -# 3. Creates the data dir and copies the unit + network into ~/.config/containers/systemd/ -# 4. daemon-reloads and starts open-webui (first run pulls the image) -# -# It is idempotent: it stops any running unit and removes any container with the -# name (a prior run or a bare-container leftover) before reinstalling and -# starting. The data dir is preserved across reruns, so existing chats survive. -# -# Usage: -# ./scripts/deploy-open-webui-quadlet.sh [--dry-run] -# ./scripts/deploy-open-webui-quadlet.sh dankclaw +#!/usr/bin/env bash +# Compatibility wrapper for the service-scoped deployment dispatcher. set -euo pipefail -REMOTE_USER="dank" -REMOTE_HOST="dankclaw" -REMOTE_DIR="/home/dank/serverAppConfigs" -QDIR="dankclaw/open-webui/quadlet" -UNIT="open-webui.service" -DATA_DIR="/home/dank/Containers/open-webui/data" - TARGET="${1:-}" -DRY_RUN=0 -if [ "${2:-}" = "--dry-run" ]; then - DRY_RUN=1 -fi - -if [ -z "${TARGET:-}" ]; then - echo "Usage: $0 [--dry-run]" - exit 1 -fi -if [ "${TARGET}" != "dankclaw" ]; then - echo "This stack runs only on dankclaw (got: ${TARGET})" - exit 1 -fi - -# --------------------------------------------------------------------------- -# Local pre-flight -# --------------------------------------------------------------------------- -cd "$(dirname "$0")/.." -if ! git diff --quiet HEAD; then - echo "FAIL: Local working tree has uncommitted changes. Commit first." - exit 1 -fi - -if [ "$DRY_RUN" = "1" ]; then - echo "[dry-run] Would push main, pull on ${REMOTE_HOST}, install ${UNIT}, and start it (data at ${DATA_DIR})." - exit 0 -fi - -echo "=== Deploying open-webui quadlet to ${REMOTE_HOST} ===" -echo "[1/3] Pushing local main to origin..." -git push origin main - -REMOTE="${REMOTE_USER}@${REMOTE_HOST}" - -echo "[2/3] Installing and starting ${UNIT} on ${REMOTE_HOST}..." -ssh "$REMOTE" bash -s </dev/null || true - -echo ' Removing any container with the name (prior run or bare-container leftover)...' -podman rm -f open-webui 2>/dev/null || true - -echo ' Creating the data dir (preserved across reruns)...' -mkdir -p '${DATA_DIR}' - -echo ' Copying the quadlet unit + network into ~/.config/containers/systemd/ ...' -mkdir -p ~/.config/containers/systemd -cp ${QDIR}/open-webui.container ~/.config/containers/systemd/ -cp ${QDIR}/open-webui.network ~/.config/containers/systemd/ - -echo ' daemon-reload (generates the unit)...' -# Quadlet units are generator-managed: systemctl enable fails on them. The -# [Install] WantedBy=default.target in the .container file is itself the -# enablement (is-enabled reports "generated"); with user linger on it starts at -# boot. So: daemon-reload, then start. -systemctl --user daemon-reload - -echo ' Starting (first run pulls the image; this can take a while)...' -systemctl --user start '${UNIT}' - -echo ' Unit state:' -systemctl --user --no-pager status '${UNIT}' 2>&1 | grep -E "●|Active:" || true -REMOTE_EOF - -echo "[3/3] Verifying open-webui is up..." -ssh "$REMOTE" ' - echo " is-enabled: $(systemctl --user is-enabled open-webui.service 2>&1) (boot-persistent)" - echo " port 3005:"; (ss -tlnp 2>/dev/null || netstat -tlnp 2>/dev/null) | grep -E ":3005\b" || echo " (not listening yet)" - echo " HTTP probe:"; curl -s -o /dev/null -w " http://127.0.0.1:3005 -> %{http_code}\n" -m 10 http://127.0.0.1:3005/ || echo " (no HTTP yet)" -' || true - -echo "=== open-webui quadlet deployed to ${REMOTE_HOST} ===" +shift || true +case "$TARGET" in + dankclaw) exec "$(dirname "$0")/deploy.sh" dankclaw --service open-webui "$@" ;; + *) + echo "This stack runs only on dankclaw (got: ${TARGET:-})" >&2 + exit 2 + ;; + esac diff --git a/scripts/deploy-pihole-quadlet.sh b/scripts/deploy-pihole-quadlet.sh index d832784..b09ae96 100755 --- a/scripts/deploy-pihole-quadlet.sh +++ b/scripts/deploy-pihole-quadlet.sh @@ -1,118 +1,13 @@ #!/usr/bin/env bash -# deploy-pihole-quadlet.sh — Install and (re)start the Pi-hole LAN DNS on -# dankclaw. Pi-hole retired the dnsmasq quadlet (2026-08-24). -# -# Pi-hole is boot-persistent via Quadlet (see AGENTS.md). This script: -# 1. Pushes main to origin -# 2. Pulls the repo on the target -# 3. Creates the data dirs and copies the quadlet unit + env + addn-hosts -# 4. daemon-reloads and starts pihole (first run pulls the image) -# -# It is idempotent: it stops any running unit and removes any container with -# the name (a prior run or a bare-container leftover) before reinstalling and -# starting. The data dirs (etc-pihole, etc-dockers) are preserved across -# reruns, so the blocklist and web settings survive. The env file is created -# from env.example only if absent; fill the real WEBPASSWORD before the web -# UI is usable. -# -# Usage: -# ./scripts/deploy-pihole-quadlet.sh [--dry-run] -# ./scripts/deploy-pihole-quadlet.sh dankclaw +# Compatibility wrapper for the service-scoped deployment dispatcher. set -euo pipefail -REMOTE_USER="dank" -REMOTE_HOST="dankclaw" -REMOTE_DIR="/home/dank/serverAppConfigs" -QDIR="dankclaw/pihole" -UNIT="pihole.service" -DATA_DIR="/home/dank/Containers/pihole" - TARGET="${1:-}" -DRY_RUN=0 -if [ "${2:-}" = "--dry-run" ]; then - DRY_RUN=1 -fi - -if [ -z "${TARGET:-}" ]; then - echo "Usage: $0 [--dry-run]" - exit 1 -fi -if [ "${TARGET}" != "dankclaw" ]; then - echo "This stack runs only on dankclaw (got: ${TARGET})" - exit 1 -fi - -# --------------------------------------------------------------------------- -# Local pre-flight -# --------------------------------------------------------------------------- -cd "$(dirname "$0")/.." -if ! git diff --quiet HEAD; then - echo "FAIL: Local working tree has uncommitted changes. Commit first." - exit 1 -fi - -if [ "$DRY_RUN" = "1" ]; then - echo "[dry-run] Would push main, pull on ${REMOTE_HOST}, install ${UNIT}, and start it (data at ${DATA_DIR})." - exit 0 -fi - -echo "=== Deploying pihole quadlet to ${REMOTE_HOST} ===" -echo "[1/3] Pushing local main to origin..." -git push origin main - -REMOTE="${REMOTE_USER}@${REMOTE_HOST}" - -echo "[2/3] Installing and starting ${UNIT} on ${REMOTE_HOST}..." -ssh "$REMOTE" bash -s </dev/null || true - -echo ' Removing any container with the name (prior run or bare-container leftover)...' -podman rm -f pihole 2>/dev/null || true - -echo ' Creating the data dirs (preserved across reruns)...' -mkdir -p '${DATA_DIR}/etc-pihole' '${DATA_DIR}/etc-dockers' - -echo ' Creating the env file from env.example if absent...' -if [ ! -f '${DATA_DIR}/env' ]; then - cp ${QDIR}/env.example '${DATA_DIR}/env' - echo ' env created: set the real WEBPASSWORD before the web UI is usable' -else - echo ' env present, keeping it' -fi - -echo ' Copying the record file (addn-hosts)...' -cp ${QDIR}/addn-hosts '${DATA_DIR}/addn-hosts' - -echo ' Copying the quadlet unit into ~/.config/containers/systemd/ ...' -mkdir -p ~/.config/containers/systemd -cp ${QDIR}/quadlet/pihole.container ~/.config/containers/systemd/ - -echo ' daemon-reload (generates the unit)...' -# Quadlet units are generator-managed: systemctl enable fails on them. The -# [Install] WantedBy=default.target in the .container file is itself the -# enablement (is-enabled reports "generated"); with user linger on it starts -# at boot. So: daemon-reload, then start. -systemctl --user daemon-reload - -echo ' Starting (first run pulls the image; this can take a while)...' -systemctl --user start '${UNIT}' - -echo ' Unit state:' -systemctl --user --no-pager status '${UNIT}' 2>&1 | grep -E "●|Active:" || true -REMOTE_EOF - -echo "[3/3] Verifying pihole is up..." -ssh "$REMOTE" ' - echo " is-enabled: $(systemctl --user is-enabled pihole.service 2>&1) (boot-persistent)" - echo " port 53:"; (ss -ulpn 2>/dev/null) | grep -E ":53\b" || echo " (not listening yet)" - echo " resolution:"; dig @127.0.0.1 romm.franksdank.cloud +short 2>/dev/null || echo " (no answer yet)" -' || true - -echo "=== pihole quadlet deployed to ${REMOTE_HOST} ===" +shift || true +case "$TARGET" in + dankclaw) exec "$(dirname "$0")/deploy.sh" dankclaw --service pihole "$@" ;; + *) + echo "This stack runs only on dankclaw (got: ${TARGET:-})" >&2 + exit 2 + ;; +esac diff --git a/scripts/deploy-plan.py b/scripts/deploy-plan.py new file mode 100755 index 0000000..f5ddbb1 --- /dev/null +++ b/scripts/deploy-plan.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +"""Plan runtime services affected by repository changes. + +The planner is intentionally side-effect free. It accepts either an explicit +path list or two Git revisions. The latter lets compose-file changes resolve +to only the service definitions that actually changed instead of restarting an +entire compose project. +""" +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import PurePosixPath +from typing import Any + +import yaml + +# Dependency-sensitive services come before their consumers. Independent +# compose services follow. LLM services are deliberately last and are never +# implied by a generic host, stack, documentation, or config change. +SERVICE_ORDER = ( + "gluetun", + "searxng-valkey", + "searxng", + "pihole", + "open-webui", + "gitea-postgres", + "gitea-server", + "gitea-caddy", + "comfyui", + "hermes", + "hermes-dashboard", + "prometheus", + "grafana", + "loki", + "alloy", + "node-exporter", + "authentik-postgres", + "authentik-server", + "authentik-worker", + "caddy", + "homepage", + "mariadb", + "redis", + "romm", + "copyparty", + "jellyfin", + "calibre-web", + "prowlarr", + "qbittorrent", + "radarr", + "sonarr", + "lidarr", + "readarr", + "frank-default-large", + "frank-default-npu", + "frank-default-small", +) + +DANKCLAW_SERVICES = { + "gluetun", + "searxng-valkey", + "searxng", + "pihole", + "open-webui", + "gitea-postgres", + "gitea-server", + "gitea-caddy", + "comfyui", + "hermes", + "hermes-dashboard", + "prometheus", + "grafana", + "loki", + "alloy", + "node-exporter", + "authentik-postgres", + "authentik-server", + "authentik-worker", + "caddy", + "homepage", + "frank-default-large", + "frank-default-npu", +} + +DANKSERVER_SERVICES = { + "mariadb", + "redis", + "romm", + "copyparty", + "jellyfin", + "calibre-web", + "prowlarr", + "qbittorrent", + "radarr", + "sonarr", + "lidarr", + "readarr", + "homepage", + "frank-default-small", +} + +DANKCLAW_COMPOSE_SERVICES = { + "comfyui", + "hermes", + "hermes-dashboard", + "prometheus", + "grafana", + "loki", + "alloy", + "node-exporter", + "authentik-postgres", + "authentik-server", + "authentik-worker", + "caddy", +} + +DANKSERVER_COMPOSE_SERVICES = { + "mariadb", + "redis", + "romm", + "copyparty", + "jellyfin", + "calibre-web", + "prowlarr", + "qbittorrent", + "radarr", + "sonarr", + "lidarr", + "readarr", +} + + +def _add(services: set[str], *names: str) -> None: + services.update(names) + + +def target_services(target: str) -> set[str]: + if target == "dankclaw": + return DANKCLAW_SERVICES + if target == "dankserver": + return DANKSERVER_SERVICES + raise ValueError(f"unknown target: {target}") + + +def _compose_services(target: str) -> set[str]: + return DANKCLAW_COMPOSE_SERVICES if target == "dankclaw" else DANKSERVER_COMPOSE_SERVICES + + +def _git_show(repo: str, revision: str, path: str) -> str | None: + result = subprocess.run( + ["git", "-C", repo, "show", f"{revision}:{path}"], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + return result.stdout if result.returncode == 0 else None + + +def _yaml_document(contents: str | None) -> dict[str, Any]: + if contents is None: + return {} + try: + data = yaml.safe_load(contents) or {} + except yaml.YAMLError: + return {} + return data if isinstance(data, dict) else {} + + +def _yaml_services(contents: str | None) -> dict[str, Any]: + services = _yaml_document(contents).get("services", {}) + return services if isinstance(services, dict) else {} + + +def compose_changes(repo: str, target: str, base: str, head: str) -> set[str]: + """Return only compose services whose YAML definitions or dependencies changed.""" + path = f"{target}/compose.yaml" + old_doc = _yaml_document(_git_show(repo, base, path)) + new_doc = _yaml_document(_git_show(repo, head, path)) + old = old_doc.get("services", {}) if isinstance(old_doc.get("services", {}), dict) else {} + new = new_doc.get("services", {}) if isinstance(new_doc.get("services", {}), dict) else {} + names = set(old) | set(new) + allowed = _compose_services(target) + + # An unavailable or malformed revision is safer as a broad compose plan + # than as a silent no-op. It still cannot include LLM units. + if (old_doc and not new_doc) or (new_doc and not old_doc and names): + return allowed + + # Top-level networks/configuration can affect every compose service even + # when no individual service mapping changed. Ignore the services key while + # comparing the rest of the Compose document. + old_globals = {key: value for key, value in old_doc.items() if key != "services"} + new_globals = {key: value for key, value in new_doc.items() if key != "services"} + if old_globals != new_globals: + return allowed + + return {name for name in names if name in allowed and old.get(name) != new.get(name)} + + +def services_for_path(path: str, target: str) -> set[str]: + """Return services affected by one repository-relative path.""" + path = path.replace("\\", "/") + p = PurePosixPath(path) + parts = p.parts + services: set[str] = set() + if not parts: + return services + + # Non-runtime material is deliberately a no-op. + if parts[0] in {"docs", "tests", "old", ".githooks"}: + return services + if path in {"README.md", "AGENTS.md"} or parts[0] == ".obsidian": + return services + + if parts[0] == "shared": + if len(parts) < 2: + return services + group = parts[1] + if group == "searxng": + _add(services, "searxng") + elif group == "prometheus": + _add(services, "prometheus") + elif group == "loki": + _add(services, "loki") + elif group == "alloy": + _add(services, "alloy") + elif group == "homepage": + # Homepage is a Dankclaw-owned deployment. Shared homepage files + # are intentionally not copied during a Dankserver-only run. + if target == "dankclaw": + _add(services, "homepage") + return services + + if parts[0] == "dankclaw": + if len(parts) >= 2 and parts[1] == "compose.yaml": + # Revision-aware callers replace this broad fallback with the + # service-level YAML comparison in compose_changes(). + _add(services, *DANKCLAW_COMPOSE_SERVICES) + return services + if len(parts) >= 2 and parts[1] == "Caddyfile": + _add(services, "caddy") + return services + if len(parts) >= 2 and parts[1] == "Containerfile.comfyui": + _add(services, "comfyui") + return services + if len(parts) >= 2 and parts[1] in {"caddy", "comfyui", "loki", "alloy", "hermes"}: + _add(services, parts[1]) + return services + if len(parts) >= 2 and parts[1] == "searxng": + if len(parts) >= 4 and parts[2] == "quadlet": + name = parts[3] + if name == "gluetun.container": + # SearXNG shares Gluetun's network namespace. A Gluetun + # recreation requires both units to be restarted. + _add(services, "gluetun", "searxng") + elif name == "searxng.container": + _add(services, "searxng") + elif name == "searxng-valkey.container": + _add(services, "searxng-valkey", "searxng") + elif name == "searxng.network": + _add(services, "gluetun", "searxng-valkey", "searxng") + return services + if len(parts) >= 2 and parts[1] == "pihole": + _add(services, "pihole") + return services + if len(parts) >= 2 and parts[1] == "open-webui": + _add(services, "open-webui") + return services + if len(parts) >= 2 and parts[1] == "gitea": + if len(parts) >= 4 and parts[2] == "quadlet": + name = parts[3] + if name == "gitea-postgres.container": + _add(services, "gitea-postgres") + elif name == "gitea-server.container": + _add(services, "gitea-server") + elif name == "gitea-caddy.container": + _add(services, "gitea-caddy") + elif name == "gitea.network": + _add(services, "gitea-postgres", "gitea-server", "gitea-caddy") + return services + return services + + if parts[0] == "dankserver": + if len(parts) >= 2 and parts[1] == "compose.yaml": + _add(services, *DANKSERVER_COMPOSE_SERVICES) + return services + + if parts[0] == "scripts": + name = p.name + if name in {"frank-default-large.env", "frank-default-large.service", "serve-frank-large.sh"}: + _add(services, "frank-default-large") + elif name in {"frank-default-npu.env", "frank-default-npu.service", "serve-frank-default-npu.sh"}: + _add(services, "frank-default-npu") + elif name in {"frank-default-small.env", "frank-default-small.service", "serve-frank-small.sh"}: + _add(services, "frank-default-small") + elif name == "chat-template.jinja": + # Only the selected target's LLM consumes this shared template. + _add(services, "frank-default-large" if target == "dankclaw" else "frank-default-small") + elif name in {"homepage-status.py", "homepage-status.service", "homepage-status.timer"}: + _add(services, "homepage") + # Deployment/test/benchmark scripts and other repo tooling do not + # restart a runtime service. + return services + + # Examples and env templates are not deployed runtime state. + if p.name in {".env.example", "env.example"}: + return services + return services + + +def changed_paths(repo: str, base: str, head: str) -> list[str]: + result = subprocess.run( + ["git", "-C", repo, "diff", "--name-only", "--diff-filter=ACDMRTUXB", base, head], + text=True, + stdout=subprocess.PIPE, + check=True, + ) + return [line for line in result.stdout.splitlines() if line] + + +def plan(paths: list[str], target: str, repo: str | None = None, + base: str | None = None, head: str | None = None) -> list[str]: + services: set[str] = set() + for path in paths: + if path == f"{target}/compose.yaml" and repo and base and head: + services.update(compose_changes(repo, target, base, head)) + else: + services.update(services_for_path(path, target)) + allowed = target_services(target) + return [name for name in SERVICE_ORDER if name in services and name in allowed] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--target", choices=("dankclaw", "dankserver"), required=True) + parser.add_argument("--paths", nargs="+") + parser.add_argument("--repo", help="repository path for revision-aware planning") + parser.add_argument("--base", help="base Git revision") + parser.add_argument("--head", help="head Git revision") + args = parser.parse_args() + + if not args.paths and not (args.repo and args.base and args.head): + parser.error("provide --paths or all of --repo, --base, and --head") + paths = args.paths or changed_paths(args.repo, args.base, args.head) + print("\n".join(plan(paths, args.target, args.repo, args.base, args.head))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh new file mode 100755 index 0000000..4b68057 --- /dev/null +++ b/scripts/deploy-remote.sh @@ -0,0 +1,474 @@ +#!/usr/bin/env bash +# deploy-remote.sh — apply an explicit service plan on one target host. +# +# This script runs on the target checkout. It deliberately does not infer a +# plan, pull Git, or restart an unrelated service. deploy.sh computes the plan +# from the pushed revision and passes the selected service names here. +# Per-service revisions live in /var/lib/serverAppConfigs/deploy-state (or +# DEPLOY_STATE_DIR), outside the checkout, so an unselected change remains +# pending until its service is explicitly deployed. +set -euo pipefail + +TARGET="${1:-}" +REVISION="${2:-}" +shift 2 || true +FROM_REVISION="" +PENDING=0 +SERVICES=() +while [[ "$#" -gt 0 ]]; do + case "$1" in + --from-revision) + [[ "$#" -ge 2 ]] || { echo "--from-revision requires a revision" >&2; exit 2; } + FROM_REVISION="$2" + shift 2 + ;; + --pending) + PENDING=1 + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + SERVICES+=("$1") + shift + ;; + esac +done + +REMOTE_DIR="${REMOTE_DIR:-/home/dank/serverAppConfigs}" +STATE_DIR="${DEPLOY_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/serverAppConfigs/deploy-state}" + +run_privileged() { + if [[ -n "${FRANK_SUDO_PW:-}" ]]; then + printf '%s\n' "$FRANK_SUDO_PW" | sudo -S -- "$@" + else + sudo -- "$@" + fi +} + +usage() { + cat <<'EOF' +Usage: deploy-remote.sh [--pending] [service]... + +The service list is explicit. An empty list is a successful no-op. +EOF +} + +if [[ "$TARGET" != dankclaw && "$TARGET" != dankserver ]]; then + usage >&2 + exit 2 +fi +if [[ -z "$REVISION" ]]; then + usage >&2 + exit 2 +fi +if [[ "$REVISION" == *[!0-9a-fA-F]* ]]; then + echo "invalid Git revision: $REVISION" >&2 + exit 2 +fi +if ! git -C "$REMOTE_DIR" cat-file -e "$REVISION^{commit}" 2>/dev/null; then + echo "revision is not a commit in the target checkout: $REVISION" >&2 + exit 2 +fi +cd "$REMOTE_DIR" + +# Keep the state directory outside Git. The file is written only after a +# handler completes successfully; an interrupted handler remains pending. +mkdir -p "$STATE_DIR" + +service_is_allowed() { + local service="$1" + case "$TARGET:$service" in + dankclaw:gluetun|dankclaw:searxng-valkey|dankclaw:searxng|dankclaw:pihole|\ + dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\ + dankclaw:gitea-caddy|dankclaw:comfyui|dankclaw:hermes|\ + dankclaw:hermes-dashboard|dankclaw:prometheus|dankclaw:grafana|\ + dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|\ + dankclaw:authentik-postgres|dankclaw:authentik-server|\ + dankclaw:authentik-worker|dankclaw:caddy|dankclaw:homepage|\ + dankclaw:frank-default-large|dankclaw:frank-default-npu|\ + dankserver:mariadb|dankserver:redis|dankserver:romm|\ + dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|\ + dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\ + dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\ + dankserver:frank-default-small) + return 0 + ;; + *) + return 1 + ;; + esac +} + +mark_status() { + local service="$1" status="$2" tmp + tmp="$(mktemp "$STATE_DIR/.${service}.XXXXXX")" + printf 'revision=%s\nstatus=%s\n' "$REVISION" "$status" > "$tmp" + mv -f -- "$tmp" "$STATE_DIR/$service" +} + +mark_deployed() { + mark_status "$1" deployed +} + +state_revision() { + local service="$1" line value + if [[ -r "$STATE_DIR/$service" ]]; then + while IFS= read -r line; do + case "$line" in + revision=*) value="${line#revision=}" ;; + esac + done < "$STATE_DIR/$service" + printf '%s' "${value:-}" + fi +} + +state_status() { + local service="$1" line value + if [[ -r "$STATE_DIR/$service" ]]; then + while IFS= read -r line; do + case "$line" in + status=*) value="${line#status=}" ;; + esac + done < "$STATE_DIR/$service" + printf '%s' "${value:-pending}" + fi +} + +add_changed_services() { + local base="$1" + local service + [[ -n "$base" ]] || return 0 + while IFS= read -r service; do + [[ -n "$service" ]] || continue + SERVICES+=("$service") + done < <(python3 scripts/deploy-plan.py --target "$TARGET" --repo "$REMOTE_DIR" \ + --base "$base" --head "$REVISION") +} + +add_pending_services() { + local service recorded changed + for state_file in "$STATE_DIR"/*; do + [[ -f "$state_file" ]] || continue + service="${state_file##*/}" + recorded="$(state_revision "$service")" + [[ -n "$recorded" && "$recorded" != "$REVISION" ]] || continue + [[ "$(state_status "$service")" != deployed ]] || continue + if [[ "$TARGET" == dankclaw && "$service" == frank-default-small ]]; then continue; fi + if [[ "$TARGET" == dankserver && "$service" == frank-default-large ]]; then continue; fi + if ! changed="$(python3 scripts/deploy-plan.py --target "$TARGET" \ + --repo "$REMOTE_DIR" --base "$recorded" --head "$REVISION")"; then + echo "WARN: unable to compare state for $service; retaining it as pending" >&2 + SERVICES+=("$service") + continue + fi + if grep -Fxq "$service" <<< "$changed"; then + SERVICES+=("$service") + fi + done +} + +copy_shared_config() { + run_privileged mkdir -p /opt/searxng/config /opt/prometheus/config /opt/loki/config /opt/alloy/config + run_privileged cp shared/searxng/settings.yml /opt/searxng/config/settings.yml + run_privileged cp shared/searxng/limiter.toml /opt/searxng/config/limiter.toml + run_privileged cp shared/prometheus/prometheus.yml /opt/prometheus/config/prometheus.yml + run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml + run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy +} + +render_caddyfile() { + local hash + hash="$(grep -E '^CADDY_BASICAUTH_HASH=' .env 2>/dev/null | tail -1 | cut -d= -f2- || true)" + if [[ -z "$hash" ]]; then + echo "WARN: CADDY_BASICAUTH_HASH is not set; refusing to overwrite rendered Caddyfile" >&2 + return 1 + fi + run_privileged mkdir -p /home/dank/Containers/caddy + sed -e "s|REPLACE_WITH_HASH|$hash|g" dankclaw/Caddyfile > /tmp/serverAppConfigs.Caddyfile + run_privileged cp /tmp/serverAppConfigs.Caddyfile /home/dank/Containers/caddy/Caddyfile +} + +deploy_homepage() { + run_privileged mkdir -p /srv/homepage + run_privileged cp shared/homepage/index.html shared/homepage/admin.html shared/homepage/app.js \ + shared/homepage/style.css shared/homepage/services.json /srv/homepage/ + if [[ -d shared/homepage/fonts ]]; then + run_privileged cp -r shared/homepage/fonts /srv/homepage/ + fi + local deployed_at + deployed_at="$(git log -1 --format=%cI)" + printf 'window.DEPLOYED_AT = "%s";\n' "$deployed_at" > /tmp/deployed.js + run_privileged cp /tmp/deployed.js /srv/homepage/deployed.js + run_privileged chmod -R a+rX /srv/homepage + run_privileged cp scripts/homepage-status.py /usr/local/bin/homepage-status.py + run_privileged cp scripts/homepage-status.service scripts/homepage-status.timer /etc/systemd/system/ + run_privileged systemctl daemon-reload + run_privileged systemctl enable homepage-status.timer + run_privileged systemctl restart homepage-status.timer + run_privileged python3 /usr/local/bin/homepage-status.py /srv/homepage/services.json /srv/homepage/status.json +} + +deploy_caddy() { + copy_shared_config + render_caddyfile + deploy_homepage + local cfhash oldhash image + cfhash="$(sha256sum dankclaw/caddy/Containerfile | cut -d' ' -f1)" + image=franksdank/caddy-porkbun + oldhash="$(podman image inspect "$image:latest" --format '{{index .Config.Labels "cfhash"}}' 2>/dev/null || echo none)" + if [[ "$oldhash" != "$cfhash" ]]; then + podman build --no-cache --label "cfhash=$cfhash" -t "$image:latest" \ + -f dankclaw/caddy/Containerfile . + fi + # The canonical reverse proxy may be gitea-caddy on a host with the old + # split deployment. Never stop or replace it implicitly. Refuse the Caddy + # deployment until the operator has explicitly migrated port ownership. + if podman container exists gitea-caddy && ! podman container exists caddy; then + echo "FAIL: gitea-caddy owns the reverse-proxy deployment; refusing to replace it with caddy." >&2 + return 1 + fi + if podman container exists gitea-caddy && podman container exists caddy; then + local gitea_state caddy_state + gitea_state="$(podman inspect gitea-caddy --format '{{.State.Status}}' 2>/dev/null || true)" + caddy_state="$(podman inspect caddy --format '{{.State.Status}}' 2>/dev/null || true)" + if [[ "$gitea_state" == running && "$caddy_state" != running ]]; then + echo "FAIL: gitea-caddy currently owns ports 80/443; refusing to start caddy." >&2 + return 1 + fi + fi + if podman container exists caddy; then + podman-compose -f dankclaw/compose.yaml up -d --no-deps caddy + local reloaded=0 + for _ in 1 2 3 4 5; do + if podman exec caddy caddy reload --config /etc/caddy/Caddyfile \ + --adapter caddyfile >/dev/null 2>&1; then + reloaded=1 + echo "Caddy reloaded." + break + fi + sleep 3 + done + if [[ "$reloaded" -ne 1 ]]; then + echo "Caddy reload failed; restarting only caddy." + podman restart caddy + fi + else + echo "WARN: caddy container is absent; no proxy was stopped or replaced." + fi +} + +deploy_compose_service() { + local service="$1" + if [[ "$TARGET" == dankserver ]]; then + # Dankserver's compose file is the source-of-truth inventory, but its + # production containers are individually systemd-managed/bare. Do not + # use compose up here: it can create a second container with a wrong + # name and can interfere with a running workload. + echo "WARN: $service is recorded in dankserver/compose.yaml; production container management is host-specific and no restart was attempted." + return 0 + fi + case "$service" in + comfyui|hermes|hermes-dashboard|prometheus|grafana|loki|alloy|node-exporter|authentik-postgres|authentik-server|authentik-worker) + case "$service" in + comfyui|loki|alloy) + build_compose_image "$service" + ;; + esac + podman-compose -f dankclaw/compose.yaml up -d --no-deps "$service" + ;; + *) + echo "unknown compose service: $service" >&2 + return 2 + ;; + esac +} + +build_compose_image() { + local service="$1" image cfhash oldhash containerfile + case "$service" in + comfyui) + image=comfyui-rocm:gfx1151 + containerfile=dankclaw/Containerfile.comfyui + ;; + loki|alloy) + image="franksdank/$service:latest" + containerfile="dankclaw/$service/Containerfile" + ;; + *) + return 0 + ;; + esac + cfhash="$(sha256sum "$containerfile" | cut -d' ' -f1)" + oldhash="$(podman image inspect "$image" --format '{{index .Config.Labels "cfhash"}}' 2>/dev/null || echo none)" + if [[ "$oldhash" != "$cfhash" ]]; then + podman build --no-cache --label "cfhash=$cfhash" -t "$image" -f "$containerfile" . + fi +} + +quadlet_unit_dir="$HOME/.config/containers/systemd" +install_quadlet() { + local service="$1" + local source_dir="$2" + local unit="$3" + mkdir -p "$quadlet_unit_dir" + cp "$source_dir/$unit.container" "$quadlet_unit_dir/" + if [[ -f "$source_dir/$unit.network" ]]; then + cp "$source_dir/$unit.network" "$quadlet_unit_dir/" + fi + systemctl --user daemon-reload +} + +deploy_searxng() { + copy_shared_config + install_quadlet searxng dankclaw/searxng/quadlet searxng + install_quadlet searxng-valkey dankclaw/searxng/quadlet searxng-valkey + install_quadlet gluetun dankclaw/searxng/quadlet gluetun + # The network is shared by all three units; install it even when only the + # SearXNG service was selected. These copies are user-owned Quadlet state. + cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/" + systemctl --user daemon-reload + systemctl --user start searxng-network.service + systemctl --user restart gluetun.service searxng-valkey.service searxng.service +} + +deploy_gluetun() { + install_quadlet gluetun dankclaw/searxng/quadlet gluetun + cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/" + systemctl --user daemon-reload + systemctl --user restart gluetun.service searxng.service +} + +deploy_searxng_valkey() { + install_quadlet searxng-valkey dankclaw/searxng/quadlet searxng-valkey + cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/" + systemctl --user daemon-reload + systemctl --user restart searxng-valkey.service searxng.service +} + +deploy_open_webui() { + install_quadlet open-webui dankclaw/open-webui/quadlet open-webui + mkdir -p /home/dank/Containers/open-webui/data + systemctl --user daemon-reload + systemctl --user restart open-webui.service +} + +deploy_pihole() { + mkdir -p /home/dank/Containers/pihole/etc-pihole /home/dank/Containers/pihole/etc-dockers + if [[ ! -f /home/dank/Containers/pihole/env ]]; then + cp dankclaw/pihole/env.example /home/dank/Containers/pihole/env + echo "WARN: created Pi-hole env from the example; set WEBPASSWORD before using its UI" >&2 + fi + cp dankclaw/pihole/addn-hosts /home/dank/Containers/pihole/addn-hosts + mkdir -p "$quadlet_unit_dir" + cp dankclaw/pihole/quadlet/pihole.container "$quadlet_unit_dir/" + systemctl --user daemon-reload + systemctl --user restart pihole.service +} + +deploy_gitea_unit() { + local service="$1" + local unit="$2" + mkdir -p "$quadlet_unit_dir" + cp "dankclaw/gitea/quadlet/$unit.container" "$quadlet_unit_dir/" + [[ -f dankclaw/gitea/quadlet/gitea.network ]] && cp dankclaw/gitea/quadlet/gitea.network "$quadlet_unit_dir/" + systemctl --user daemon-reload + systemctl --user restart "$service.service" +} + +relink_unit() { + local service="$1" + run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "/etc/systemd/system/$service.service" +} + +deploy_llm_large() { + relink_unit frank-default-large + # NPU is intentionally not restarted here. It is a separate, retired + # endpoint and its unit is only relinked when explicitly selected. + run_privileged systemctl daemon-reload + run_privileged systemctl enable frank-default-large + run_privileged systemctl restart frank-default-large + wait_for_health frank-default-large http://127.0.0.1:7770/health 1200 +} + +deploy_llm_small() { + relink_unit frank-default-small + run_privileged systemctl daemon-reload + run_privileged systemctl enable frank-default-small + run_privileged systemctl restart frank-default-small + wait_for_health frank-default-small http://127.0.0.1:7770/health 1200 +} + +deploy_llm_npu() { + relink_unit frank-default-npu + run_privileged systemctl daemon-reload + echo "Relinked frank-default-npu; it was not restarted." +} + +wait_for_health() { + local service="$1" url="$2" limit="$3" + local i + for ((i=1; i<=limit; i++)); do + if curl -fsS "$url" >/dev/null 2>&1; then + echo "$service healthy after approximately $((i * 2))s" + return 0 + fi + sleep 2 + done + echo "FAIL: $service did not become healthy in $((limit * 2))s" >&2 + return 1 +} + +deploy_service() { + local service="$1" + case "$service" in + caddy) deploy_caddy ;; + homepage) deploy_homepage ;; + searxng) deploy_searxng ;; + gluetun) deploy_gluetun ;; + searxng-valkey) deploy_searxng_valkey ;; + open-webui) deploy_open_webui ;; + pihole) deploy_pihole ;; + gitea-postgres) deploy_gitea_unit gitea-postgres gitea-postgres ;; + gitea-server) deploy_gitea_unit gitea-server gitea-server ;; + gitea-caddy) deploy_gitea_unit gitea-caddy gitea-caddy ;; + frank-default-large) deploy_llm_large ;; + frank-default-small) deploy_llm_small ;; + frank-default-npu) deploy_llm_npu ;; + *) deploy_compose_service "$service" ;; + esac +} + +if [[ "$PENDING" -eq 1 ]]; then + add_changed_services "$FROM_REVISION" + add_pending_services +fi + +# Remove duplicate service names while preserving caller/planner order. +UNIQUE_SERVICES=() +declare -A SEEN_SERVICES=() +for service in "${SERVICES[@]}"; do + [[ -n "$service" ]] || continue + if [[ -z "${SEEN_SERVICES[$service]:-}" ]]; then + UNIQUE_SERVICES+=("$service") + SEEN_SERVICES[$service]=1 + fi +done +SERVICES=("${UNIQUE_SERVICES[@]}") + +if [[ "${#SERVICES[@]}" -eq 0 ]]; then + echo "No services selected; nothing to deploy." + exit 0 +fi + +for service in "${SERVICES[@]}"; do + [[ "$service" =~ ^[a-z0-9][a-z0-9-]*$ ]] || { echo "invalid service: $service" >&2; exit 2; } + service_is_allowed "$service" || { echo "service $service is not deployed on $TARGET" >&2; exit 2; } + echo "=== Deploying $service at $REVISION ===" + mark_status "$service" pending + deploy_service "$service" + mark_deployed "$service" + echo " STATE[$service] = $REVISION (status=deployed)" +done diff --git a/scripts/deploy-searxng-quadlet.sh b/scripts/deploy-searxng-quadlet.sh index d63a906..6fa783b 100755 --- a/scripts/deploy-searxng-quadlet.sh +++ b/scripts/deploy-searxng-quadlet.sh @@ -1,97 +1,16 @@ #!/usr/bin/env bash -# deploy-searxng-quadlet.sh — Install and (re)start the SearXNG quadlet stack. -# -# The gluetun + searxng + valkey trio is boot-persistent via Quadlet, not -# podman-compose (see AGENTS.md). This script: -# 1. Pushes main to origin -# 2. Pulls the repo on the target -# 3. Copies the quadlet units into ~/.config/containers/systemd/ -# 4. daemon-reloads, enables the units for boot, and starts them in order -# -# It is idempotent: it stops any running units and removes any containers with -# the trio's names (compose leftovers or the previous quadlet run) before -# reinstalling and starting, so it is safe to run on first migration and on -# every later config change. -# -# Usage: -# ./scripts/deploy-searxng-quadlet.sh [--dry-run] -# ./scripts/deploy-searxng-quadlet.sh dankclaw +#!/usr/bin/env bash +# Compatibility wrapper for the service-scoped deployment dispatcher. +# Use deploy.sh for Git push/pull and planning; this name remains for operators +# and runbooks that historically deployed the Quadlet trio directly. set -euo pipefail -REMOTE_USER="dank" -REMOTE_HOST="dankclaw" -REMOTE_DIR="/home/dank/serverAppConfigs" -QDIR="dankclaw/searxng/quadlet" -UNITS="searxng-network.service gluetun.service searxng-valkey.service searxng.service" - TARGET="${1:-}" -DRY_RUN=0 -if [ "${2:-}" = "--dry-run" ]; then - DRY_RUN=1 -fi - -if [ -z "${TARGET:-}" ]; then - echo "Usage: $0 [--dry-run]" - exit 1 -fi -if [ "${TARGET}" != "dankclaw" ]; then - echo "This stack runs only on dankclaw (got: ${TARGET})" - exit 1 -fi - -# --------------------------------------------------------------------------- -# Local pre-flight -# --------------------------------------------------------------------------- -cd "$(dirname "$0")/.." -if ! git diff --quiet HEAD; then - echo "FAIL: Local working tree has uncommitted changes. Commit first." - exit 1 -fi - -if [ "$DRY_RUN" = "1" ]; then - echo "[dry-run] Would push main, pull on ${REMOTE_HOST}, install quadlet units, and start: ${UNITS}" - exit 0 -fi - -echo "=== Deploying SearXNG quadlet stack to ${REMOTE_HOST} ===" -echo "[1/3] Pushing local main to origin..." -git push origin main - -REMOTE="${REMOTE_USER}@${REMOTE_HOST}" - -echo "[2/3] Installing and starting the quadlet units on ${REMOTE_HOST}..." -ssh "$REMOTE" bash -s </dev/null || true - -echo ' Removing any containers with the trio names (compose leftovers or prior run)...' -podman rm -f searxng searxng-valkey gluetun 2>/dev/null || true - -echo ' Copying quadlet units into ~/.config/containers/systemd/ ...' -mkdir -p ~/.config/containers/systemd -cp ${QDIR}/*.container ${QDIR}/*.network ~/.config/containers/systemd/ - -echo ' daemon-reload (generates the units)...' -# Quadlet units are generator-managed: systemctl enable fails on them ("transient -# or generated"). The [Install] WantedBy=default.target in each .container file is -# itself the enablement (is-enabled reports "generated"), and with user linger on -# they start at boot. The .network unit is pulled in by the containers that set -# Wants=/After=searxng-network.service. So: daemon-reload, then start. -systemctl --user daemon-reload - -echo ' Starting in dependency order...' -systemctl --user start searxng-network.service -systemctl --user start gluetun.service searxng-valkey.service -systemctl --user start searxng.service -REMOTE_EOF - -echo "[3/3] Verifying unit state..." -ssh "$REMOTE" 'systemctl --user --no-pager status searxng-network.service gluetun.service searxng-valkey.service searxng.service 2>&1 | grep -E "●|Active:"' || true - -echo "=== SearXNG quadlet stack deployed to ${REMOTE_HOST} ===" +shift || true +case "$TARGET" in + dankclaw) exec "$(dirname "$0")/deploy.sh" dankclaw --service searxng "$@" ;; + *) + echo "This stack runs only on dankclaw (got: ${TARGET:-})" >&2 + exit 2 + ;; + esac diff --git a/scripts/deploy.sh b/scripts/deploy.sh index d41acd8..8c0bda3 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -1,304 +1,214 @@ #!/usr/bin/env bash -# deploy.sh — Deploy the serverAppConfigs repo to a target and restart its -# podman stack and LLM fleet services. +# deploy.sh — push main, pull one target, and deploy only selected services. # -# Implements the git deployment workflow from AGENTS.md. Assumes local -# changes are committed and pushed, then: -# 1. Pushes main to origin -# 2. Pulls the repo on the target -# 3. Copies shared configs into place and restarts the podman stack -# (dankclaw only; the dankserver media stack runs as bare containers) -# 4. Relinks the LLM systemd units and restarts the active LLM service -# 5. Waits for the Caddy homepage and the LLM health endpoint -# -# Sudo: the fleet hosts allow passwordless sudo for the deploy user. For -# headless runs on a host without that, set FRANK_SUDO_PW so remote sudo -# reads the password from stdin (`sudo -S`). The password is never stored -# in this repo. +# Planning is path-aware and happens before the push. The target-side runner +# then applies only those services, with per-service deployment revisions kept +# outside the Git checkout. A docs-only commit is a no-op. An LLM service is +# restarted only when its own files (or the shared chat template) are selected. # # Usage: -# ./scripts/deploy.sh [--dry-run] +# ./scripts/deploy.sh [--service NAME]... [--dry-run] # -# Targets: -# dankclaw - AI/search/dev/observability stack + frank-default-large -# (CIRU ROCmFPX llama.cpp, Qwen3.8-27B, port 7770) -# dankserver - media/library stack (bare containers) + frank-default-small -# (upstream llama.cpp, Qwen3.6-35B-A3B, port 7770) -# -# Example: -# ./scripts/deploy.sh dankclaw -# ./scripts/deploy.sh dankserver --dry-run +# Without --service, the planner selects services changed since origin/main and +# the remote runner also applies pending service changes from its state store. +# --service intentionally restricts the run to the named services; it is useful +# for a safe, explicit retry after another service failed. set -euo pipefail -# --------------------------------------------------------------------------- -# Target configuration -# --------------------------------------------------------------------------- -declare -A REMOTE_USER REMOTE_HOST REMOTE_DIR COMPOSE HEALTH_URL COMPOSE_UP COPY_SHARED -declare -A LLM_SERVICES LLM_UNITS LLM_HEALTH LLM_WAIT +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "$SCRIPT_DIR/.." && pwd)" +PLANNER="$SCRIPT_DIR/deploy-plan.py" +REMOTE_USER="dank" +REMOTE_DIR="/home/dank/serverAppConfigs" -# Caddy is consolidated on dankclaw: it fronts both hosts' services. The -# dankserver media stack runs as bare containers (not this repo's compose), -# so its stack deploy is pull-only; health is checked against the shared Caddy. -REMOTE_USER[dankclaw]=dank -REMOTE_HOST[dankclaw]=dankclaw -REMOTE_DIR[dankclaw]="/home/dank/serverAppConfigs" -COMPOSE[dankclaw]="dankclaw/compose.yaml" -HEALTH_URL[dankclaw]="http://127.0.0.1/" -COMPOSE_UP[dankclaw]=1 -COPY_SHARED[dankclaw]=1 -# LLM: frank-default-large is the active service (restart + health wait). -# frank-default-npu is relinked only (not enabled on the host); its unit -# picks up the new paths at its next manual start. -LLM_SERVICES[dankclaw]="frank-default-large" -LLM_UNITS[dankclaw]="frank-default-large frank-default-npu" -LLM_HEALTH[dankclaw]="http://127.0.0.1:7770/health" -# CIRU llama.cpp (ROCm 7.15) loads the v3 five-file payload (GGUF + FFN + -# GDN + Output-K8 .pfs + mmproj, ~36.7 GiB) + MTP draft/Output-K8 proxy and -# initializes PromptForge views; allow 20 minutes. -LLM_WAIT[dankclaw]=600 +usage() { + cat <<'EOF' +Usage: deploy.sh [options] -REMOTE_USER[dankserver]=dank -REMOTE_HOST[dankserver]=dankserver -REMOTE_DIR[dankserver]="/home/dank/serverAppConfigs" -COMPOSE[dankserver]="dankserver/compose.yaml" -HEALTH_URL[dankserver]="http://192.168.0.143/" -COMPOSE_UP[dankserver]=0 -COPY_SHARED[dankserver]=0 -# LLM: frank-default-small is the active service (restart + health wait). -LLM_SERVICES[dankserver]="frank-default-small" -LLM_UNITS[dankserver]="frank-default-small" -LLM_HEALTH[dankserver]="http://127.0.0.1:7770/health" -# Cold load: --no-mmap reads the 30.6 GB Q6_K_P through RAM (MoE experts on -# CPU); allow 20 minutes. -LLM_WAIT[dankserver]=600 +Options: + --service NAME deploy only NAME (repeatable; bypasses automatic planning) + --dry-run show the plan without pushing or connecting to a target + -h, --help show this help + +Examples: + ./scripts/deploy.sh dankclaw + ./scripts/deploy.sh dankclaw --service searxng + ./scripts/deploy.sh dankserver --service frank-default-small + ./scripts/deploy.sh dankclaw --dry-run +EOF +} TARGET="${1:-}" +if [[ "$TARGET" == "-h" || "$TARGET" == "--help" ]]; then + usage + exit 0 +fi +if [[ -z "$TARGET" ]]; then + usage >&2 + exit 2 +fi +shift + +case "$TARGET" in + dankclaw) REMOTE_HOST="dankclaw" ;; + dankserver) REMOTE_HOST="dankserver" ;; + *) + echo "Unknown target: $TARGET" >&2 + usage >&2 + exit 2 + ;; +esac + DRY_RUN=0 -if [ "${2:-}" = "--dry-run" ]; then - DRY_RUN=1 -fi - -# --------------------------------------------------------------------------- -# Validation -# --------------------------------------------------------------------------- -if [ -z "${TARGET:-}" ]; then - echo "Usage: $0 [--dry-run]" - exit 1 -fi -if [ -z "${REMOTE_HOST[$TARGET]:-}" ]; then - echo "Unknown target: $TARGET" - exit 1 -fi - -# --------------------------------------------------------------------------- -# Sudo wrapper (headless-friendly) -# --------------------------------------------------------------------------- -SUDO="sudo" -if [ -n "${FRANK_SUDO_PW:-}" ]; then - SUDO="printf '%s\n' '$FRANK_SUDO_PW' | sudo -S" -fi - -# --------------------------------------------------------------------------- -# Local checks (pre-flight) -# --------------------------------------------------------------------------- -cd "$(dirname "$0")/.." -if ! git diff --quiet HEAD; then - echo "FAIL: Local working tree has uncommitted changes." - echo "Commit or stash them before deploying." - exit 1 -fi -if [ "$(git rev-parse --abbrev-ref HEAD)" != "main" ]; then - echo "WARN: Not on 'main'; deploying a non-main branch." -fi - -echo "=== Deploying to ${TARGET} ===" -echo " host: ${REMOTE_HOST[$TARGET]}" -echo " dir: ${REMOTE_DIR[$TARGET]}" -echo " compose: ${COMPOSE[$TARGET]}" -echo " llm svc: ${LLM_SERVICES[$TARGET]}" -echo " llm health: ${LLM_HEALTH[$TARGET]}" -echo " homepage: ${HEALTH_URL[$TARGET]}" -echo "" - -# --------------------------------------------------------------------------- -# Push (unless dry-run) -# --------------------------------------------------------------------------- -if [ "$DRY_RUN" = "1" ]; then - echo "[dry-run] Would push local main to origin" - echo "[dry-run] Would SSH to ${REMOTE_HOST[$TARGET]} and git pull" -else - echo "[1/5] Pushing local main to origin..." - git push origin main -fi - -REMOTE="$( - printf '%s@%s' "${REMOTE_USER[$TARGET]}" "${REMOTE_HOST[$TARGET]}" -)" - -# --------------------------------------------------------------------------- -# Remote: pull + copy shared configs + restart stack + restart LLM service -# --------------------------------------------------------------------------- -REMOTE_SCRIPT=" -set -euo pipefail -cd '${REMOTE_DIR[$TARGET]}' - -echo '[2/5] Pulling repo on ${TARGET}...' -git pull origin main - -if [ -f .env ]; then - echo ' .env present, using it' -else - echo ' WARN: no .env on target; copy the .env.example and fill it in' -fi - -if [ '${COPY_SHARED[$TARGET]}' = '1' ]; then - echo '[3/5] Copying shared configs into place...' - mkdir -p /opt/searxng/config /opt/prometheus/config /opt/loki/config /opt/alloy/config - ${SUDO} cp shared/searxng/settings.yml /opt/searxng/config/settings.yml - ${SUDO} cp shared/searxng/limiter.toml /opt/searxng/config/limiter.toml - ${SUDO} cp shared/prometheus/prometheus.yml /opt/prometheus/config/prometheus.yml - ${SUDO} cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml - ${SUDO} cp shared/alloy/config.alloy /opt/alloy/config/config.alloy - - echo ' Rendering Caddyfile (basicauth hash from .env)...' - # Extract the hash with grep/cut rather than sourcing .env: sourcing a - # bcrypt value in a set -u shell dies on its dollar-sign expansion. - # '|| true' so a missing hash warns below instead of aborting under set -e. - # Only the shared family hash remains: the homepage's per-user hashes were - # retired in favor of Authentik forward_auth (the /admin/ page and the user - # page are now behind the same SSO session). - CADDY_BASICAUTH_HASH=\"\$(grep -E '^CADDY_BASICAUTH_HASH=' .env 2>/dev/null | tail -1 | cut -d= -f2- || true)\" - if [ -n \"\${CADDY_BASICAUTH_HASH}\" ]; then - # '|' delimiter: the bcrypt hash contains '/' characters. - mkdir -p /home/dank/Containers/caddy - sed -e \"s|REPLACE_WITH_HASH|\${CADDY_BASICAUTH_HASH}|g\" \\ - dankclaw/Caddyfile > /home/dank/Containers/caddy/Caddyfile - else - echo ' WARN: CADDY_BASICAUTH_HASH is not set in .env; keeping the placeholder(s)' - fi - - echo ' Deploying the homepage...' - ${SUDO} mkdir -p /srv/homepage - ${SUDO} cp shared/homepage/index.html shared/homepage/admin.html shared/homepage/app.js \\ - shared/homepage/style.css shared/homepage/services.json /srv/homepage/ - if [ -d shared/homepage/fonts ]; then ${SUDO} cp -r shared/homepage/fonts /srv/homepage/; fi - # The last master deployment datetime (Q43): the deployed commit's datetime. - DEPLOYED_AT=\"\$(git log -1 --format=%cI)\" - printf 'window.DEPLOYED_AT = \"%s\";\\n' \"\$DEPLOYED_AT\" > /tmp/deployed.js - ${SUDO} cp /tmp/deployed.js /srv/homepage/deployed.js - ${SUDO} chmod -R a+rX /srv/homepage - # The status checker: install the timer + service, run it once, enable it. - ${SUDO} cp scripts/homepage-status.py /usr/local/bin/homepage-status.py - ${SUDO} cp scripts/homepage-status.service scripts/homepage-status.timer /etc/systemd/system/ - ${SUDO} systemctl daemon-reload - ${SUDO} systemctl enable homepage-status.timer - ${SUDO} systemctl restart homepage-status.timer - ${SUDO} python3 /usr/local/bin/homepage-status.py /srv/homepage/services.json /srv/homepage/status.json -fi - -if [ '${COMPOSE_UP[$TARGET]}' = '1' ]; then - echo '[4/5] Restarting stack...' - # Locally built images: caddy + porkbun module, and loki/alloy + a static - # busybox (their upstream images lack the wget the healthchecks need). - for dir in caddy loki alloy; do - img=franksdank/\$dir - [ \"\$dir\" = caddy ] && img=franksdank/caddy-porkbun - # Rebuild when the Containerfile changed. buildah's layer cache can - # serve a stale COPY --from layer (keyed on stage name), so a rebuild - # always runs with --no-cache. - cfhash=\$(sha256sum dankclaw/\$dir/Containerfile | cut -d' ' -f1) - oldhash=\$(podman image inspect \$img:latest --format '{{index .Config.Labels \"cfhash\"}}' 2>/dev/null || echo none) - if [ \"\$oldhash\" != \"\$cfhash\" ]; then - echo \" \$img image missing or stale; building it...\" - podman build --no-cache --label cfhash=\$cfhash -t \$img:latest -f dankclaw/\$dir/Containerfile . - fi - done - if ! podman-compose -f '${COMPOSE[$TARGET]}' up -d; then - echo ' WARN: compose restart failed; continuing with LLM deployment.' >&2 - fi - - # A Caddyfile render is a bind-mount content change, not a container - # config change: the caddy container often survives 'up -d' and keeps - # serving its in-memory config. Reload explicitly; the file watcher - # (admin on) is the backstop if the API is not up yet. If the reload - # cannot be applied at all (e.g. the running config has the admin API - # disabled -- the fix cannot ride the API it is meant to enable), - # restart the container so it reads the file on start. - caddy_reloaded=0 - for i in 1 2 3 4 5; do - if podman exec caddy caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile >/dev/null 2>&1; then - caddy_reloaded=1 - echo \" Caddy reloaded.\" - break - fi - sleep 3 - done - if [ \"\$caddy_reloaded\" -ne 1 ]; then - echo \" Caddy reload failed; restarting the container to load the file.\" - podman restart caddy || true - fi -else - echo '[4/5] Skipping compose restart (stack runs as bare containers on this host)' -fi - -echo '[4/5] Relinking LLM units and restarting LLM services...' -for unit in ${LLM_UNITS[$TARGET]}; do - # The loop variable must expand on the remote, so it is escaped here and - # the path is unquoted (it has no spaces). Quoting the source path would - # ship a literal loop-variable token and create a dangling symlink - # (the 2026-08-22 relink bug). - ${SUDO} ln -sf ${REMOTE_DIR[$TARGET]}/scripts/\${unit}.service /etc/systemd/system/\${unit}.service -done -${SUDO} systemctl daemon-reload -for svc in ${LLM_SERVICES[$TARGET]}; do - ${SUDO} systemctl enable \"\${svc}\" - ${SUDO} systemctl restart \"\${svc}\" - echo \" restarted \${svc}\" -done -" - -if [ "$DRY_RUN" = "1" ]; then - echo "[dry-run] Remote commands would run:" - echo "$REMOTE_SCRIPT" -else - # shellcheck disable=SC2087 - ssh "$REMOTE" bash -s < /dev/null 2>&1; then - echo " Homepage healthy after approx $((i * 2))s" - break - fi - sleep 2 - done - if ! ssh "$REMOTE" "curl -sf '$HEALTH'" > /dev/null 2>&1; then - echo "FAIL: Homepage did not become healthy in 120s" - exit 1 - fi - - echo "[5/5] Waiting for LLM health ${LLM_HEALTH[$TARGET]} (via ssh)..." - LLM_URL="${LLM_HEALTH[$TARGET]}" - LLM_ITERS="${LLM_WAIT[$TARGET]}" - for i in $(seq 1 "$LLM_ITERS"); do - if ssh "$REMOTE" "curl -sf '$LLM_URL'" > /dev/null 2>&1; then - echo " LLM healthy after approx $((i * 2))s" - echo "" - echo "=== Deploy complete: ${TARGET} ===" +EXPLICIT_SERVICES=() +while [[ "$#" -gt 0 ]]; do + case "$1" in + --dry-run) + DRY_RUN=1 + shift + ;; + --service) + [[ "$#" -ge 2 ]] || { echo "--service requires a name" >&2; exit 2; } + EXPLICIT_SERVICES+=("$2") + shift 2 + ;; + -h|--help) + usage exit 0 - fi - sleep 2 - done - echo "FAIL: LLM service did not become healthy in $((LLM_ITERS * 2))s" + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +cd "$REPO_ROOT" +if [[ ! -x "$PLANNER" ]]; then + echo "FAIL: missing executable planner: $PLANNER" >&2 exit 1 fi + +# A deployment is a release operation, not a way to publish an arbitrary +# worktree. Include untracked files in this guard; the old diff-only check let +# an untracked secret or half-written config pass the pre-flight. +if [[ -n "$(git status --porcelain)" ]]; then + echo "FAIL: Local working tree has uncommitted changes." >&2 + echo "Commit or remove them before deploying." >&2 + exit 1 +fi +if [[ "$(git branch --show-current)" != "main" ]]; then + echo "FAIL: deploy.sh must run from the main branch." >&2 + exit 1 +fi + +HEAD_REVISION="$(git rev-parse HEAD)" +if BASE_REVISION="$(git rev-parse origin/main 2>/dev/null)"; then + : +else + BASE_REVISION="$(git rev-parse HEAD^ 2>/dev/null || git rev-parse HEAD)" +fi + +valid_service() { + local service="$1" + case "$TARGET:$service" in + dankclaw:gluetun|dankclaw:searxng-network|dankclaw:searxng-valkey|dankclaw:searxng|\ + dankclaw:pihole|dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\ + dankclaw:gitea-caddy|dankclaw:comfyui|dankclaw:hermes|dankclaw:hermes-dashboard|\ + dankclaw:prometheus|dankclaw:grafana|dankclaw:loki|dankclaw:alloy|\ + dankclaw:node-exporter|dankclaw:authentik-postgres|dankclaw:authentik-server|\ + dankclaw:authentik-worker|dankclaw:caddy|dankclaw:homepage|\ + dankclaw:frank-default-large|dankclaw:frank-default-npu|\ + dankserver:mariadb|dankserver:redis|dankserver:romm|dankserver:copyparty|\ + dankserver:jellyfin|dankserver:calibre-web|dankserver:prowlarr|\ + dankserver:qbittorrent|dankserver:radarr|dankserver:sonarr|\ + dankserver:lidarr|dankserver:readarr|dankserver:node-exporter|\ + dankserver:frank-default-small) + return 0 + ;; + *) + return 1 + ;; + esac +} + +for service in "${EXPLICIT_SERVICES[@]}"; do + if ! valid_service "$service"; then + echo "Service '$service' is not deployed on $TARGET." >&2 + exit 2 + fi +done + +PLAN=() +if [[ "${#EXPLICIT_SERVICES[@]}" -gt 0 ]]; then + PLAN=("${EXPLICIT_SERVICES[@]}") +else + mapfile -t PLAN < <( + python3 "$PLANNER" --target "$TARGET" --repo "$REPO_ROOT" \ + --base "$BASE_REVISION" --head "$HEAD_REVISION" + ) +fi + +# Preserve planner order while removing duplicate explicit service names. +UNIQUE_PLAN=() +declare -A SEEN=() +for service in "${PLAN[@]}"; do + [[ -n "$service" ]] || continue + if [[ -z "${SEEN[$service]:-}" ]]; then + UNIQUE_PLAN+=("$service") + SEEN[$service]=1 + fi +done +PLAN=("${UNIQUE_PLAN[@]}") + +printf '=== Deployment plan: %s ===\n' "$TARGET" +printf ' checkout: %s\n' "$HEAD_REVISION" +if [[ "${#PLAN[@]}" -eq 0 ]]; then + echo " changed services: (none; the remote will check pending service state)" +else + printf ' changed services:\n' + printf ' - %s\n' "${PLAN[@]}" +fi + +REMOTE="${REMOTE_USER}@${REMOTE_HOST}" +REMOTE_ARGS=("$TARGET" "$HEAD_REVISION") +if [[ "${#EXPLICIT_SERVICES[@]}" -eq 0 ]]; then + REMOTE_ARGS+=(--pending) +fi +REMOTE_ARGS+=("${PLAN[@]}") + +if [[ "$DRY_RUN" -eq 1 ]]; then + echo "[dry-run] Would push main to origin." + echo "[dry-run] Would SSH to $REMOTE, pull with --ff-only, and run the service-scoped runner." + printf '[dry-run] Runner arguments:' + printf ' %q' "${REMOTE_ARGS[@]}" + printf '\n' + exit 0 +fi + +echo "[1/2] Pushing main to origin..." +git push origin main + +echo "[2/2] Pulling and deploying on $REMOTE_HOST..." +# The target runner computes any pending revisions from its own state after +# the pull. This keeps a target that missed an earlier service deployment +# recoverable without restarting services that are already current. +ssh "$REMOTE" bash -s -- "${REMOTE_ARGS[@]}" <<'REMOTE_EOF' +set -euo pipefail +TARGET="$1" +EXPECTED_REVISION="$2" +shift 2 +cd "/home/dank/serverAppConfigs" +OLD_REVISION="$(git rev-parse HEAD)" +echo " pulling origin/main..." +git pull --ff-only origin main +NEW_REVISION="$(git rev-parse HEAD)" +if [[ "$NEW_REVISION" != "$EXPECTED_REVISION" ]]; then + echo "FAIL: target checkout is $NEW_REVISION, expected $EXPECTED_REVISION" >&2 + exit 1 +fi +exec ./scripts/deploy-remote.sh "$TARGET" "$NEW_REVISION" "$@" +REMOTE_EOF + +echo "=== Deploy complete: $TARGET ===" diff --git a/scripts/test-changed.sh b/scripts/test-changed.sh index 062fb96..e416e02 100755 --- a/scripts/test-changed.sh +++ b/scripts/test-changed.sh @@ -82,9 +82,11 @@ for path in "${CHANGED[@]}"; do dankserver/compose.yaml|dankclaw/compose.yaml) add_test tests/compose_validity.bats add_test tests/env_consistency.bats + add_test tests/deployment_scoping.bats ;; dankserver/Caddyfile|dankclaw/Caddyfile) add_test tests/caddyfile.bats + add_test tests/deployment_scoping.bats ;; dankserver/.env.example|dankclaw/.env.example) add_test tests/env_consistency.bats @@ -95,15 +97,23 @@ for path in "${CHANGED[@]}"; do add_test tests/homepage_deploy.bats add_test tests/homepage_pages.bats add_test tests/homepage_checker.bats + add_test tests/deployment_scoping.bats ;; shared/searxng/*) add_test tests/searxng_limiter.bats add_test tests/searxng_quadlet.bats + add_test tests/deployment_scoping.bats ;; dankclaw/*.container|dankclaw/*.network) add_test tests/open_webui_quadlet.bats add_test tests/pihole.bats add_test tests/searxng_quadlet.bats + add_test tests/deployment_scoping.bats + ;; + scripts/deploy.sh|scripts/deploy-remote.sh|scripts/deploy-plan.py|scripts/deploy-*-quadlet.sh) + add_test tests/deployment_flow.bats + add_test tests/deployment_scoping.bats + add_test tests/scripts_syntax.bats ;; scripts/mount-entropy.sh) add_test tests/mount_entropy.bats @@ -121,19 +131,23 @@ for path in "${CHANGED[@]}"; do scripts/*.service) add_test tests/llm_units.bats add_test tests/scripts_syntax.bats + add_test tests/deployment_scoping.bats ;; scripts/*) add_test tests/scripts_syntax.bats ;; docs/*) add_test tests/docs_completeness.bats + add_test tests/deployment_scoping.bats ;; dankclaw/hermes/*) add_test tests/docs_completeness.bats ;; - AGENTS.md) - # The authoring rulebook; validate the docs vault whenever it changes. + README.md|AGENTS.md) + # Project guidance changes can affect the documented deployment + # contract; validate the docs vault and scoped deployment seams. add_test tests/docs_completeness.bats + add_test tests/deployment_scoping.bats ;; *) die "no focused-test mapping for '$path'; add one before committing" diff --git a/tests/deployment_flow.bats b/tests/deployment_flow.bats index a825317..6520908 100644 --- a/tests/deployment_flow.bats +++ b/tests/deployment_flow.bats @@ -1,8 +1,9 @@ #!/usr/bin/env bats # Seam: target reachability and deployment flow integrity. -# These tests assert the deployment contract of the unified repo: -# - a deploy.sh exists and accepts and --dry-run -# - deploy.sh covers both the podman stack and the LLM fleet services +# These tests assert the service-scoped deployment contract: +# - deploy.sh accepts a target, optional service scope, and --dry-run +# - deploy-remote.sh owns per-service handlers and deployment state +# - no host-wide inference restart is hidden in the dispatcher # - every host referenced in the repo is reachable over SSH (dank@host) # - the local git remote is configured before deploy is attempted # Reachability tests are SKIPPED when the target is unreachable so the suite @@ -38,44 +39,30 @@ load helpers # Regression: the Caddyfile render step in deploy.sh must survive a bcrypt # hash (which contains '/' characters). Reproduce the sed expression with a # slash-bearing hash and confirm the placeholder is replaced. -@test "deploy.sh reloads Caddy after rendering (hot reload, not just container start)" { - # The caddy container often survives 'up -d' (a Caddyfile change is a - # bind-mount content change, not a container config change). Without an - # explicit reload, a rendered fix can sit on disk while Caddy serves the - # stale in-memory config. - grep -q 'caddy reload' "$REPO_ROOT/scripts/deploy.sh" +@test "scoped remote runner owns Caddy reload and safe fallback" { + # Caddy is handled only when selected by the planner. A failed reload may + # restart Caddy, but must not restart the whole compose project. + grep -q 'caddy reload' "$REPO_ROOT/scripts/deploy-remote.sh" + grep -q 'podman restart caddy' "$REPO_ROOT/scripts/deploy-remote.sh" + ! grep -qE 'podman-compose.*(up|restart).*\$\(' "$REPO_ROOT/scripts/deploy-remote.sh" } -@test "deploy.sh restarts caddy when the reload cannot be applied" { - # If the running config has the admin API disabled (or the API is down), - # 'caddy reload' can never apply the new file -- the fix cannot be - # delivered over the API it is meant to enable. The fallback is a - # container restart, which always reads the file. - grep -q 'podman restart caddy' "$REPO_ROOT/scripts/deploy.sh" +@test "remote runner has separate handlers for both inference services" { + grep -q 'deploy_llm_large' "$REPO_ROOT/scripts/deploy-remote.sh" + grep -q 'deploy_llm_small' "$REPO_ROOT/scripts/deploy-remote.sh" } -# The unified deploy.sh must manage the LLM fleet services on both hosts: -# frank-default-small (dankserver) and frank-default-large (dankclaw). A -# deploy that only restarts the podman stack leaves the LLM units pointing -# at the retired vllmScripts checkout. -@test "deploy.sh restarts the LLM services on both targets" { - grep -q 'frank-default-small' "$REPO_ROOT/scripts/deploy.sh" - grep -q 'frank-default-large' "$REPO_ROOT/scripts/deploy.sh" +@test "inference restarts are absent from the top-level dispatcher" { + ! grep -qE 'systemctl .*restart (frank-default-small|frank-default-large)' "$REPO_ROOT/scripts/deploy.sh" } -@test "deploy.sh health-checks both LLM endpoints (both on :7770, by host)" { - # 2026-08-22: frank-default-large moved from :52625 to :7770 on - # dankclaw. Both LLM services now listen on :7770, distinguished by - # host (dankserver vs dankclaw). - grep -qE '^LLM_HEALTH\[dankserver\]="http://127\.0\.0\.1:7770/health"' "$REPO_ROOT/scripts/deploy.sh" - grep -qE '^LLM_HEALTH\[dankclaw\]="http://127\.0\.0\.1:7770/health"' "$REPO_ROOT/scripts/deploy.sh" +@test "remote runner health-checks an inference endpoint" { + grep -q '127.0.0.1:7770/health' "$REPO_ROOT/scripts/deploy-remote.sh" } -@test "deploy.sh pulls on the merged checkout on both targets" { - # Both targets run the whole stack (podman + LLM) from one checkout. - local count - count=$(grep -c '/home/dank/serverAppConfigs' "$REPO_ROOT/scripts/deploy.sh") - [ "$count" -ge 2 ] +@test "deploy.sh pulls the merged checkout on the target" { + grep -q 'git pull --ff-only origin main' "$REPO_ROOT/scripts/deploy.sh" + grep -q '/home/dank/serverAppConfigs' "$REPO_ROOT/scripts/deploy.sh" } @test "deploy.sh Caddyfile render survives a slash-bearing bcrypt hash" { diff --git a/tests/deployment_scoping.bats b/tests/deployment_scoping.bats new file mode 100644 index 0000000..f35d086 --- /dev/null +++ b/tests/deployment_scoping.bats @@ -0,0 +1,127 @@ +#!/usr/bin/env bats +# Seam: service-scoped deployment planning. +# A repository commit can contain changes for several workloads while only a +# subset is safe to restart. The planner must keep inference isolated from +# unrelated service changes and treat docs-only changes as a no-op. + +load helpers + +PLANNER="$REPO_ROOT/scripts/deploy-plan.py" +REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" + +@test "deployment planner maps SearXNG changes to SearXNG only" { + run python3 "$PLANNER" --target dankclaw --paths \ + shared/searxng/settings.yml docs/Dankclaw/SearXNG.md + [ "$status" -eq 0 ] + [ "$output" = "searxng" ] +} + +@test "deployment planner maps large-model changes to the large model only" { + run python3 "$PLANNER" --target dankclaw --paths \ + scripts/frank-default-large.env scripts/frank-default-large.service + [ "$status" -eq 0 ] + [ "$output" = "frank-default-large" ] +} + +@test "shared chat-template changes map to the selected host's inference service" { + run python3 "$PLANNER" --target dankclaw --paths scripts/chat-template.jinja + [ "$status" -eq 0 ] + [ "$output" = "frank-default-large" ] + + run python3 "$PLANNER" --target dankserver --paths scripts/chat-template.jinja + [ "$status" -eq 0 ] + [ "$output" = "frank-default-small" ] +} + +@test "deployment planner maps small-model changes to the small model only" { + run python3 "$PLANNER" --target dankserver --paths \ + scripts/frank-default-small.service + [ "$status" -eq 0 ] + [ "$output" = "frank-default-small" ] +} + +@test "documentation-only changes plan no runtime deployment" { + run python3 "$PLANNER" --target dankclaw --paths \ + docs/Architecture/Deployment\ Flow.md docs/Dankclaw/SearXNG.md + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "deployment planner maps shared config to one service" { + run python3 "$PLANNER" --target dankclaw --paths \ + shared/prometheus/prometheus.yml + [ "$status" -eq 0 ] + [ "$output" = "prometheus" ] +} + +@test "deployment planner maps service-owned non-compose files" { + run python3 "$PLANNER" --target dankclaw --paths dankclaw/hermes/config.yaml.example + [ "$status" -eq 0 ] + [ "$output" = "hermes" ] + + run python3 "$PLANNER" --target dankclaw --paths scripts/serve-frank-default-npu.sh scripts/frank-default-npu.env + [ "$status" -eq 0 ] + [ "$output" = "frank-default-npu" ] +} + +@test "homepage is owned by Dankclaw's consolidated proxy" { + run python3 "$PLANNER" --target dankserver --paths shared/homepage/index.html + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "compose planner narrows a compose edit to the changed service" { + local base head + base=$(git -C "$REPO_ROOT" rev-parse HEAD) + head=$(git -C "$REPO_ROOT" rev-parse HEAD) + run python3 "$PLANNER" --target dankclaw --repo "$REPO_ROOT" \ + --base "$base" --head "$head" --paths dankclaw/compose.yaml + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "network-coupled SearXNG config changes include the required peers" { + run python3 "$PLANNER" --target dankclaw --paths \ + dankclaw/searxng/quadlet/gluetun.container + [ "$status" -eq 0 ] + [ "$output" = $'gluetun\nsearxng' ] +} + +@test "remote deployer carries per-service deployment state" { + [ -x "$REMOTE_DEPLOY" ] + grep -q 'deploy-state' "$REMOTE_DEPLOY" + grep -q 'STATE\[' "$REMOTE_DEPLOY" + grep -q 'mark_status.*pending\|status=pending' "$REMOTE_DEPLOY" + grep -q 'mark_status.*deployed\|status=deployed' "$REMOTE_DEPLOY" +} + +@test "remote deployer routes inference restarts through selected handlers" { + # The runner may restart inference only from the service-dispatch path. It + # must not contain a host-wide restart loop. + grep -q 'deploy_llm_large' "$REMOTE_DEPLOY" + grep -q 'deploy_llm_small' "$REMOTE_DEPLOY" + grep -q 'deploy_service' "$REMOTE_DEPLOY" + ! grep -qE 'for .*frank-default-(small|large)' "$REMOTE_DEPLOY" +} + +@test "remote runner rejects services belonging to the other host" { + run env DEPLOY_STATE_DIR="$(mktemp -d)" REMOTE_DIR="$REPO_ROOT" \ + bash "$REMOTE_DEPLOY" dankserver "$(git rev-parse HEAD)" frank-default-large + [ "$status" -eq 2 ] +} + +@test "top-level deployer delegates to the service-scoped remote runner" { + grep -q 'deploy-remote.sh' "$REPO_ROOT/scripts/deploy.sh" + grep -qE 'SERVICE|service' "$REPO_ROOT/scripts/deploy.sh" + grep -q 'OLD_REVISION' "$REPO_ROOT/scripts/deploy.sh" + grep -q 'OLD_REVISION\|old checkout revision' "$REPO_ROOT/scripts/deploy.sh" +} + +@test "legacy service deploy wrappers delegate to the scoped deployer" { + for script in deploy-searxng-quadlet.sh deploy-pihole-quadlet.sh deploy-open-webui-quadlet.sh; do + grep -q 'deploy.sh' "$REPO_ROOT/scripts/$script" || { + echo "$script does not delegate to deploy.sh" + return 1 + } + done +} diff --git a/tests/homepage_deploy.bats b/tests/homepage_deploy.bats index f620b96..dbd8570 100644 --- a/tests/homepage_deploy.bats +++ b/tests/homepage_deploy.bats @@ -10,10 +10,12 @@ load helpers @test "deploy.sh ships the homepage (assets, timer, basicauth hash, deployed.js)" { local d="$REPO_ROOT/scripts/deploy.sh" - grep -q "/srv/homepage" "$d" - grep -q "homepage-status" "$d" - grep -q "CADDY_BASICAUTH_HASH" "$d" - grep -q "deployed.js" "$d" + local r="$REPO_ROOT/scripts/deploy-remote.sh" + grep -q "deploy-remote.sh" "$d" + grep -q "/srv/homepage" "$r" + grep -q "homepage-status" "$r" + grep -q "CADDY_BASICAUTH_HASH" "$r" + grep -q "deployed.js" "$r" } @test "the checker runs as a systemd timer pointing at the checker" { @@ -27,5 +29,5 @@ load helpers } @test "deploy.sh is git-driven: it pulls main on the target" { - grep -q "git pull origin main" "$REPO_ROOT/scripts/deploy.sh" + grep -q "git pull --ff-only origin main" "$REPO_ROOT/scripts/deploy.sh" } diff --git a/tests/searxng_limiter.bats b/tests/searxng_limiter.bats index 0600fd4..c0e31a8 100644 --- a/tests/searxng_limiter.bats +++ b/tests/searxng_limiter.bats @@ -90,8 +90,9 @@ PYEOF } @test "deploy.sh copies limiter.toml into /opt/searxng/config" { - grep -q "shared/searxng/limiter.toml" "$REPO_ROOT/scripts/deploy.sh" \ - || { echo "deploy.sh does not copy shared/searxng/limiter.toml"; return 1; } - grep -q "/opt/searxng/config/limiter.toml" "$REPO_ROOT/scripts/deploy.sh" \ - || { echo "deploy.sh does not place limiter.toml at /opt/searxng/config"; return 1; } + local d="$REPO_ROOT/scripts/deploy-remote.sh" + grep -q "shared/searxng/limiter.toml" "$d" \ + || { echo "deploy-remote.sh does not copy shared/searxng/limiter.toml"; return 1; } + grep -q "/opt/searxng/config/limiter.toml" "$d" \ + || { echo "deploy-remote.sh does not place limiter.toml at /opt/searxng/config"; return 1; } }