From 37f05a5f9ea9ce45d007358578c463fc4f546f9e Mon Sep 17 00:00:00 2001
From: Frank <frank@dankware>
Date: Sat, 22 Aug 2026 03:04:36 -0700
Subject: [PATCH] observability configs: loki delete_request_store, prometheus
 targets, grafana root URL

- loki: recent releases require compactor.delete_request_store when
  retention is enabled (was crash-looping on start)
- prometheus: scrape targets carried legacy dankclaw.local names with
  URL schemes (invalid); point at published LAN ports and the gitea
  quadlet's published port instead
- grafana: GF_SERVER_ROOT_URL to the canonical HTTPS name
- tests: 3 new seams for the deployed observability configs
---
 dankclaw/compose.yaml            |  2 +-
 shared/loki/loki-config.yaml     |  2 ++
 shared/prometheus/prometheus.yml | 13 ++++++-------
 tests/caddyfile.bats             | 18 ++++++++++++++++++
 4 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/dankclaw/compose.yaml b/dankclaw/compose.yaml
index 2e80e55..985671c 100644
--- a/dankclaw/compose.yaml
+++ b/dankclaw/compose.yaml
@@ -124,7 +124,7 @@ services:
     container_name: grafana
     restart: unless-stopped
     environment:
-      GF_SERVER_ROOT_URL: "http://grafana.dankclaw.local"
+      GF_SERVER_ROOT_URL: "https://grafana.franksdank.cloud"
       GF_SECURITY_ADMIN_USER: "${GRAFANA_ADMIN_USER}"
       GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_ADMIN_PASSWORD}"
     ports:
diff --git a/shared/loki/loki-config.yaml b/shared/loki/loki-config.yaml
index ceb00ea..6274898 100644
--- a/shared/loki/loki-config.yaml
+++ b/shared/loki/loki-config.yaml
@@ -40,6 +40,8 @@ compactor:
   retention_enabled: true
   retention_delete_delay: 2m
   retention_delete_worker_count: 1
+  # Required by recent Loki when retention is enabled.
+  delete_request_store: filesystem
 
 ruler:
   storage:
diff --git a/shared/prometheus/prometheus.yml b/shared/prometheus/prometheus.yml
index 7e9f3e5..200269d 100644
--- a/shared/prometheus/prometheus.yml
+++ b/shared/prometheus/prometheus.yml
@@ -21,10 +21,10 @@ scrape_configs:
     static_configs:
       - targets: ["192.168.0.235:9100"]
 
-  # Gitea metrics
+  # Gitea metrics (quadlet on an isolated network; use the published LAN port)
   - job_name: "gitea"
     static_configs:
-      - targets: ["gitea:3000"]
+      - targets: ["192.168.0.143:3000"]
     metrics_path: "/metrics"
 
   # Grafana metrics
@@ -45,12 +45,11 @@ scrape_configs:
     static_configs:
       - targets: ["comfyui:8188"]
 
-  # Blackbox-style probes for service up/down
+  # Blackbox-style probes for service up/down (published LAN ports;
+  # gitea and comfyui are covered by their own jobs above)
   - job_name: "services"
     metrics_path: "/"
     static_configs:
       - targets:
-          - "http://search.dankclaw.local"
-          - "http://chat.dankclaw.local"
-          - "http://gitea.dankclaw.local"
-          - "http://comfyui.dankclaw.local"
+          - "192.168.0.143:8081"   # SearXNG (via the Gluetun tunnel)
+          - "192.168.0.143:3005"   # OpenWebUI
diff --git a/tests/caddyfile.bats b/tests/caddyfile.bats
index 72443bb..e6bc0ec 100644
--- a/tests/caddyfile.bats
+++ b/tests/caddyfile.bats
@@ -100,3 +100,21 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
     grep -q 'xcaddy build' "$cf"
     grep -q 'github.com/caddy-dns/porkbun' "$cf"
 }
+
+# --- Observability configs (deployed by deploy.sh step 3) ---
+
+@test "no legacy .local service names remain in shared configs or compose" {
+    ! grep -rn 'dankclaw\.local\|dankserver\.local' \
+        "$REPO_ROOT/shared" "$REPO_ROOT/dankclaw/compose.yaml"
+}
+
+@test "loki compactor has a delete_request_store (required when retention is on)" {
+    local loki="$REPO_ROOT/shared/loki/loki-config.yaml"
+    if grep -q 'retention_enabled: true' "$loki"; then
+        grep -q 'delete_request_store:' "$loki"
+    fi
+}
+
+@test "prometheus targets are host:port (no URL schemes)" {
+    ! grep -nE '^\s*-\s*"?http' "$REPO_ROOT/shared/prometheus/prometheus.yml"
+}