observability configs: loki delete_request_store, prometheus targets, grafana root URL

- loki: recent releases require compactor.delete_request_store when
  retention is enabled (was crash-looping on start)
- prometheus: scrape targets carried legacy dankclaw.local names with
  URL schemes (invalid); point at published LAN ports and the gitea
  quadlet's published port instead
- grafana: GF_SERVER_ROOT_URL to the canonical HTTPS name
- tests: 3 new seams for the deployed observability configs

dankclaw/compose.yaml

@@ -124,7 +124,7 @@ services:
     container_name: grafana
     restart: unless-stopped
     environment:
-      GF_SERVER_ROOT_URL: "http://grafana.dankclaw.local"
+      GF_SERVER_ROOT_URL: "https://grafana.franksdank.cloud"
       GF_SECURITY_ADMIN_USER: "${GRAFANA_ADMIN_USER}"
       GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_ADMIN_PASSWORD}"
     ports:

shared/loki/loki-config.yaml

@@ -40,6 +40,8 @@ compactor:
   retention_enabled: true
   retention_delete_delay: 2m
   retention_delete_worker_count: 1
+  # Required by recent Loki when retention is enabled.
+  delete_request_store: filesystem
 
 ruler:
   storage:

shared/prometheus/prometheus.yml

@@ -21,10 +21,10 @@ scrape_configs:
     static_configs:
       - targets: ["192.168.0.235:9100"]
 
-  # Gitea metrics
+  # Gitea metrics (quadlet on an isolated network; use the published LAN port)
   - job_name: "gitea"
     static_configs:
-      - targets: ["gitea:3000"]
+      - targets: ["192.168.0.143:3000"]
     metrics_path: "/metrics"
 
   # Grafana metrics
@@ -45,12 +45,11 @@ scrape_configs:
     static_configs:
       - targets: ["comfyui:8188"]
 
-  # Blackbox-style probes for service up/down
+  # Blackbox-style probes for service up/down (published LAN ports;
+  # gitea and comfyui are covered by their own jobs above)
   - job_name: "services"
     metrics_path: "/"
     static_configs:
       - targets:
-          - "http://search.dankclaw.local"
-          - "http://chat.dankclaw.local"
-          - "http://gitea.dankclaw.local"
-          - "http://comfyui.dankclaw.local"
+          - "192.168.0.143:8081"   # SearXNG (via the Gluetun tunnel)
+          - "192.168.0.143:3005"   # OpenWebUI

tests/caddyfile.bats

@@ -100,3 +100,21 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
     grep -q 'xcaddy build' "$cf"
     grep -q 'github.com/caddy-dns/porkbun' "$cf"
 }
+
+# --- Observability configs (deployed by deploy.sh step 3) ---
+
+@test "no legacy .local service names remain in shared configs or compose" {
+    ! grep -rn 'dankclaw\.local\|dankserver\.local' \
+        "$REPO_ROOT/shared" "$REPO_ROOT/dankclaw/compose.yaml"
+}
+
+@test "loki compactor has a delete_request_store (required when retention is on)" {
+    local loki="$REPO_ROOT/shared/loki/loki-config.yaml"
+    if grep -q 'retention_enabled: true' "$loki"; then
+        grep -q 'delete_request_store:' "$loki"
+    fi
+}
+
+@test "prometheus targets are host:port (no URL schemes)" {
+    ! grep -nE '^\s*-\s*"?http' "$REPO_ROOT/shared/prometheus/prometheus.yml"
+}