Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Frank
|
ab06e4c221 |
@@ -1,144 +0,0 @@
|
||||
{
|
||||
"uid": "host-system",
|
||||
"title": "Hosts \u2014 Dankclaw & Dankserver",
|
||||
"description": "node_exporter for both servers. Starter set \u2014 throw-at-the-wall breadth; refine the windows and drop the noise. The picker selects the host; CPU/load/memory/filesystem/network are per-node.",
|
||||
"tags": ["host", "system"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "node",
|
||||
"label": "host",
|
||||
"type": "query",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"query": { "query": "label_values(up{job=~\"node-.*\"}, job)", "refId": "A" },
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"current": { "text": "All", "value": "$__all" }
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1, "type": "stat", "title": "Uptime",
|
||||
"description": "Time since boot. A drop means a reboot.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
|
||||
"targets": [{ "expr": "time() - node_boot_time_seconds{job=~\"$node\"}", "legendFormat": "{{job}}", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 2, "type": "stat", "title": "Memory used",
|
||||
"description": "1 - MemAvailable/MemTotal. The honest \"in use\" figure (excludes reclaimable cache).",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.8 }, { "color": "red", "value": 0.92 }] } }, "overrides": [] },
|
||||
"targets": [{ "expr": "1 - node_memory_MemAvailable_bytes{job=~\"$node\"} / node_memory_MemTotal_bytes{job=~\"$node\"}", "legendFormat": "{{job}}", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 3, "type": "stat", "title": "Load 5m / core",
|
||||
"description": "Per-core load5. >1 sustained means runnable tasks are queuing behind cores.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 1 }] } }, "overrides": [] },
|
||||
"targets": [{ "expr": "node_load5{job=~\"$node\"} / count by (job) (node_cpu_seconds_total{job=~\"$node\",mode=\"idle\"})", "legendFormat": "{{job}}", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 4, "type": "stat", "title": "Open file handles",
|
||||
"description": "FDs allocated vs the kernel maximum. Rising toward the max precedes EMFILE across the host.",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit", "max": 1, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 0.9 }] } }, "overrides": [] },
|
||||
"targets": [{ "expr": "node_filefd_allocated{job=~\"$node\"} / node_filefd_maximum{job=~\"$node\"}", "legendFormat": "{{job}}", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 5, "type": "timeseries", "title": "CPU busy %",
|
||||
"description": "CPU seconds/s in each mode = cores busy per mode; the stack sums to total cores in use (16/host here). A user or system spike points at what is loading the box; steal > 0 means the hypervisor is throttling.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "min": 0, "custom": { "fillOpacity": 10, "lineWidth": 1, "stacking": { "mode": "normal" } } }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "expr": "sum by (job,mode) (rate(node_cpu_seconds_total{job=~\"$node\",mode!=\"idle\",mode!=\"iowait\"}[5m]))", "legendFormat": "{{job}} {{mode}}", "refId": "A" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6, "type": "timeseries", "title": "Load average",
|
||||
"description": "load1/5/15 per host.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "expr": "node_load1{job=~\"$node\"}", "legendFormat": "{{job}} load1", "refId": "A" },
|
||||
{ "expr": "node_load5{job=~\"$node\"}", "legendFormat": "{{job}} load5", "refId": "B" },
|
||||
{ "expr": "node_load15{job=~\"$node\"}", "legendFormat": "{{job}} load15", "refId": "C" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7, "type": "timeseries", "title": "Memory breakdown",
|
||||
"description": "Total, free, buffers+cache. Reclaimable cache is not pressure; MemAvailable is the real headroom.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "expr": "node_memory_MemTotal_bytes{job=~\"$node\"} - node_memory_MemAvailable_bytes{job=~\"$node\"}", "legendFormat": "{{job}} used", "refId": "A" },
|
||||
{ "expr": "node_memory_MemAvailable_bytes{job=~\"$node\"}", "legendFormat": "{{job}} available", "refId": "B" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8, "type": "timeseries", "title": "Filesystem free %",
|
||||
"description": "Per mountpoint. The media/library stacks fill disks.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0.15 }] } }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "expr": "node_filesystem_avail_bytes{job=~\"$node\",fstype!~\"tmpfs|overlay|squashfs\"} / node_filesystem_size_bytes{job=~\"$node\",fstype!~\"tmpfs|overlay|squashfs\"}", "legendFormat": "{{job}} {{mountpoint}}", "refId": "A" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9, "type": "timeseries", "title": "Disk throughput",
|
||||
"description": "Read/write bytes/s per device.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "expr": "sum by (job,device) (rate(node_disk_read_bytes_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} R", "refId": "A" },
|
||||
{ "expr": "-sum by (job,device) (rate(node_disk_written_bytes_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} W", "refId": "B" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10, "type": "timeseries", "title": "Disk busy %",
|
||||
"description": "Fraction of time the device had I/O in flight. Near 1 = the device is the bottleneck.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1 }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "expr": "rate(node_disk_io_time_seconds_total{job=~\"$node\"}[5m])", "legendFormat": "{{job}} {{device}}", "refId": "A" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11, "type": "timeseries", "title": "Network throughput",
|
||||
"description": "Receive (up) / transmit (down) bytes/s per interface.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "expr": "sum by (job,device) (rate(node_network_receive_bytes_total{job=~\"$node\",device!~\"lo|veth.*|br.*|docker.*\"}[5m]))", "legendFormat": "{{job}} {{device}} RX", "refId": "A" },
|
||||
{ "expr": "-sum by (job,device) (rate(node_network_transmit_bytes_total{job=~\"$node\",device!~\"lo|veth.*|br.*|docker.*\"}[5m]))", "legendFormat": "{{job}} {{device}} TX", "refId": "B" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12, "type": "timeseries", "title": "Network errors / drops",
|
||||
"description": "Interface errors and drops \u2014 the signature of a saturated link or a bad NIC/cable.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "expr": "sum by (job,device) (rate(node_network_receive_errs_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} RX err", "refId": "A" },
|
||||
{ "expr": "sum by (job,device) (rate(node_network_transmit_errs_total{job=~\"$node\"}[5m]))", "legendFormat": "{{job}} {{device}} TX err", "refId": "B" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
{
|
||||
"uid": "logs-overview",
|
||||
"title": "Logs \u2014 Loki (systemd journal)",
|
||||
"description": "Everything in Loki arrives as the journal (job=systemd-journal); the container is a JSON field parsed at query time, not a stream label. Starter breadth \u2014 the volume/error panels group by the parsed container_name. Loki keeps ~7 days; wide ranges are cheap counts, not the raw log panel.",
|
||||
"tags": ["logs", "loki"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "container",
|
||||
"label": "container",
|
||||
"type": "query",
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"query": { "query": "{job=\"systemd-journal\"} | json", "label": "container_name", "refId": "A" },
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"refresh": 2,
|
||||
"current": { "text": "All", "value": "$__all" }
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1, "type": "timeseries", "title": "Log volume by container (lines/s)",
|
||||
"description": "Parse the journal JSON, count lines per container. Spikes = a chatty or crashing unit.",
|
||||
"gridPos": { "h": 9, "w": 24, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps", "custom": { "fillOpacity": 20, "stacking": { "mode": "normal" } } }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum by (container_name) (count_over_time({job=\"systemd-journal\"} | json [$__auto]))", "legendFormat": "{{container_name}}", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 2, "type": "timeseries", "title": "Total log rate",
|
||||
"description": "All journal lines per second.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} [$__auto]))", "legendFormat": "all lines/s", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 3, "type": "timeseries", "title": "Error-level line rate",
|
||||
"description": "Lines matching error/panic/fatal/fail. A blunt text match \u2014 the journal carries no structured level for all units.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "fixed", "fixedColor": "red" } }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal|fail)\" [$__auto]))", "legendFormat": "error lines/s", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 4, "type": "timeseries", "title": "Errors by container",
|
||||
"description": "Which unit is throwing. Rank by the top offenders.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum by (container_name) (count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" | json [$__auto]))", "legendFormat": "{{container_name}}", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 5, "type": "stat", "title": "Error lines (last 15m)",
|
||||
"description": "Absolute count over the window, not a rate.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 500 }] } }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum(count_over_time({job=\"systemd-journal\"} |~ \"(?i)(error|panic|fatal)\" [15m]))", "instant": true, "legendFormat": "errors/15m", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 6, "type": "logs", "title": "Journal (filtered)",
|
||||
"description": "Raw lines for the selected containers. Set the container picker; keep the window tight.",
|
||||
"gridPos": { "h": 12, "w": 24, "x": 0, "y": 25 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true },
|
||||
"targets": [{ "expr": "{job=\"systemd-journal\"} | json | container_name=~\"$container\"", "refId": "A" }]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
{
|
||||
"uid": "observability",
|
||||
"title": "Observability \u2014 self-monitoring",
|
||||
"description": "The health of the stack that watches the fleet: Prometheus scrape/TSDB, Loki ingest, Grafana. If this is red the dashboards lie. Note: Alertmanager is running but NOT scraped by Prometheus (no job) \u2014 its metrics are absent by design gap, not fault.",
|
||||
"tags": ["observability", "meta"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"panels": [
|
||||
{
|
||||
"id": 1, "type": "stat", "title": "Targets up",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } }, "overrides": [] },
|
||||
"targets": [{ "expr": "count(up == 1)", "instant": true, "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 2, "type": "stat", "title": "Targets down",
|
||||
"description": "Known structural downs: comfyui (no metrics endpoint), gitea (metrics off), and the homepage 'services' probes. Real regressions ride up from here too \u2014 check the table before dismissing.",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] } }, "overrides": [] },
|
||||
"targets": [{ "expr": "count(up == 0)", "instant": true, "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 3, "type": "stat", "title": "TSDB head series",
|
||||
"description": "Active series in memory. A jump = a new exporter or a label-cardinality leak.",
|
||||
"gridPos": { "h": 4, "w": 5, "x": 8, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||
"targets": [{ "expr": "prometheus_tsdb_head_series", "instant": true, "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 4, "type": "stat", "title": "Discovered targets",
|
||||
"gridPos": { "h": 4, "w": 5, "x": 13, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum(prometheus_sd_discovered_targets)", "instant": true, "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 5, "type": "stat", "title": "Prometheus mem",
|
||||
"gridPos": { "h": 4, "w": 3, "x": 18, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] },
|
||||
"targets": [{ "expr": "process_resident_memory_bytes{job=\"prometheus\"}", "instant": true, "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 6, "type": "stat", "title": "Grafana up",
|
||||
"gridPos": { "h": 4, "w": 3, "x": 21, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "mappings": [{ "type": "value", "options": { "1": { "text": "UP", "color": "green" }, "0": { "text": "DOWN", "color": "red" } } }], "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } }, "overrides": [] },
|
||||
"targets": [{ "expr": "up{job=\"grafana\"}", "instant": true, "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 7, "type": "table", "title": "Target health",
|
||||
"description": "Every scrape target and its current up. Sort by Value to surface regressions.",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"options": { "sortBy": [{ "displayName": "Value", "sortOrder": "Asc" }] },
|
||||
"targets": [{ "expr": "up", "format": "table", "instant": true, "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 8, "type": "timeseries", "title": "Scrape interval drift (p99 actual)",
|
||||
"description": "Actual seconds between scrapes (summary quantile). Above the 15s scrape_interval = Prometheus is falling behind its own cadence (slow endpoints or overload).",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
|
||||
"targets": [{ "expr": "prometheus_target_interval_length_seconds{quantile=\"0.99\"}", "legendFormat": "{{instance}}", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 9, "type": "timeseries", "title": "Scrape samples/s",
|
||||
"description": "Total samples ingested across all jobs.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum(rate(scrape_samples_scraped[5m]))", "legendFormat": "samples/s", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 10, "type": "timeseries", "title": "Loki request rate",
|
||||
"description": "Loki served requests/s (push, query). A drop to zero means ingest or query has stopped.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum by (route) (rate(loki_request_duration_seconds_count[5m]))", "legendFormat": "{{route}}", "refId": "A" }]
|
||||
},
|
||||
{
|
||||
"id": 11, "type": "timeseries", "title": "Exporter process CPU",
|
||||
"description": "CPU by monitored process \u2014 spot a runaway exporter or the Prometheus process itself.",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 21 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] },
|
||||
"targets": [{ "expr": "sum by (job) (rate(process_cpu_seconds_total[5m]))", "legendFormat": "{{job}}", "refId": "A" }]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -31,8 +31,12 @@ export default defineConfig({
|
||||
// container name, so it needs its own pass condition). 18- joined on
|
||||
// 2026-09-24 (OpenWebUI SSO through authentik). 20- joined on 2026-10-21
|
||||
// (Grafana's OAuth2 relying-party login, which the edge-only 13- never
|
||||
// follows).
|
||||
testMatch: /(10|12|13|16|17|18|20)-.*\.spec\.ts/,
|
||||
// follows). 21- joined 2026-10-21 (the open-notebook vhost, deployed
|
||||
// 2026-10-19 - the newest guarded app, in neither GUARDED list nor any
|
||||
// spec until now: the 17-hermes "uncovered vhost broke silently" class).
|
||||
// 22- joins Prometheus's app-is-really-there proof (its own API), closing
|
||||
// the "not a login wall but maybe a 404" hole 20- closed for Grafana.
|
||||
testMatch: /(10|12|13|16|17|18|20|21|22)-.*\.spec\.ts/,
|
||||
use: { storageState: ".auth/qa-e2e.json" },
|
||||
dependencies: ["setup"],
|
||||
},
|
||||
|
||||
@@ -15,7 +15,7 @@ const BASE = "https://www.franksdank.cloud";
|
||||
// Every vhost that Caddy routes through anubis:8923 (see dankclaw/Caddyfile).
|
||||
// Keep this list in lockstep with the Caddyfile: a new guarded vhost must be
|
||||
// added here or the guard-chain audit is incomplete.
|
||||
const GUARDED = ["www", "search", "romm", "files", "seerr", "grafana", "prometheus", "loki"];
|
||||
const GUARDED = ["www", "search", "romm", "files", "seerr", "grafana", "prometheus", "loki", "notebook"];
|
||||
|
||||
// Markers that prove app content leaked past the guard to an anonymous user.
|
||||
const LEAK_MARKERS: Record<string, string[]> = {
|
||||
@@ -25,6 +25,7 @@ const LEAK_MARKERS: Record<string, string[]> = {
|
||||
"seerr.franksdank.cloud": ["Request Movie"],
|
||||
"grafana.franksdank.cloud": ["Dashboards"],
|
||||
"prometheus.franksdank.cloud": ["Expression Browser", "Graph"],
|
||||
"notebook.franksdank.cloud": ["Open Notebook"],
|
||||
};
|
||||
|
||||
test("homepage root (/) is guarded for unauthenticated users", async ({ page }) => {
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
|
||||
// open-notebook (notebook.franksdank.cloud), deployed 2026-10-19, is a guarded
|
||||
// vhost whose outpost (notebook-proxy) fronts an app with its OWN password
|
||||
// login - the same shape as hermes (17-) and OpenWebUI (18-): the guard chain
|
||||
// (Anubis PoW -> authentik outpost) must hand off to the app, and the app's own
|
||||
// login page is the expected landing. Until now this host appeared in neither
|
||||
// the 13-/14- GUARDED lists nor any dedicated spec, so a broken upstream would
|
||||
// have stayed invisible - the exact 17-hermes "uncovered vhost" failure class.
|
||||
// 14- now covers the unauthenticated gate; this spec owns authenticated
|
||||
// reachability with a POSITIVE app marker, not just "not a login wall".
|
||||
|
||||
const VHOST = "https://notebook.franksdank.cloud/";
|
||||
const CHALLENGE = "Making sure you're not a bot";
|
||||
|
||||
test("notebook vhost reaches the open-notebook app through the guard chain", async ({ page }) => {
|
||||
test.setTimeout(180_000);
|
||||
await page.goto(VHOST);
|
||||
|
||||
// The Anubis proof-of-work clears in a real browser and the outpost hands off
|
||||
// to the app; the user must never be parked on an authentik flow.
|
||||
await expect
|
||||
.poll(async () => !(await page.locator("body").innerText()).includes(CHALLENGE), { timeout: 90_000 })
|
||||
.toBe(true);
|
||||
await expect
|
||||
.poll(() => new URL(page.url()).hostname, { timeout: 60_000 })
|
||||
.toBe("notebook.franksdank.cloud");
|
||||
expect(
|
||||
new URL(page.url()).pathname.startsWith("/if/"),
|
||||
"must not be parked on an authentik flow",
|
||||
).toBe(false);
|
||||
expect(
|
||||
await page.locator('input[name="uidField"]').count(),
|
||||
"must not show the authentik login form",
|
||||
).toBe(0);
|
||||
|
||||
// Positive identity: the page really is open-notebook's own login, not a
|
||||
// Caddy error or the IdP shell. The app answering its own login is the pass.
|
||||
await expect(page, "destination title must be Open Notebook").toHaveTitle(/open notebook/i, {
|
||||
timeout: 30_000,
|
||||
});
|
||||
await expect(page.locator("body"), "open-notebook login marker").toContainText(
|
||||
/enter your password/i,
|
||||
{ timeout: 30_000 },
|
||||
);
|
||||
const body = await page.locator("body").innerText();
|
||||
expect(body, "must not leak the IdP flow shell").not.toContain("Powered by authentik");
|
||||
expect(body, "must not be a Caddy upstream error").not.toContain("Bad Gateway");
|
||||
|
||||
const resp = await page.request.get(VHOST);
|
||||
expect(resp.status(), "open-notebook must answer, not error").toBeLessThan(500);
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
|
||||
// Prometheus (prometheus.franksdank.cloud) sits behind the same embedded outpost
|
||||
// as Grafana. 13-sso and 11-admin only ever assert it is "not a login wall and
|
||||
// under 500" - which a Prometheus serving a 404 or a blank page would ALSO pass.
|
||||
// This is the app-is-really-there proof 20- provides for Grafana: with the
|
||||
// standard session the outpost hands off silently to the Prometheus UI, and the
|
||||
// app's own API answers success. A guard-route or upstream regression that lets
|
||||
// any non-Prometheus answer through fails here.
|
||||
|
||||
test("SSO: Prometheus app answers its own API through authentik", async ({ page, request }) => {
|
||||
test.setTimeout(180_000);
|
||||
await page.goto("https://prometheus.franksdank.cloud/");
|
||||
|
||||
// Challenge clears, outpost hands off silently to the app (no credential prompt).
|
||||
await expect
|
||||
.poll(
|
||||
async () => !(await page.locator("body").innerText()).includes("Making sure you're not a bot"),
|
||||
{ timeout: 90_000 },
|
||||
)
|
||||
.toBe(true);
|
||||
await expect
|
||||
.poll(() => new URL(page.url()).hostname, { timeout: 60_000 })
|
||||
.toBe("prometheus.franksdank.cloud");
|
||||
expect(await page.locator('input[name="uidField"]').count(), "no authentik form").toBe(0);
|
||||
await expect(page, "must land on the Prometheus UI").toHaveTitle(/prometheus/i, { timeout: 30_000 });
|
||||
|
||||
// Load-bearing assertion: the Prometheus API itself answers, authenticated,
|
||||
// through the outpost. An IdP bounce, a 404, or a raw upstream error fails here.
|
||||
const api = await request.get("https://prometheus.franksdank.cloud/api/v1/status/buildinfo");
|
||||
expect(api.status(), "Prometheus API must answer 200 through the guard").toBe(200);
|
||||
expect((await api.json()).status, "Prometheus API must report success").toBe("success");
|
||||
});
|
||||
Reference in New Issue
Block a user