Compare commits

...
Author SHA1 Message Date
Frank ab06e4c221 test(e2e): cover the two guard-surface gaps found in the full-suite audit
Audit of all 12 e2e specs for the 'green-but-vacuous' class that broke
Grafana's SSO. The suite is otherwise sound (incident-driven, leakage-based);
two real coverage gaps surfaced:

- notebook.franksdank.cloud (open-notebook, deployed 2026-10-19) was a guarded
  vhost in NEITHER the 13-/14- GUARDED lists NOR any dedicated spec. A broken
  upstream would stay invisible - the exact 17-hermes class. Added to the 14-
  guard audit (unauth -> authentik gate, 'Open Notebook' leak marker) plus a
  new 21- that proves authenticated reachability to open-notebook's own login
  through the guard chain, with a positive app marker.
- prometheus.franksdank.cloud was only ever asserted '<500 and not a login
  wall' (13-/11-), which a 404 also passes. New 22- adds the app-is-really-
  there proof 20- gives Grafana: SSO handoff + its own /api/v1/status/buildinfo
  answers {status:success}.

Both specs wired into the standard project (playwright testMatch), which the
e2e_scaffold bats gate enforces. Verified live against the fleet; full bats
suite green.
2026-10-20 21:16:23 -07:00
dank 7d85fed1c9 Merge pull request 'Fix Grafana Authentik SSO (500 + missing email claim) + RP landing test' (#54) from fix/grafana-authentik-sso into main 2026-10-20 21:04:02 -07:00
dank 980491668c Merge pull request 'Alloy ships logs: root host-network service so it can read the journal' (#53) from feat/alloy-root-journal into main 2026-10-20 20:05:00 -07:00
dank 2fe68e69af Merge pull request 'Observability: vllm rule group for frank-archon + Loki datasource; correct stale log-pipeline facts' (#52) from fix/observability-vllm-loki into main 2026-10-20 19:39:38 -07:00
4 changed files with 93 additions and 3 deletions
+6 -2
View File
@@ -31,8 +31,12 @@ export default defineConfig({
// container name, so it needs its own pass condition). 18- joined on
// 2026-09-24 (OpenWebUI SSO through authentik). 20- joined on 2026-10-21
// (Grafana's OAuth2 relying-party login, which the edge-only 13- never
// follows).
testMatch: /(10|12|13|16|17|18|20)-.*\.spec\.ts/,
// follows). 21- joined 2026-10-21 (the open-notebook vhost, deployed
// 2026-10-19 - the newest guarded app, in neither GUARDED list nor any
// spec until now: the 17-hermes "uncovered vhost broke silently" class).
// 22- joins Prometheus's app-is-really-there proof (its own API), closing
// the "not a login wall but maybe a 404" hole 20- closed for Grafana.
testMatch: /(10|12|13|16|17|18|20|21|22)-.*\.spec\.ts/,
use: { storageState: ".auth/qa-e2e.json" },
dependencies: ["setup"],
},
+2 -1
View File
@@ -15,7 +15,7 @@ const BASE = "https://www.franksdank.cloud";
// Every vhost that Caddy routes through anubis:8923 (see dankclaw/Caddyfile).
// Keep this list in lockstep with the Caddyfile: a new guarded vhost must be
// added here or the guard-chain audit is incomplete.
const GUARDED = ["www", "search", "romm", "files", "seerr", "grafana", "prometheus", "loki"];
const GUARDED = ["www", "search", "romm", "files", "seerr", "grafana", "prometheus", "loki", "notebook"];
// Markers that prove app content leaked past the guard to an anonymous user.
const LEAK_MARKERS: Record<string, string[]> = {
@@ -25,6 +25,7 @@ const LEAK_MARKERS: Record<string, string[]> = {
"seerr.franksdank.cloud": ["Request Movie"],
"grafana.franksdank.cloud": ["Dashboards"],
"prometheus.franksdank.cloud": ["Expression Browser", "Graph"],
"notebook.franksdank.cloud": ["Open Notebook"],
};
test("homepage root (/) is guarded for unauthenticated users", async ({ page }) => {
+52
View File
@@ -0,0 +1,52 @@
import { expect, test } from "@playwright/test";
// open-notebook (notebook.franksdank.cloud), deployed 2026-10-19, is a guarded
// vhost whose outpost (notebook-proxy) fronts an app with its OWN password
// login - the same shape as hermes (17-) and OpenWebUI (18-): the guard chain
// (Anubis PoW -> authentik outpost) must hand off to the app, and the app's own
// login page is the expected landing. Until now this host appeared in neither
// the 13-/14- GUARDED lists nor any dedicated spec, so a broken upstream would
// have stayed invisible - the exact 17-hermes "uncovered vhost" failure class.
// 14- now covers the unauthenticated gate; this spec owns authenticated
// reachability with a POSITIVE app marker, not just "not a login wall".
const VHOST = "https://notebook.franksdank.cloud/";
const CHALLENGE = "Making sure you're not a bot";
test("notebook vhost reaches the open-notebook app through the guard chain", async ({ page }) => {
test.setTimeout(180_000);
await page.goto(VHOST);
// The Anubis proof-of-work clears in a real browser and the outpost hands off
// to the app; the user must never be parked on an authentik flow.
await expect
.poll(async () => !(await page.locator("body").innerText()).includes(CHALLENGE), { timeout: 90_000 })
.toBe(true);
await expect
.poll(() => new URL(page.url()).hostname, { timeout: 60_000 })
.toBe("notebook.franksdank.cloud");
expect(
new URL(page.url()).pathname.startsWith("/if/"),
"must not be parked on an authentik flow",
).toBe(false);
expect(
await page.locator('input[name="uidField"]').count(),
"must not show the authentik login form",
).toBe(0);
// Positive identity: the page really is open-notebook's own login, not a
// Caddy error or the IdP shell. The app answering its own login is the pass.
await expect(page, "destination title must be Open Notebook").toHaveTitle(/open notebook/i, {
timeout: 30_000,
});
await expect(page.locator("body"), "open-notebook login marker").toContainText(
/enter your password/i,
{ timeout: 30_000 },
);
const body = await page.locator("body").innerText();
expect(body, "must not leak the IdP flow shell").not.toContain("Powered by authentik");
expect(body, "must not be a Caddy upstream error").not.toContain("Bad Gateway");
const resp = await page.request.get(VHOST);
expect(resp.status(), "open-notebook must answer, not error").toBeLessThan(500);
});
+33
View File
@@ -0,0 +1,33 @@
import { expect, test } from "@playwright/test";
// Prometheus (prometheus.franksdank.cloud) sits behind the same embedded outpost
// as Grafana. 13-sso and 11-admin only ever assert it is "not a login wall and
// under 500" - which a Prometheus serving a 404 or a blank page would ALSO pass.
// This is the app-is-really-there proof 20- provides for Grafana: with the
// standard session the outpost hands off silently to the Prometheus UI, and the
// app's own API answers success. A guard-route or upstream regression that lets
// any non-Prometheus answer through fails here.
test("SSO: Prometheus app answers its own API through authentik", async ({ page, request }) => {
test.setTimeout(180_000);
await page.goto("https://prometheus.franksdank.cloud/");
// Challenge clears, outpost hands off silently to the app (no credential prompt).
await expect
.poll(
async () => !(await page.locator("body").innerText()).includes("Making sure you're not a bot"),
{ timeout: 90_000 },
)
.toBe(true);
await expect
.poll(() => new URL(page.url()).hostname, { timeout: 60_000 })
.toBe("prometheus.franksdank.cloud");
expect(await page.locator('input[name="uidField"]').count(), "no authentik form").toBe(0);
await expect(page, "must land on the Prometheus UI").toHaveTitle(/prometheus/i, { timeout: 30_000 });
// Load-bearing assertion: the Prometheus API itself answers, authenticated,
// through the outpost. An IdP bounce, a 404, or a raw upstream error fails here.
const api = await request.get("https://prometheus.franksdank.cloud/api/v1/status/buildinfo");
expect(api.status(), "Prometheus API must answer 200 through the guard").toBe(200);
expect((await api.json()).status, "Prometheus API must report success").toBe("success");
});