From ab06e4c2216730be15ccf4da4049da7b855ec26c Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 20 Oct 2026 21:16:23 -0700 Subject: [PATCH] test(e2e): cover the two guard-surface gaps found in the full-suite audit Audit of all 12 e2e specs for the 'green-but-vacuous' class that broke Grafana's SSO. The suite is otherwise sound (incident-driven, leakage-based); two real coverage gaps surfaced: - notebook.franksdank.cloud (open-notebook, deployed 2026-10-19) was a guarded vhost in NEITHER the 13-/14- GUARDED lists NOR any dedicated spec. A broken upstream would stay invisible - the exact 17-hermes class. Added to the 14- guard audit (unauth -> authentik gate, 'Open Notebook' leak marker) plus a new 21- that proves authenticated reachability to open-notebook's own login through the guard chain, with a positive app marker. - prometheus.franksdank.cloud was only ever asserted '<500 and not a login wall' (13-/11-), which a 404 also passes. New 22- adds the app-is-really- there proof 20- gives Grafana: SSO handoff + its own /api/v1/status/buildinfo answers {status:success}. Both specs wired into the standard project (playwright testMatch), which the e2e_scaffold bats gate enforces. Verified live against the fleet; full bats suite green. --- tests/e2e/playwright.config.ts | 8 +++- tests/e2e/specs/14-guard.spec.ts | 3 +- tests/e2e/specs/21-notebook.spec.ts | 52 +++++++++++++++++++++++ tests/e2e/specs/22-prometheus-sso.spec.ts | 33 ++++++++++++++ 4 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/specs/21-notebook.spec.ts create mode 100644 tests/e2e/specs/22-prometheus-sso.spec.ts diff --git a/tests/e2e/playwright.config.ts b/tests/e2e/playwright.config.ts index 611bffa..986b2e8 100644 --- a/tests/e2e/playwright.config.ts +++ b/tests/e2e/playwright.config.ts @@ -31,8 +31,12 @@ export default defineConfig({ // container name, so it needs its own pass condition). 18- joined on // 2026-09-24 (OpenWebUI SSO through authentik). 20- joined on 2026-10-21 // (Grafana's OAuth2 relying-party login, which the edge-only 13- never - // follows). - testMatch: /(10|12|13|16|17|18|20)-.*\.spec\.ts/, + // follows). 21- joined 2026-10-21 (the open-notebook vhost, deployed + // 2026-10-19 - the newest guarded app, in neither GUARDED list nor any + // spec until now: the 17-hermes "uncovered vhost broke silently" class). + // 22- joins Prometheus's app-is-really-there proof (its own API), closing + // the "not a login wall but maybe a 404" hole 20- closed for Grafana. + testMatch: /(10|12|13|16|17|18|20|21|22)-.*\.spec\.ts/, use: { storageState: ".auth/qa-e2e.json" }, dependencies: ["setup"], }, diff --git a/tests/e2e/specs/14-guard.spec.ts b/tests/e2e/specs/14-guard.spec.ts index 344d1af..4856bcf 100644 --- a/tests/e2e/specs/14-guard.spec.ts +++ b/tests/e2e/specs/14-guard.spec.ts @@ -15,7 +15,7 @@ const BASE = "https://www.franksdank.cloud"; // Every vhost that Caddy routes through anubis:8923 (see dankclaw/Caddyfile). // Keep this list in lockstep with the Caddyfile: a new guarded vhost must be // added here or the guard-chain audit is incomplete. -const GUARDED = ["www", "search", "romm", "files", "seerr", "grafana", "prometheus", "loki"]; +const GUARDED = ["www", "search", "romm", "files", "seerr", "grafana", "prometheus", "loki", "notebook"]; // Markers that prove app content leaked past the guard to an anonymous user. const LEAK_MARKERS: Record = { @@ -25,6 +25,7 @@ const LEAK_MARKERS: Record = { "seerr.franksdank.cloud": ["Request Movie"], "grafana.franksdank.cloud": ["Dashboards"], "prometheus.franksdank.cloud": ["Expression Browser", "Graph"], + "notebook.franksdank.cloud": ["Open Notebook"], }; test("homepage root (/) is guarded for unauthenticated users", async ({ page }) => { diff --git a/tests/e2e/specs/21-notebook.spec.ts b/tests/e2e/specs/21-notebook.spec.ts new file mode 100644 index 0000000..68a2999 --- /dev/null +++ b/tests/e2e/specs/21-notebook.spec.ts @@ -0,0 +1,52 @@ +import { expect, test } from "@playwright/test"; + +// open-notebook (notebook.franksdank.cloud), deployed 2026-10-19, is a guarded +// vhost whose outpost (notebook-proxy) fronts an app with its OWN password +// login - the same shape as hermes (17-) and OpenWebUI (18-): the guard chain +// (Anubis PoW -> authentik outpost) must hand off to the app, and the app's own +// login page is the expected landing. Until now this host appeared in neither +// the 13-/14- GUARDED lists nor any dedicated spec, so a broken upstream would +// have stayed invisible - the exact 17-hermes "uncovered vhost" failure class. +// 14- now covers the unauthenticated gate; this spec owns authenticated +// reachability with a POSITIVE app marker, not just "not a login wall". + +const VHOST = "https://notebook.franksdank.cloud/"; +const CHALLENGE = "Making sure you're not a bot"; + +test("notebook vhost reaches the open-notebook app through the guard chain", async ({ page }) => { + test.setTimeout(180_000); + await page.goto(VHOST); + + // The Anubis proof-of-work clears in a real browser and the outpost hands off + // to the app; the user must never be parked on an authentik flow. + await expect + .poll(async () => !(await page.locator("body").innerText()).includes(CHALLENGE), { timeout: 90_000 }) + .toBe(true); + await expect + .poll(() => new URL(page.url()).hostname, { timeout: 60_000 }) + .toBe("notebook.franksdank.cloud"); + expect( + new URL(page.url()).pathname.startsWith("/if/"), + "must not be parked on an authentik flow", + ).toBe(false); + expect( + await page.locator('input[name="uidField"]').count(), + "must not show the authentik login form", + ).toBe(0); + + // Positive identity: the page really is open-notebook's own login, not a + // Caddy error or the IdP shell. The app answering its own login is the pass. + await expect(page, "destination title must be Open Notebook").toHaveTitle(/open notebook/i, { + timeout: 30_000, + }); + await expect(page.locator("body"), "open-notebook login marker").toContainText( + /enter your password/i, + { timeout: 30_000 }, + ); + const body = await page.locator("body").innerText(); + expect(body, "must not leak the IdP flow shell").not.toContain("Powered by authentik"); + expect(body, "must not be a Caddy upstream error").not.toContain("Bad Gateway"); + + const resp = await page.request.get(VHOST); + expect(resp.status(), "open-notebook must answer, not error").toBeLessThan(500); +}); diff --git a/tests/e2e/specs/22-prometheus-sso.spec.ts b/tests/e2e/specs/22-prometheus-sso.spec.ts new file mode 100644 index 0000000..d531016 --- /dev/null +++ b/tests/e2e/specs/22-prometheus-sso.spec.ts @@ -0,0 +1,33 @@ +import { expect, test } from "@playwright/test"; + +// Prometheus (prometheus.franksdank.cloud) sits behind the same embedded outpost +// as Grafana. 13-sso and 11-admin only ever assert it is "not a login wall and +// under 500" - which a Prometheus serving a 404 or a blank page would ALSO pass. +// This is the app-is-really-there proof 20- provides for Grafana: with the +// standard session the outpost hands off silently to the Prometheus UI, and the +// app's own API answers success. A guard-route or upstream regression that lets +// any non-Prometheus answer through fails here. + +test("SSO: Prometheus app answers its own API through authentik", async ({ page, request }) => { + test.setTimeout(180_000); + await page.goto("https://prometheus.franksdank.cloud/"); + + // Challenge clears, outpost hands off silently to the app (no credential prompt). + await expect + .poll( + async () => !(await page.locator("body").innerText()).includes("Making sure you're not a bot"), + { timeout: 90_000 }, + ) + .toBe(true); + await expect + .poll(() => new URL(page.url()).hostname, { timeout: 60_000 }) + .toBe("prometheus.franksdank.cloud"); + expect(await page.locator('input[name="uidField"]').count(), "no authentik form").toBe(0); + await expect(page, "must land on the Prometheus UI").toHaveTitle(/prometheus/i, { timeout: 30_000 }); + + // Load-bearing assertion: the Prometheus API itself answers, authenticated, + // through the outpost. An IdP bounce, a 404, or a raw upstream error fails here. + const api = await request.get("https://prometheus.franksdank.cloud/api/v1/status/buildinfo"); + expect(api.status(), "Prometheus API must answer 200 through the guard").toBe(200); + expect((await api.json()).status, "Prometheus API must report success").toBe("success"); +}); -- 2.54.0