diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 295d338..5c1a608 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -1,25 +1,102 @@ #!/usr/bin/env bash -# Secret gate: block any commit whose staged diff contains live-secret shapes. -# This repo is pushed to a Gitea remote; anything committed is instantly visible +# Secret gate: block any commit whose staged diff contains live-secret shapes or +# credential assignments. This repo IS the deployment and every workstation +# pulls it, so a committed secret is instantly in every agent's context window # on the fleet. Deliberate override: ALLOW_SECRETS=1 git commit ... +# +# Two rules, deliberately different in scope: +# +# 1. Token SHAPES are scanned across the ENTIRE staged diff, whatever the file +# type. The first version of this gate whitelisted extensions +# (*.md *.yml *.yaml *.json *.toml *.sh *.ts *.js), so a `postgres.env`, a +# `.container` quadlet, an `.ini`, or an extension-less dotfile — exactly +# the shapes the dotfiles tier is made of — was never read at all. +# 2. Credential ASSIGNMENTS (PASSWORD=…, secret_key: …) are scanned only in +# non-prose files. The auto-learned memory tier is prose ABOUT credentials, +# and a gate that fires there gets overridden until it means nothing; the +# 2026-10-20 audit measured 7 such prose hits on the tracked tree. Markdown +# keeps shape detection only, which is what it had before. +# +# Bcrypt hashes and Basic-auth headers joined the shape list on 2026-10-20 +# (issue #40) after a live Caddy basicauth hash plus its plaintext password — +# and the Pi-hole internal password — were found in a tracked memory file. set -uo pipefail -staged=$(git diff --cached --diff-filter=ACM -U0 -- '*.md' '*.yml' '*.yaml' '*.json' '*.toml' '*.sh' '*.ts' '*.js') -[ -z "$staged" ] && exit 0 +shapes='sk-[A-Za-z0-9]{18,}|ghp_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{22,}|glpat-[A-Za-z0-9_-]{20}|xox[baprs]-[A-Za-z0-9-]{10,}|AKIA[A-Z0-9]{16}|ASIA[A-Z0-9]{16}|BEGIN [A-Z ]*PRIVATE KEY|aes_[A-Za-z0-9/+]{40}|eyJ[A-Za-z0-9_-]{20,}\.ey|\$2[aby]\$[0-9]{2}\$[./A-Za-z0-9]{20,}|Authorization:[[:space:]]*[Bb]asic[[:space:]]+[A-Za-z0-9+/=]{24,}' -patterns='sk-[A-Za-z0-9]{18,}|ghp_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{22,}|glpat-[A-Za-z0-9_-]{20}|xox[baprs]-[A-Za-z0-9-]{10,}|AKIA[A-Z0-9]{16}|ASIA[A-Z0-9]{16}|BEGIN [A-Z ]*PRIVATE KEY|aes_[A-Za-z0-9/+]{40}|eyJ[A-Za-z0-9_-]{20,}\.ey' +# `newpass1234`, `secret_key: "rotated-value"` — an assignment with a literal +# value. `$VAR`, `${VAULT}`, and empty values cannot match. +assign='[A-Za-z0-9_]*(PASSW|SECRET|TOKEN|API_?KEY|ACCESS_KEY|PRIVATE_KEY|CREDENTIAL)[A-Za-z0-9_]*[[:space:]]*[:=][[:space:]]*["'"'"']?[A-Za-z0-9/+._@-]{8,}' +placeholders='changeme|change[-_]|placeholder|redacted|example|sample|dummy|nokey|no[-_]key|not[-_]needed|your[-_]|localhost|xxxx|yyyy|\$\{|<[a-z]|\*\*\*|%s|null|false|true' + +# Exemptions: a variable naming a location (`…_FILE=`, `…_PATH=`) or a value +# that is a filesystem path is a pointer, not a credential. Flagging those is +# how a gate gets permanently overridden. +assign_exempt='(_FILE|_PATH|_DIR|_URL|_ENDPOINT|_SOCKET)[[:space:]]*[:=]|[:=][[:space:]]*["'"'"']?(/|\.\./)' +# Fail closed: a gate that cannot read the staged diff must not report success. +if ! files=$(git diff --cached --diff-filter=ACM --name-only 2>&1); then + { + echo "SECRET GATE: cannot read the staged diff — refusing to commit." + printf ' %s\n' "$files" + echo "Override deliberately: ALLOW_SECRETS=1 git commit ..." + } >&2 + [ -n "${ALLOW_SECRETS:-}" ] && exit 0 + exit 1 +fi +# stderr is captured so the failure branch can quote it; on success it must not +# leak a `warning:` line into the file list and have it mistaken for a path. +files=$(printf '%s\n' "$files" | grep -vE '^(warning|fatal|error):' || true) +[ -z "${files//[[:space:]]/}" ] && exit 0 + +report="" +exempt="" +count=0 +while IFS= read -r f; do + [ -z "$f" ] && continue + # Self-exemption, deliberately an exact filename list and not `tests/`: the + # gate's fixtures must contain credential-shaped strings or there is nothing to + # test against, and this hook documents the shapes it looks for. A secret parked + # in any other test file is still caught. + case "$f" in + .githooks/pre-commit|tests/secret-gate.bats) exempt="$exempt $f"; continue ;; + esac + diff=$(git diff --cached -U0 -- "$f") + [ -z "$diff" ] && continue + # Only ADDED lines can leak. Matching removals too blocks the very commit that + # deletes a credential — which is exactly what happened on 2026-10-20, when the + # gate refused the redaction of the password it exists to catch. + added=$(printf '%s\n' "$diff" | grep -E '^\+' | grep -vE '^\+\+\+' | sed 's/^+//' || true) + [ -z "$added" ] && continue + hits=$(printf '%s\n' "$added" | grep -Ei "$shapes" || true) + if [ "${f##*.}" != "md" ]; then + assigns=$(printf '%s\n' "$added" | grep -Ei "$assign" | grep -Ev "$assign_exempt" | grep -Eiv "$placeholders" || true) + hits=$(printf '%s\n%s\n' "$hits" "$assigns" | sed '/^[[:space:]]*$/d') + fi + if [ -n "$hits" ]; then + n=$(printf '%s\n' "$hits" | wc -l) + count=$((count + n)) + report+=" $f"$'\n'"$(printf '%s\n' "$hits" | cut -c1-100 | sed 's/^/ /')" + fi +done <<< "$files" + +# Say so out loud: an exemption that prints nothing is how a carve-out becomes +# invisible, and invisible is the property a secret wants. +if [ -n "$exempt" ]; then + printf 'secret-gate: self-test fixtures exempt:%s\n' "$exempt" >&2 +fi -hits=$(printf '%s\n' "$staged" | grep -Ei "$patterns" || true) if [ -n "${ALLOW_SECRETS:-}" ]; then - [ -n "$hits" ] && printf 'secret-gate: ALLOW_SECRETS set, allowing %s secret-shaped string(s) through.\n' "$(printf '%s\n' "$hits" | wc -l)" + [ -n "$report" ] && printf 'secret-gate: ALLOW_SECRETS set, allowing %s secret-shaped string(s) through.\n' "$count" exit 0 fi -if [ -n "$hits" ]; then +if [ -n "$report" ]; then { - echo "SECRET GATE: staged diff contains $(printf '%s\n' "$hits" | wc -l) secret-shaped string(s):" - printf '%s\n' "$hits" | sed 's/^/ /' + echo "SECRET GATE: staged diff contains $count secret-shaped string(s):" + printf '%s\n' "$report" echo "Fix it, or override deliberately: ALLOW_SECRETS=1 git commit ..." + echo "A credential that reaches a tracked file is already in history on every" + echo "machine that pulls this repo — deleting it later does not un-leak it." } >&2 exit 1 fi diff --git a/.gitignore b/.gitignore index 8f2272f..bc12bf6 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,10 @@ auth.json **/auth.json .env **/.env +*.env +# `*.env` is the rule that matters: `.env` alone does not match `postgres.env` +# or `frank-job-hunt.env`, and those are the files that actually carry +# POSTGRES_PASSWORD / API keys on this fleet (issue #40). trust.json **/trust.json *.pem @@ -74,3 +78,12 @@ wt/ # A session run inside a git worktree mints a bucket named after the worktree # path; the worktree is deleted, the bucket is orphan state, not config. agent/memories/--home-blank-.omp-wt-* + +# --- theme media: content the pipeline consumes, not configuration --------- +# 23 wallpapers, 80 MiB. Tracked until 2026-10-20; every clone of a config repo +# was carrying a library no diff can review. The files stay on disk because +# ~/.local/share/noctalia/wallpapers symlinks into here, and every filename +# carries its wallhaven ID, so a fresh box restores them with +# `curl -L -o https://wallhaven.cc/w/` + deploy.sh. +# Law: omp-docs/tools/dotfiles-deploy.md +dotfiles/noctalia/.local/share/noctalia/wallpapers/ diff --git a/AGENTS.md b/AGENTS.md index faa1f32..2dc1de5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,12 +111,22 @@ Single-context, inside the existing knowledge base: `omp-docs/glossary.md` and ## Secret gate (HARD rule) The pre-commit hook (`.githooks/pre-commit`, enabled via -`git config core.hooksPath .githooks`) scans every staged diff for live-secret -shapes (`sk-…`, `ghp_…`, `github_pat_…`, `glpat-…`, `xox…`, `AKIA…`, PEM headers, -JWTs) and blocks the commit. Quarantine list — never tracked, ever: `auth.json`, -`.env`, `trust.json`, `*.pem`, `*.key`, `identity*`. Deliberate override: +`git config core.hooksPath .githooks`) blocks the commit in two ways. Token +**shapes** (`sk-…`, `ghp_…`, `github_pat_…`, `glpat-…`, `xox…`, `AKIA…`, PEM +headers, JWTs, bcrypt hashes, `Authorization: Basic` headers) are scanned across +the **entire** staged diff — every file type, because the first version of the +gate whitelisted extensions and never read a `.env`, `.container`, `.ini`, or +extension-less dotfile at all. Credential **assignments** (`PASSWORD=…`, +`secret_key: …`) are scanned in non-prose files only; markdown keeps shape +detection, because the auto-learned memory tier is prose *about* credentials and a +gate that fires there gets overridden until it means nothing. The gate **fails +closed**: if it cannot read the staged diff it refuses the commit rather than +passing it. Quarantine list — never tracked, ever: `auth.json`, `.env` and +`*.env` (`postgres.env`, `frank-job-hunt.env` are the ones that carry real +values), `trust.json`, `*.pem`, `*.key`, `identity*`. Deliberate override: `ALLOW_SECRETS=1 git commit …`. Remember why: a secret in a tracked file is a -secret in every agent's context window on every machine that pulls this repo. +secret in every agent's context window on every machine that pulls this repo, and +redacting it later does not un-leak the history. ## Machine facts (verify before trusting) diff --git a/tests/secret-gate.bats b/tests/secret-gate.bats new file mode 100755 index 0000000..c354765 --- /dev/null +++ b/tests/secret-gate.bats @@ -0,0 +1,159 @@ +#!/usr/bin/env bats +# tests/secret-gate.bats — black-box tests for .githooks/pre-commit +# +# Contract: a commit whose staged content contains a credential is blocked +# (non-zero exit, stderr names the file); a commit whose content is benign +# configuration or prose about credentials is allowed. The gate is copied into a +# scratch repo so git itself invokes it — no sourcing, no internal state. + +setup() { + HOOK="$BATS_TEST_DIRNAME/../.githooks/pre-commit" + REPO="$(mktemp -d)/g" + mkdir -p "$REPO/.git/hooks" + git init -q "$REPO" + git -C "$REPO" config user.email gate@test + git -C "$REPO" config user.name gate + cp "$HOOK" "$REPO/.git/hooks/pre-commit" + chmod +x "$REPO/.git/hooks/pre-commit" +} + +teardown() { rm -rf "$(dirname "$REPO")"; } + +# try_commit -> sets status/output from the real commit attempt +try_commit() { + local file="$1" content="$2" + mkdir -p "$REPO/$(dirname "$file")" + printf '%s\n' "$content" > "$REPO/$file" + run git -C "$REPO" add -A + run git -C "$REPO" commit -q -m "attempt: $file" +} + +@test "blocks a low-entropy password in a .env file" { + try_commit "postgres.env" 'POSTGRES_PASSWORD=hunter2corp2026' + [ "$status" -ne 0 ] + [[ "$output" == *"SECRET GATE"* ]] + [[ "$output" == *"postgres.env"* ]] +} + +@test "blocks a credential in a file type the extension whitelist used to skip" { + try_commit "app.conf" '[app] +secret_key = "rotatedvalue99"' + [ "$status" -ne 0 ] + [[ "$output" == *"app.conf"* ]] +} + +@test "blocks a token shape in a plain text file" { + try_commit "notes.txt" "backup remote: ghp_$(printf 'a%.0s' {1..36})" + [ "$status" -ne 0 ] + [[ "$output" == *"notes.txt"* ]] +} + +@test "blocks a bcrypt hash" { + try_commit "dump.txt" 'basicauth: $2b$14$'"$(printf 'a%.0s' {1..53})" + [ "$status" -ne 0 ] + [[ "$output" == *"dump.txt"* ]] +} + +@test "blocks a credential in a quadlet unit" { + try_commit "db.container" '[Container] +Environment=POSTGRES_PASSWORD=devpostgres2026' + [ "$status" -ne 0 ] + [[ "$output" == *"db.container"* ]] +} + +@test "prose about credentials in markdown is still allowed" { + try_commit "docs/runbook.md" 'The GITEA_TOKEN moved out of .zshrc into secrets.env. +Set password: user.set_password(value) and api_key = "sk-no-key-required".' + [ "$status" -eq 0 ] +} + +@test "placeholder and indirected values in config files are allowed" { + try_commit "app.conf" '[db] +PASSWORD = CHANGEME +SECRET_KEY = ${VAULT_SECRET} +api_key = "sk-no-key-required" +token_file = ' + [ "$status" -eq 0 ] +} + +@test "environment variable indirection is not a secret" { + try_commit "start.sh" '#!/bin/sh +export GITEA_TOKEN="$GITEA_TOKEN" +export API_KEY_FILE=/run/secrets/api_key' + [ "$status" -eq 0 ] +} + +@test "binary content never trips the gate" { + head -c 4096 /dev/urandom > "$REPO/wall.jpg" + run git -C "$REPO" add -A + run git -C "$REPO" commit -q -m "binary" + [ "$status" -eq 0 ] +} + +@test "ALLOW_SECRETS still overrides, and says so" { + try_commit "postgres.env" 'POSTGRES_PASSWORD=hunter2corp2026' + [ "$status" -ne 0 ] + mkdir -p "$REPO/.git/hooks" + run env ALLOW_SECRETS=1 git -C "$REPO" commit -q -m "override" + [ "$status" -eq 0 ] + [[ "$output" == *"ALLOW_SECRETS set"* ]] +} + +@test "this repo's ignore policy quarantines env files at any name" { + repo="$BATS_TEST_DIRNAME/.." + run git -C "$repo" check-ignore -q --no-index dotfiles/db/postgres.env + [ "$status" -eq 0 ] + run git -C "$repo" check-ignore -q --no-index dotfiles/app/frank-job-hunt.env + [ "$status" -eq 0 ] + run git -C "$repo" check-ignore -q --no-index dotfiles/bat/config + [ "$status" -ne 0 ] +} + +@test "the gate refuses to pass when it cannot read the staged diff" { + # A gate that swallows a git error reports success on every commit. This is not + # hypothetical: the 2026-10-20 rewrite passed `-I` to git diff, which is not a + # valid flag there, and the empty file list made every secret commit sail through. + nodir="$(mktemp -d)" + run bash -c "cd '$nodir' && '$HOOK'" + rm -rf "$nodir" + [ "$status" -ne 0 ] + [[ "$output" == *"cannot read the staged diff"* ]] +} + +@test "deleting a credential is never blocked" { + # The commit that removes a secret must always be possible, or the gate holds a + # leak hostage. It did exactly that on 2026-10-20: matching removed lines made + # the redaction commit look like the leak. Seed with the documented override, + # then remove with an ordinary commit. + printf 'POSTGRES_PASSWORD=hunter2corp2026\n' > "$REPO/db.conf" + run git -C "$REPO" add -A + run env ALLOW_SECRETS=1 git -C "$REPO" commit -q -m "seed a leak" + [ "$status" -eq 0 ] + printf '[db]\nPASSWORD = ${VAULT_SECRET}\n' > "$REPO/db.conf" + run git -C "$REPO" add -A + run git -C "$REPO" commit -q -m "redact the leaked password" + [ "$status" -eq 0 ] +} + +@test "the gate can commit itself" { + # Fixtures have to look like credentials, so a gate that scans its own test file + # without exemption can never be committed — it holds its own maintenance + # hostage. The exemption is two exact paths, and it must announce itself. + mkdir -p "$REPO/.githooks" "$REPO/tests" + cp "$HOOK" "$REPO/.githooks/pre-commit" + printf 'POSTGRES_PASSWORD=hunter2corp2026\n' > "$REPO/tests/secret-gate.bats" + run git -C "$REPO" add -A + run git -C "$REPO" commit -q -m "gate and its fixtures" + [ "$status" -eq 0 ] + [[ "$output" == *"self-test fixtures exempt"* ]] +} + +@test "a credential hidden in some other test file is still caught" { + # The exemption is a filename list, not the tests/ directory. + mkdir -p "$REPO/tests" + printf 'POSTGRES_PASSWORD=hunter2corp2026\n' > "$REPO/tests/other.bats" + run git -C "$REPO" add -A + run git -C "$REPO" commit -q -m "smuggle" + [ "$status" -ne 0 ] + [[ "$output" == *"tests/other.bats"* ]] +}