From e3cd2aca1b56eac9d21a54a728c3a99bd92ae0f5 Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 18 Aug 2026 13:55:37 -0700 Subject: [PATCH] SearXNG: add limiter.toml trusted group (LAN /24), wire into deploy The trusted group lives in limiter.toml, not settings.yml. Add shared/searxng/limiter.toml trusting 192.168.0.0/24 in pass_ip and trusted_proxies, copy it into /opt/searxng/config on deploy, and correct the SearXNG doc. New test seam: tests/searxng_limiter.bats. --- docs/Dankclaw/SearXNG.md | 13 +++++++++- scripts/deploy.sh | 1 + shared/searxng/limiter.toml | 43 +++++++++++++++++++++++++++++++++ tests/searxng_limiter.bats | 48 +++++++++++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 shared/searxng/limiter.toml create mode 100644 tests/searxng_limiter.bats diff --git a/docs/Dankclaw/SearXNG.md b/docs/Dankclaw/SearXNG.md index b130d1d..9d0cc30 100644 --- a/docs/Dankclaw/SearXNG.md +++ b/docs/Dankclaw/SearXNG.md @@ -64,9 +64,20 @@ outgoing: SearXNG responds on `/healthz` with 200 when ready. +## Trusted Group (limiter.toml) + +The trusted group lives in `shared/searxng/limiter.toml`, not in `settings.yml`. The `settings.yml` file only turns the limiter on. The file deploys to `/opt/searxng/config/limiter.toml`, mounted at `/etc/searxng/limiter.toml`. + +Two lists control access: + +- `pass_ip` — an IP in this list gets unrestricted access and bypasses all bot detection. +- `trusted_proxies` — if the request IP is in this list, SearXNG reads the real client IP from `X-Forwarded-For` (set by Caddy). + +The whole homelab LAN `192.168.0.0/24` is in both lists. This lets local agents and browsers poll the JSON API without tripping the 4-per-hour limit. `deploy.sh` copies the file into place on each deploy. + ## Gotchas -- The JSON API is rate-limited hard by the limiter (4 per 3600s per IP for untrusted). Add the agent's IP to the trusted group in `settings.yml` if it polls. +- The JSON API is rate-limited hard by the limiter (4 per 3600s per IP for untrusted). Add the agent's IP to `pass_ip` in `limiter.toml` if it polls. - `base_url` must match the public address or proxy rewrites break. - Keep the valkey cache container on the same podman network as searxng. diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 45c4550..0b2d500 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -104,6 +104,7 @@ git pull origin main echo '[3/4] Copying shared configs into place...' mkdir -p /opt/searxng/config /opt/prometheus/config /opt/loki/config /opt/alloy/config cp shared/searxng/settings.yml /opt/searxng/config/settings.yml +cp shared/searxng/limiter.toml /opt/searxng/config/limiter.toml cp shared/prometheus/prometheus.yml /opt/prometheus/config/prometheus.yml cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml cp shared/alloy/config.alloy /opt/alloy/config/config.alloy diff --git a/shared/searxng/limiter.toml b/shared/searxng/limiter.toml new file mode 100644 index 0000000..61a37aa --- /dev/null +++ b/shared/searxng/limiter.toml @@ -0,0 +1,43 @@ +# SearXNG limiter / bot-detection config. +# Deployed to /opt/searxng/config/limiter.toml on Dankclaw (mounted at +# /etc/searxng/limiter.toml). Loaded when server.limiter: true in settings.yml. +# +# The "trusted group" lives HERE, not in settings.yml. +# Docs: https://docs.searxng.org/admin/searx.limiter.html + +[botdetection] + +# Number of leading bits compared to group addresses into a (client) network. +ipv4_prefix = 32 +ipv6_prefix = 48 + +# If the request IP is in trusted_proxies, the real client IP is read from the +# X-Forwarded-For / X-Real-IP headers (set by the Caddy reverse proxy). +trusted_proxies = [ + '127.0.0.0/8', + '::1', + '192.168.0.0/24', +] + +[botdetection.ip_limit] + +# Link-local addresses (fe80::/10, 169.254.0.0/16) are not monitored. +filter_link_local = false + +# link_token: per-session URL bypass token. Keep off. +link_token = false + +[botdetection.ip_lists] + +# ip_lists outranks every other method: an IP in pass_ip gets unrestricted +# access and is not checked by user-agent or other bot detection. +# The whole homelab LAN is trusted. +pass_ip = [ + '192.168.0.0/24', +] + +# Blocklist (none). +block_ip = [] + +# Keep the SearXNG org passlist (e.g. check.searx.space) enabled. +pass_searxng_org = true diff --git a/tests/searxng_limiter.bats b/tests/searxng_limiter.bats new file mode 100644 index 0000000..91a963b --- /dev/null +++ b/tests/searxng_limiter.bats @@ -0,0 +1,48 @@ +#!/usr/bin/env bats +# Seam: SearXNG limiter / trusted-group configuration. +# The trusted group lives in shared/searxng/limiter.toml (NOT in settings.yml). +# It must be valid TOML, trust the homelab LAN, and be copied into place by +# deploy.sh so the deployed /etc/searxng/limiter.toml matches the repo. + +load helpers + +@test "shared/searxng/limiter.toml exists" { + [ -f "$REPO_ROOT/shared/searxng/limiter.toml" ] +} + +@test "limiter.toml is valid TOML" { + python3 - "$REPO_ROOT/shared/searxng/limiter.toml" <<'PYEOF' +import sys, tomllib +with open(sys.argv[1], "rb") as fh: + tomllib.load(fh) +PYEOF +} + +@test "limiter.toml trusts the homelab LAN in pass_ip" { + python3 - "$REPO_ROOT/shared/searxng/limiter.toml" <<'PYEOF' +import sys, tomllib +with open(sys.argv[1], "rb") as fh: + data = tomllib.load(fh) +ip_lists = data.get("botdetection", {}).get("ip_lists", {}) +pass_ip = ip_lists.get("pass_ip", []) +assert "192.168.0.0/24" in pass_ip, f"LAN missing from pass_ip; got {pass_ip}" +PYEOF +} + +@test "limiter.toml trusts the LAN as a proxy for X-Forwarded-For" { + python3 - "$REPO_ROOT/shared/searxng/limiter.toml" <<'PYEOF' +import sys, tomllib +with open(sys.argv[1], "rb") as fh: + data = tomllib.load(fh) +bd = data.get("botdetection", {}) +proxies = bd.get("trusted_proxies", []) +assert "192.168.0.0/24" in proxies, f"LAN missing from trusted_proxies; got {proxies}" +PYEOF +} + +@test "deploy.sh copies limiter.toml into /opt/searxng/config" { + grep -q "shared/searxng/limiter.toml" "$REPO_ROOT/scripts/deploy.sh" \ + || { echo "deploy.sh does not copy shared/searxng/limiter.toml"; return 1; } + grep -q "/opt/searxng/config/limiter.toml" "$REPO_ROOT/scripts/deploy.sh" \ + || { echo "deploy.sh does not place limiter.toml at /opt/searxng/config"; return 1; } +}