Files
serverAppConfigs/tests/searxng_limiter.bats
T
Frank 7a8d992fce searxng: wire paid Kagi engine with deploy-time key rendering
The stock SearXNG image ships a native kagi engine; enabling it needs an
API key that must never enter git, and SearXNG does not expand env vars
in settings.yml (the live secret_key literally was '${SEARXNG_SECRET}'
until now — the app tolerated the placeholder string silently).

- settings.yml: kagi engine enabled (weight 2, kg shortcut, paging off so
  exactly one API credit per aggregated query, timeout 8.0 vs 5.0 scrapers
  so a slow paid query is not a wasted credit); api_key stays ${KAGI_API_KEY}.
- new scripts/searxng-render-settings.py: deploy-time renderer, allowlisted
  to KAGI_API_KEY/SEARXNG_SECRET/SEARXNG_BASE_URL, fails the deploy loudly
  on an unresolvable placeholder, idempotent on redeploys.
- deploy-remote.sh copy_searxng_config runs the renderer against
  /opt/searxng/.env after copying settings.yml into /opt/searxng/config.
- KAGI_API_KEY placeholder added to dankclaw/.env.example (real key staged
  into the host-side /opt/searxng/.env, mode 600, out of band).
- bats: engine wiring, renderer substitution, loud-failure, deployer seam.
- docs: SearXNG.md Kagi section + renderer law; Providers.md carve-out on
  the no-credentials policy (paid engine acceptable when bounded + ordered).

Verified live on dankclaw: direct API probe via the gluetun netns (200,
data.search[] shape), then an aggregated query returned 29/29 results
tagged kagi, zero unresponsive engines.
2026-09-26 23:25:55 -07:00

147 lines
6.3 KiB
Bash

#!/usr/bin/env bats
# Seam: SearXNG limiter / trusted-group configuration.
# The trusted group lives in shared/searxng/limiter.toml (NOT in settings.yml).
# It must be valid TOML, trust the homelab LAN, and be copied into place by
# deploy.sh so the deployed /etc/searxng/limiter.toml matches the repo.
load helpers
@test "shared/searxng/limiter.toml exists" {
[ -f "$REPO_ROOT/shared/searxng/limiter.toml" ]
}
@test "limiter.toml is valid TOML" {
python3 - "$REPO_ROOT/shared/searxng/limiter.toml" <<'PYEOF'
import sys, tomllib
with open(sys.argv[1], "rb") as fh:
tomllib.load(fh)
PYEOF
}
@test "limiter.toml trusts the homelab LAN in pass_ip" {
python3 - "$REPO_ROOT/shared/searxng/limiter.toml" <<'PYEOF'
import sys, tomllib
with open(sys.argv[1], "rb") as fh:
data = tomllib.load(fh)
ip_lists = data.get("botdetection", {}).get("ip_lists", {})
pass_ip = ip_lists.get("pass_ip", [])
assert "192.168.0.0/24" in pass_ip, f"LAN missing from pass_ip; got {pass_ip}"
PYEOF
}
@test "limiter.toml trusts the LAN as a proxy for X-Forwarded-For" {
python3 - "$REPO_ROOT/shared/searxng/limiter.toml" <<'PYEOF'
import sys, tomllib
with open(sys.argv[1], "rb") as fh:
data = tomllib.load(fh)
bd = data.get("botdetection", {})
proxies = bd.get("trusted_proxies", [])
assert "192.168.0.0/24" in proxies, f"LAN missing from trusted_proxies; got {proxies}"
PYEOF
}
@test "limiter.toml pass_ip covers local + podman sources (loopback + slirp/bridge)" {
# Rootless podman strips the client IP on published-port hits: the container
# sees a slirp/bridge source (10.88.x, 10.89.x), never the real client. And
# the JSON rate limit (API_MAX=4/hour, hardcoded) 429s any non-passed IP.
# So local subagents must present a pass_ip source. Loopback + the host's
# private podman ranges are trusted; they are reachable only from the host.
python3 - "$REPO_ROOT/shared/searxng/limiter.toml" <<'PYEOF'
import sys, tomllib
with open(sys.argv[1], "rb") as fh:
data = tomllib.load(fh)
pass_ip = data.get("botdetection", {}).get("ip_lists", {}).get("pass_ip", [])
need = ["127.0.0.1/32", "::1", "192.168.0.0/24", "10.88.0.0/16", "10.89.0.0/16"]
missing = [n for n in need if n not in pass_ip]
assert not missing, f"pass_ip missing {missing}; got {pass_ip}"
PYEOF
}
@test "SearXNG outgoing policy allows modest VPN latency without retry amplification" {
python3 - "$REPO_ROOT/shared/searxng/settings.yml" <<'PYEOF'
import sys, yaml
with open(sys.argv[1]) as fh:
data = yaml.safe_load(fh)
outgoing = data.get("outgoing", {})
assert outgoing.get("request_timeout") == 5.0, outgoing
assert outgoing.get("max_request_timeout") == 15.0, outgoing
assert outgoing.get("retries") == 1, outgoing
assert outgoing["request_timeout"] < outgoing["max_request_timeout"], outgoing
PYEOF
}
@test "SearXNG default provider policy favors anonymous healthy engines" {
python3 - "$REPO_ROOT/shared/searxng/settings.yml" <<'PYEOF'
import sys, yaml
with open(sys.argv[1]) as fh:
data = yaml.safe_load(fh)
engines = {entry["name"]: entry for entry in data.get("engines", [])}
assert engines["bing"]["disabled"] is False, engines.get("bing")
assert engines["bing"]["weight"] > 1, engines["bing"]
assert engines["bing"]["paging"] is False, engines["bing"]
assert engines["duckduckgo"]["disabled"] is False, engines["duckduckgo"]
assert engines["duckduckgo"]["paging"] is False, engines["duckduckgo"]
for name in ("google cse", "startpage", "brave"):
assert engines[name]["disabled"] is True, engines.get(name)
assert engines[name].get("inactive", False) is False, engines[name]
for name in ("bing", "duckduckgo", "wikipedia"):
assert engines[name]["timeout"] == 5.0, engines[name]
PYEOF
}
@test "deploy.sh copies limiter.toml into /opt/searxng/config" {
local d="$REPO_ROOT/scripts/deploy-remote.sh"
grep -q "shared/searxng/limiter.toml" "$d" \
|| { echo "deploy-remote.sh does not copy shared/searxng/limiter.toml"; return 1; }
grep -q "/opt/searxng/config/limiter.toml" "$d" \
|| { echo "deploy-remote.sh does not place limiter.toml at /opt/searxng/config"; return 1; }
}
@test "kagi engine is wired enabled with an env-referenced key" {
python3 - "$REPO_ROOT/shared/searxng/settings.yml" <<'PYEOF'
import sys, yaml
with open(sys.argv[1]) as fh:
data = yaml.safe_load(fh)
engines = {entry["name"]: entry for entry in data.get("engines", [])}
assert "kagi" in engines, "kagi engine block missing from settings.yml"
k = engines["kagi"]
assert k["engine"] == "kagi", k
assert k["disabled"] is False, k
assert k["weight"] == 2, k
assert k["paging"] is False, k
assert k["kagi_categ"] == "search", k
# The tracked file must reference the env placeholder, never a literal key.
assert k["api_key"] == "${KAGI_API_KEY}", k
PYEOF
}
@test "searxng render script substitutes allowlisted vars and nothing else" {
local tmp="$BATS_TEST_TMPDIR/render"
mkdir -p "$tmp"
printf 'SEARXNG_SECRET=abc123\nKAGI_API_KEY=k-xyz\nSEARXNG_BASE_URL=http://x/y\nOTHER=must-not-appear\n' > "$tmp/env"
printf 'server:\n secret_key: "${SEARXNG_SECRET}"\n base_url: "${SEARXNG_BASE_URL}"\nengines:\n - name: kagi\n api_key: "${KAGI_API_KEY}"\n' > "$tmp/settings.yml"
python3 "$REPO_ROOT/scripts/searxng-render-settings.py" "$tmp/env" "$tmp/settings.yml"
grep -qF 'secret_key: "abc123"' "$tmp/settings.yml"
grep -qF 'base_url: "http://x/y"' "$tmp/settings.yml"
grep -qF 'api_key: "k-xyz"' "$tmp/settings.yml"
! grep -qF '${' "$tmp/settings.yml"
! grep -qF 'must-not-appear' "$tmp/settings.yml"
}
@test "searxng render script fails loudly on an unresolvable placeholder" {
local tmp="$BATS_TEST_TMPDIR/unresolved"
mkdir -p "$tmp"
printf 'SEARXNG_SECRET=abc123\n' > "$tmp/env"
printf 'engines:\n - name: kagi\n api_key: "${KAGI_API_KEY}"\n' > "$tmp/settings.yml"
run python3 "$REPO_ROOT/scripts/searxng-render-settings.py" "$tmp/env" "$tmp/settings.yml"
[ "$status" -ne 0 ]
[[ "$output" == *"KAGI_API_KEY"* ]]
}
@test "deployer renders the live searxng settings from the env file" {
grep -q "searxng-render-settings.py" "$REPO_ROOT/scripts/deploy-remote.sh" \
|| { echo "deploy-remote.sh must run searxng-render-settings.py"; return 1; }
grep -q "/opt/searxng/.env" "$REPO_ROOT/scripts/deploy-remote.sh" \
|| { echo "deploy-remote.sh must source /opt/searxng/.env for rendering"; return 1; }
}