The stock SearXNG image ships a native kagi engine; enabling it needs an
API key that must never enter git, and SearXNG does not expand env vars
in settings.yml (the live secret_key literally was '${SEARXNG_SECRET}'
until now — the app tolerated the placeholder string silently).
- settings.yml: kagi engine enabled (weight 2, kg shortcut, paging off so
exactly one API credit per aggregated query, timeout 8.0 vs 5.0 scrapers
so a slow paid query is not a wasted credit); api_key stays ${KAGI_API_KEY}.
- new scripts/searxng-render-settings.py: deploy-time renderer, allowlisted
to KAGI_API_KEY/SEARXNG_SECRET/SEARXNG_BASE_URL, fails the deploy loudly
on an unresolvable placeholder, idempotent on redeploys.
- deploy-remote.sh copy_searxng_config runs the renderer against
/opt/searxng/.env after copying settings.yml into /opt/searxng/config.
- KAGI_API_KEY placeholder added to dankclaw/.env.example (real key staged
into the host-side /opt/searxng/.env, mode 600, out of band).
- bats: engine wiring, renderer substitution, loud-failure, deployer seam.
- docs: SearXNG.md Kagi section + renderer law; Providers.md carve-out on
the no-credentials policy (paid engine acceptable when bounded + ordered).
Verified live on dankclaw: direct API probe via the gluetun netns (200,
data.search[] shape), then an aggregated query returned 29/29 results
tagged kagi, zero unresponsive engines.
147 lines
6.3 KiB
Bash
147 lines
6.3 KiB
Bash
#!/usr/bin/env bats
|
|
# Seam: SearXNG limiter / trusted-group configuration.
|
|
# The trusted group lives in shared/searxng/limiter.toml (NOT in settings.yml).
|
|
# It must be valid TOML, trust the homelab LAN, and be copied into place by
|
|
# deploy.sh so the deployed /etc/searxng/limiter.toml matches the repo.
|
|
|
|
load helpers
|
|
|
|
@test "shared/searxng/limiter.toml exists" {
|
|
[ -f "$REPO_ROOT/shared/searxng/limiter.toml" ]
|
|
}
|
|
|
|
@test "limiter.toml is valid TOML" {
|
|
python3 - "$REPO_ROOT/shared/searxng/limiter.toml" <<'PYEOF'
|
|
import sys, tomllib
|
|
with open(sys.argv[1], "rb") as fh:
|
|
tomllib.load(fh)
|
|
PYEOF
|
|
}
|
|
|
|
@test "limiter.toml trusts the homelab LAN in pass_ip" {
|
|
python3 - "$REPO_ROOT/shared/searxng/limiter.toml" <<'PYEOF'
|
|
import sys, tomllib
|
|
with open(sys.argv[1], "rb") as fh:
|
|
data = tomllib.load(fh)
|
|
ip_lists = data.get("botdetection", {}).get("ip_lists", {})
|
|
pass_ip = ip_lists.get("pass_ip", [])
|
|
assert "192.168.0.0/24" in pass_ip, f"LAN missing from pass_ip; got {pass_ip}"
|
|
PYEOF
|
|
}
|
|
|
|
@test "limiter.toml trusts the LAN as a proxy for X-Forwarded-For" {
|
|
python3 - "$REPO_ROOT/shared/searxng/limiter.toml" <<'PYEOF'
|
|
import sys, tomllib
|
|
with open(sys.argv[1], "rb") as fh:
|
|
data = tomllib.load(fh)
|
|
bd = data.get("botdetection", {})
|
|
proxies = bd.get("trusted_proxies", [])
|
|
assert "192.168.0.0/24" in proxies, f"LAN missing from trusted_proxies; got {proxies}"
|
|
PYEOF
|
|
}
|
|
|
|
@test "limiter.toml pass_ip covers local + podman sources (loopback + slirp/bridge)" {
|
|
# Rootless podman strips the client IP on published-port hits: the container
|
|
# sees a slirp/bridge source (10.88.x, 10.89.x), never the real client. And
|
|
# the JSON rate limit (API_MAX=4/hour, hardcoded) 429s any non-passed IP.
|
|
# So local subagents must present a pass_ip source. Loopback + the host's
|
|
# private podman ranges are trusted; they are reachable only from the host.
|
|
python3 - "$REPO_ROOT/shared/searxng/limiter.toml" <<'PYEOF'
|
|
import sys, tomllib
|
|
with open(sys.argv[1], "rb") as fh:
|
|
data = tomllib.load(fh)
|
|
pass_ip = data.get("botdetection", {}).get("ip_lists", {}).get("pass_ip", [])
|
|
need = ["127.0.0.1/32", "::1", "192.168.0.0/24", "10.88.0.0/16", "10.89.0.0/16"]
|
|
missing = [n for n in need if n not in pass_ip]
|
|
assert not missing, f"pass_ip missing {missing}; got {pass_ip}"
|
|
PYEOF
|
|
}
|
|
|
|
@test "SearXNG outgoing policy allows modest VPN latency without retry amplification" {
|
|
python3 - "$REPO_ROOT/shared/searxng/settings.yml" <<'PYEOF'
|
|
import sys, yaml
|
|
with open(sys.argv[1]) as fh:
|
|
data = yaml.safe_load(fh)
|
|
outgoing = data.get("outgoing", {})
|
|
assert outgoing.get("request_timeout") == 5.0, outgoing
|
|
assert outgoing.get("max_request_timeout") == 15.0, outgoing
|
|
assert outgoing.get("retries") == 1, outgoing
|
|
assert outgoing["request_timeout"] < outgoing["max_request_timeout"], outgoing
|
|
PYEOF
|
|
}
|
|
|
|
@test "SearXNG default provider policy favors anonymous healthy engines" {
|
|
python3 - "$REPO_ROOT/shared/searxng/settings.yml" <<'PYEOF'
|
|
import sys, yaml
|
|
with open(sys.argv[1]) as fh:
|
|
data = yaml.safe_load(fh)
|
|
engines = {entry["name"]: entry for entry in data.get("engines", [])}
|
|
assert engines["bing"]["disabled"] is False, engines.get("bing")
|
|
assert engines["bing"]["weight"] > 1, engines["bing"]
|
|
assert engines["bing"]["paging"] is False, engines["bing"]
|
|
assert engines["duckduckgo"]["disabled"] is False, engines["duckduckgo"]
|
|
assert engines["duckduckgo"]["paging"] is False, engines["duckduckgo"]
|
|
for name in ("google cse", "startpage", "brave"):
|
|
assert engines[name]["disabled"] is True, engines.get(name)
|
|
assert engines[name].get("inactive", False) is False, engines[name]
|
|
for name in ("bing", "duckduckgo", "wikipedia"):
|
|
assert engines[name]["timeout"] == 5.0, engines[name]
|
|
PYEOF
|
|
}
|
|
|
|
@test "deploy.sh copies limiter.toml into /opt/searxng/config" {
|
|
local d="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
grep -q "shared/searxng/limiter.toml" "$d" \
|
|
|| { echo "deploy-remote.sh does not copy shared/searxng/limiter.toml"; return 1; }
|
|
grep -q "/opt/searxng/config/limiter.toml" "$d" \
|
|
|| { echo "deploy-remote.sh does not place limiter.toml at /opt/searxng/config"; return 1; }
|
|
}
|
|
|
|
@test "kagi engine is wired enabled with an env-referenced key" {
|
|
python3 - "$REPO_ROOT/shared/searxng/settings.yml" <<'PYEOF'
|
|
import sys, yaml
|
|
with open(sys.argv[1]) as fh:
|
|
data = yaml.safe_load(fh)
|
|
engines = {entry["name"]: entry for entry in data.get("engines", [])}
|
|
assert "kagi" in engines, "kagi engine block missing from settings.yml"
|
|
k = engines["kagi"]
|
|
assert k["engine"] == "kagi", k
|
|
assert k["disabled"] is False, k
|
|
assert k["weight"] == 2, k
|
|
assert k["paging"] is False, k
|
|
assert k["kagi_categ"] == "search", k
|
|
# The tracked file must reference the env placeholder, never a literal key.
|
|
assert k["api_key"] == "${KAGI_API_KEY}", k
|
|
PYEOF
|
|
}
|
|
|
|
@test "searxng render script substitutes allowlisted vars and nothing else" {
|
|
local tmp="$BATS_TEST_TMPDIR/render"
|
|
mkdir -p "$tmp"
|
|
printf 'SEARXNG_SECRET=abc123\nKAGI_API_KEY=k-xyz\nSEARXNG_BASE_URL=http://x/y\nOTHER=must-not-appear\n' > "$tmp/env"
|
|
printf 'server:\n secret_key: "${SEARXNG_SECRET}"\n base_url: "${SEARXNG_BASE_URL}"\nengines:\n - name: kagi\n api_key: "${KAGI_API_KEY}"\n' > "$tmp/settings.yml"
|
|
python3 "$REPO_ROOT/scripts/searxng-render-settings.py" "$tmp/env" "$tmp/settings.yml"
|
|
grep -qF 'secret_key: "abc123"' "$tmp/settings.yml"
|
|
grep -qF 'base_url: "http://x/y"' "$tmp/settings.yml"
|
|
grep -qF 'api_key: "k-xyz"' "$tmp/settings.yml"
|
|
! grep -qF '${' "$tmp/settings.yml"
|
|
! grep -qF 'must-not-appear' "$tmp/settings.yml"
|
|
}
|
|
|
|
@test "searxng render script fails loudly on an unresolvable placeholder" {
|
|
local tmp="$BATS_TEST_TMPDIR/unresolved"
|
|
mkdir -p "$tmp"
|
|
printf 'SEARXNG_SECRET=abc123\n' > "$tmp/env"
|
|
printf 'engines:\n - name: kagi\n api_key: "${KAGI_API_KEY}"\n' > "$tmp/settings.yml"
|
|
run python3 "$REPO_ROOT/scripts/searxng-render-settings.py" "$tmp/env" "$tmp/settings.yml"
|
|
[ "$status" -ne 0 ]
|
|
[[ "$output" == *"KAGI_API_KEY"* ]]
|
|
}
|
|
|
|
@test "deployer renders the live searxng settings from the env file" {
|
|
grep -q "searxng-render-settings.py" "$REPO_ROOT/scripts/deploy-remote.sh" \
|
|
|| { echo "deploy-remote.sh must run searxng-render-settings.py"; return 1; }
|
|
grep -q "/opt/searxng/.env" "$REPO_ROOT/scripts/deploy-remote.sh" \
|
|
|| { echo "deploy-remote.sh must source /opt/searxng/.env for rendering"; return 1; }
|
|
}
|