diff --git a/dankclaw/.env.example b/dankclaw/.env.example index d928ccd..3affd70 100644 --- a/dankclaw/.env.example +++ b/dankclaw/.env.example @@ -4,6 +4,11 @@ # SearXNG SEARXNG_SECRET=changeme SEARXNG_BASE_URL=http://search.franksdank.cloud/ +# Kagi search API key (https://help.kagi.com/kagi/api/). Powers the paid +# `kagi` engine in shared/searxng/settings.yml; deploy-remote.sh bakes it into +# the live settings.yml via scripts/searxng-render-settings.py. The quadlet +# does NOT pass it into the container environment. +KAGI_API_KEY=changeme # Gluetun VPN (see https://github.com/qdm12/gluetun-wiki) # Single-peer WireGuard model. Fill from your provider's WireGuard config. diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 059431d..a28f8a5 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -209,6 +209,11 @@ copy_searxng_config() { run_privileged mkdir -p /opt/searxng/config run_privileged cp shared/searxng/settings.yml /opt/searxng/config/settings.yml run_privileged cp shared/searxng/limiter.toml /opt/searxng/config/limiter.toml + # Bake ${KAGI_API_KEY}/${SEARXNG_SECRET}/${SEARXNG_BASE_URL} into the live + # copy from the host-side /opt/searxng/.env (SearXNG does not expand env + # vars in settings.yml). Fails the deploy if a placeholder is unresolvable. + run_privileged python3 scripts/searxng-render-settings.py \ + /opt/searxng/.env /opt/searxng/config/settings.yml } copy_prometheus_config() { diff --git a/scripts/searxng-render-settings.py b/scripts/searxng-render-settings.py new file mode 100755 index 0000000..846fac9 --- /dev/null +++ b/scripts/searxng-render-settings.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Render ${VAR} placeholders in the live SearXNG settings.yml from the +host-side env file. + +Why this exists: SearXNG does NOT expand environment variables inside +settings.yml. The repo-tracked shared/searxng/settings.yml deliberately keeps +secrets as "${KAGI_API_KEY}"-style placeholders so no key ever enters git; the +deployer copies that file to /opt/searxng/config/settings.yml and runs this +script to bake in the real values. Without it the app runs with the literal +placeholder strings (observed 2026-09-26: secret_key literally +"${SEARXNG_SECRET}" and the Kagi engine failing setup() on an empty key). + +Only the allowlisted names are substituted. A placeholder that has no value in +the env file is a deploy-blocking error: serving a paid-API engine with a +literal "${KAGI_API_KEY}" bearer would waste requests and hide the misconfig. + +Usage: searxng-render-settings.py ENV_FILE SETTINGS_FILE +Both paths are taken as arguments; secrets never appear in any argv. +Exit codes: 0 rendered (idempotent on an already-rendered file), 1 unresolved +placeholder, 2 usage error. +""" + +import sys + +ALLOWED = ("KAGI_API_KEY", "SEARXNG_SECRET", "SEARXNG_BASE_URL") + + +def load_env(path: str) -> dict[str, str]: + values: dict[str, str] = {} + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + if key in ALLOWED: + values[key] = value + return values + + +def main(argv: list[str]) -> int: + args = [a for a in argv[1:]] + if len(args) != 2: + print( + "usage: searxng-render-settings.py ENV_FILE SETTINGS_FILE", + file=sys.stderr, + ) + return 2 + env_file, settings_file = args + + values = load_env(env_file) + with open(settings_file, encoding="utf-8") as fh: + text = fh.read() + + for name in ALLOWED: + if name in values: + text = text.replace("${%s}" % name, values[name]) + + unresolved = [n for n in ALLOWED if "${%s}" % n in text] + if unresolved: + print( + "unresolved placeholders in %s: %s (missing from %s)" + % (settings_file, ", ".join(unresolved), env_file), + file=sys.stderr, + ) + return 1 + + with open(settings_file, "w", encoding="utf-8") as fh: + fh.write(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/serverAppConfigs-docs/Dankclaw/SearXNG Providers.md b/serverAppConfigs-docs/Dankclaw/SearXNG Providers.md index 53717db..3876561 100644 --- a/serverAppConfigs-docs/Dankclaw/SearXNG Providers.md +++ b/serverAppConfigs-docs/Dankclaw/SearXNG Providers.md @@ -363,8 +363,15 @@ misleading). disable pagination for rate-sensitive engines. 3. **Promote by evidence** — raise `weight` only for engines that return useful results consistently; weight is not a health check. -4. **Avoid credentials** — do not add API-backed engines unless the provider is - genuinely free, anonymous, and compatible with our privacy requirements. +4. **Avoid credentials, with one carve-out** — do not add API-backed engines + unless the provider is genuinely free, anonymous, and compatible with our + privacy requirements. Carve-out (2026-09-26): a paid, user-ordered API + engine (Kagi) is acceptable when the blast radius is bounded — the + `kagi` engine runs with `paging: false` and weight 2, so it spends exactly + one API credit per aggregated query, and its key never enters git + (deploy-time rendering from `/opt/searxng/.env`; see [[SearXNG]]). It is + also immune to the shared-VPN-exit-IP blocking this page is about: it + authenticates, so reputation of the exit IP is irrelevant. 5. **Verify first** — `/json` + `?engines=google` and the searx-checker to see exactly which engines are down and why. Confirm the running SearXNG version (check whether `parsers`/`rate_limit` exist in the pinned build). diff --git a/serverAppConfigs-docs/Dankclaw/SearXNG.md b/serverAppConfigs-docs/Dankclaw/SearXNG.md index 4cb969c..7d06352 100644 --- a/serverAppConfigs-docs/Dankclaw/SearXNG.md +++ b/serverAppConfigs-docs/Dankclaw/SearXNG.md @@ -60,8 +60,8 @@ Deploy with `scripts/deploy-searxng-quadlet.sh dankclaw`. It copies the units, d ```yaml use_default_settings: true server: - secret_key: "${SEARXNG_SECRET}" # from .env - base_url: "${SEARXNG_BASE_URL}" # resolves to http://192.168.0.143:8081/ + secret_key: "${SEARXNG_SECRET}" # rendered at deploy by scripts/searxng-render-settings.py + base_url: "${SEARXNG_BASE_URL}" # same renderer; SearXNG does NOT expand env vars itself limiter: true method: "POST" @@ -90,8 +90,16 @@ engines: - name: duckduckgo disabled: false weight: 2 - paging: false - timeout: 5.0 + - name: kagi + engine: kagi + categories: [general, web] + shortcut: kg + api_key: "${KAGI_API_KEY}" # rendered at deploy from /opt/searxng/.env + kagi_categ: search + disabled: false + weight: 2 + paging: false # one API credit per aggregated query + timeout: 8.0 - name: google cse disabled: true - name: brave @@ -168,10 +176,39 @@ root cause, the suspend-model table, the `engines:` config, and the LAN by construction (no public route to the LAN IP). No second agent-only instance is needed; a duplicate would double upstream request volume from the same rate-sensitive VPN exit IP for zero benefit. -- The config file is root-owned (uid of the rootless podman namespace). Update it with a one-off `sudo cp`, then run `systemctl --user restart searxng.service` to reload. +- The config file is root-owned (uid of the rootless podman namespace). The + official path is `deploy.sh --service searxng`: it copies the repo settings + and renders the `${VAR}` placeholders via `scripts/searxng-render-settings.py` + from `/opt/searxng/.env`. A hotfix can still hand-copy with `sudo cp`, but + remember the renderer: an unrendered copy silently serves the literal + placeholders (before 2026-09-26 the live `secret_key` literally was + `"${SEARXNG_SECRET}"` and nobody noticed). - `base_url` must match the public address or proxy rewrites break. - Keep the valkey cache container on the same podman network as searxng. + +## Kagi (paid API engine) + +Added 2026-09-26 (user-ordered). The stock SearXNG image ships a native +`kagi` engine module (`searx/engines/kagi.py`); it authenticates with +`Authorization: Bearer ` against `POST https://kagi.com/api/v1/search`. + +- **Enabled**: `weight: 2` alongside bing/duckduckgo, categories + `[general, web]`, shortcut `kg`, `kagi_categ: search`. +- **Cost control**: `paging: false` — exactly one API credit per aggregated + query; `timeout: 8.0` so a slow paid query is not wasted (scrapers use 5.0). +- **Key handling**: the repo file keeps `api_key: "${KAGI_API_KEY}"`; + `deploy-remote.sh` bakes the real key into the live settings.yml via + `scripts/searxng-render-settings.py`, reading `/opt/searxng/.env` + (dank-owned, mode 600). The key never enters git and never appears in any + process argv. SearXNG itself does NOT expand env vars in settings.yml — + without the renderer the engine fails `setup()` and is dropped at load. +- **Immune to the VPN-exit-IP problem**: Kagi authenticates, so it does not + care about the shared Gluetun exit IP reputation (see [[SearXNG Providers]]). +- **Verified live 2026-09-26**: direct API probe through the gluetun netns + (200, `data.search[]` shape), then a live aggregated query returned + 29/29 results tagged `kagi`, zero unresponsive engines. + ## See Also - [[Dankclaw Stack]] diff --git a/shared/searxng/settings.yml b/shared/searxng/settings.yml index e469f3a..55a098b 100644 --- a/shared/searxng/settings.yml +++ b/shared/searxng/settings.yml @@ -1,5 +1,7 @@ # SearXNG settings — deployed to /opt/searxng/config/settings.yml on Dankclaw. -# Secret key and base URL come from environment variables (see .env.example). +# ${VAR} placeholders below are rendered at deploy time by +# scripts/searxng-render-settings.py from the host-side /opt/searxng/.env +# (SearXNG does not expand env vars in settings.yml itself). use_default_settings: true @@ -61,6 +63,18 @@ engines: - name: duckduckgo videos paging: false timeout: 5.0 + - name: kagi + engine: kagi + categories: [general, web] + shortcut: kg + api_key: "${KAGI_API_KEY}" + kagi_categ: search + disabled: false + weight: 2 + # Paid API (billed per search): no pagination fan-out, and a wider + # timeout than the scrapers above so slow queries are not wasted credits. + paging: false + timeout: 8.0 - name: wikipedia weight: 1 timeout: 5.0 diff --git a/tests/searxng_limiter.bats b/tests/searxng_limiter.bats index c0e31a8..63d81a8 100644 --- a/tests/searxng_limiter.bats +++ b/tests/searxng_limiter.bats @@ -96,3 +96,51 @@ PYEOF grep -q "/opt/searxng/config/limiter.toml" "$d" \ || { echo "deploy-remote.sh does not place limiter.toml at /opt/searxng/config"; return 1; } } + +@test "kagi engine is wired enabled with an env-referenced key" { + python3 - "$REPO_ROOT/shared/searxng/settings.yml" <<'PYEOF' +import sys, yaml +with open(sys.argv[1]) as fh: + data = yaml.safe_load(fh) +engines = {entry["name"]: entry for entry in data.get("engines", [])} +assert "kagi" in engines, "kagi engine block missing from settings.yml" +k = engines["kagi"] +assert k["engine"] == "kagi", k +assert k["disabled"] is False, k +assert k["weight"] == 2, k +assert k["paging"] is False, k +assert k["kagi_categ"] == "search", k +# The tracked file must reference the env placeholder, never a literal key. +assert k["api_key"] == "${KAGI_API_KEY}", k +PYEOF +} + +@test "searxng render script substitutes allowlisted vars and nothing else" { + local tmp="$BATS_TEST_TMPDIR/render" + mkdir -p "$tmp" + printf 'SEARXNG_SECRET=abc123\nKAGI_API_KEY=k-xyz\nSEARXNG_BASE_URL=http://x/y\nOTHER=must-not-appear\n' > "$tmp/env" + printf 'server:\n secret_key: "${SEARXNG_SECRET}"\n base_url: "${SEARXNG_BASE_URL}"\nengines:\n - name: kagi\n api_key: "${KAGI_API_KEY}"\n' > "$tmp/settings.yml" + python3 "$REPO_ROOT/scripts/searxng-render-settings.py" "$tmp/env" "$tmp/settings.yml" + grep -qF 'secret_key: "abc123"' "$tmp/settings.yml" + grep -qF 'base_url: "http://x/y"' "$tmp/settings.yml" + grep -qF 'api_key: "k-xyz"' "$tmp/settings.yml" + ! grep -qF '${' "$tmp/settings.yml" + ! grep -qF 'must-not-appear' "$tmp/settings.yml" +} + +@test "searxng render script fails loudly on an unresolvable placeholder" { + local tmp="$BATS_TEST_TMPDIR/unresolved" + mkdir -p "$tmp" + printf 'SEARXNG_SECRET=abc123\n' > "$tmp/env" + printf 'engines:\n - name: kagi\n api_key: "${KAGI_API_KEY}"\n' > "$tmp/settings.yml" + run python3 "$REPO_ROOT/scripts/searxng-render-settings.py" "$tmp/env" "$tmp/settings.yml" + [ "$status" -ne 0 ] + [[ "$output" == *"KAGI_API_KEY"* ]] +} + +@test "deployer renders the live searxng settings from the env file" { + grep -q "searxng-render-settings.py" "$REPO_ROOT/scripts/deploy-remote.sh" \ + || { echo "deploy-remote.sh must run searxng-render-settings.py"; return 1; } + grep -q "/opt/searxng/.env" "$REPO_ROOT/scripts/deploy-remote.sh" \ + || { echo "deploy-remote.sh must source /opt/searxng/.env for rendering"; return 1; } +}