diff --git a/dankclaw/voiceservice/tts-engine-warm.sh b/dankclaw/voiceservice/tts-engine-warm.sh index f835ece..e83f4c4 100755 --- a/dankclaw/voiceservice/tts-engine-warm.sh +++ b/dankclaw/voiceservice/tts-engine-warm.sh @@ -16,17 +16,17 @@ # voice that did not warm becomes a deploy result instead of only a journal # line (the unit's `-` prefix deliberately cannot fail the unit). # -# WHY IT EXISTS AT ALL (the cold-voice law). The FIRST speech request for a -# registered voice decodes the whole reference clip to snapshot its codec state: -# 6.2-8.0 s, first audio byte at 5,530 ms. Every later request for that voice -# restores the snapshot in 0.5 ms. Two facts make that a boot problem rather than -# a curiosity: the voice registry is MEMORY ONLY (a restart forgets every voice), -# and re-registering a voice does not warm it — only synthesizing does. So a boot -# that registers the voices and stops leaves the first caller of the day paying -# 5.5 s of silence. This script speaks one short sentence per voice instead, then -# speaks a second one so the journal carries the warm number too: the line -# `voice cold=…ms warm=…ms` is the repo's standing proof that the second -# request of a voice answers well under a second. +# WHY IT EXISTS AT ALL (the registry law). The voice registry is MEMORY ONLY: a +# restart forgets every voice, and re-registering a voice does not warm it — only +# synthesizing does. A boot that registers nothing therefore leaves frank-tts +# answering 400 for every name, so this script registers every pack it finds. The +# per-voice cold penalty is real but small, and smaller than the trial reported: +# measured on the deployed build 2026-10-18 against an empty registry, the FIRST +# synthesis of a freshly registered voice cost 1,883 ms against 1,226-1,283 ms for +# the requests after it — about 0.6 s, not the trial's 6.2-8.0 s. Each voice gets +# two identical requests so the journal carries both numbers, `voice +# cold=…ms warm=…ms`; cold ≈ warm is the expected shape, and a large gap is what +# would mean the snapshot path is broken. # # THE VOICES. /voices — the host directory /home/dank/voices mounted read-only by # the unit, the same packs frank-tts mounts for its own name list. A pack is a @@ -47,7 +47,9 @@ # # NOTHING HERE IS A SECRET, and nothing is logged that could be: the audio bytes # are discarded, the synthesized text comes from the env file, and the reference -# clip is base64'd straight into the request body without ever being echoed. +# clip is base64'd into a request body that is written to a temporary file and +# handed to curl as `@file` — never echoed, and never placed on a command line, +# because of the size law explained at the mktemp below. set -euo pipefail # The engine's own address, from inside its namespace. Loopback is valid HERE: @@ -103,6 +105,18 @@ fi warm_json="${warm_text//\\/\\\\}" warm_json="${warm_json//\"/\\\"}" +# The registration body is a FILE, never an argument. The kernel caps a single +# command-line string at 32 pages — MAX_ARG_STRLEN = 131,072 bytes — and a shipped +# reference clip base64's to about 1.27 MB, so passing the body as one argument +# fails with E2BIG before curl is even executed. The first real boot of the unit +# (2026-10-18, the three shipped packs) proved it end to end: `/usr/bin/curl: +# Argument list too long` for every voice, reported as the misleading +# "registration failed (HTTP no response)". `mktemp` lands in the container's +# /tmp as the unprivileged `tts` user; one file for the whole pass, overwritten +# per voice, removed on any exit — it is container scratch, not host state. +body_file="$(mktemp -t frank-tts-warm.XXXXXX)" +trap 'rm -f "$body_file"' EXIT + fail=0 log() { printf 'frank-tts-engine-warm: %s\n' "$*"; } @@ -146,9 +160,16 @@ for dir in "$VOICES_DIR"/*/; do continue fi - body="$(printf '{"name":"%s","wav_b64":"%s"}' "$name" "$(base64 -w0 "${dir}reference.wav")")" + # The audio enters through a redirection rather than a command substitution: + # the file is several hundred KB, and both a shell variable and an argument of + # that size are what this script used to die on. + { + printf '{"name":"%s","wav_b64":"' "$name" + base64 -w0 "${dir}reference.wav" + printf '"}' + } > "$body_file" if ! code="$(curl -sS --max-time "$REQUEST_TIMEOUT" -o /dev/null -w '%{http_code}' \ - -H 'Content-Type: application/json' --data-binary "$body" \ + -H 'Content-Type: application/json' --data-binary "@${body_file}" \ "${ENGINE_URL}/v1/audio/voices")" || [[ "$code" != "200" ]]; then log "voice '${name}': registration failed (HTTP ${code:-no response}) — the shim will answer 400 for this name" fail=1 diff --git a/serverAppConfigs-docs/Dankclaw/Frank TTS Engine.md b/serverAppConfigs-docs/Dankclaw/Frank TTS Engine.md index 209c12c..08770bf 100644 --- a/serverAppConfigs-docs/Dankclaw/Frank TTS Engine.md +++ b/serverAppConfigs-docs/Dankclaw/Frank TTS Engine.md @@ -50,17 +50,17 @@ The spread is the whole argument: at ±0.01 the engine finishes synthesizing 1.5 ## Three laws this service lives under -**1. The cold-voice law → the unit warms at boot.** The **first** speech request for a registered voice decodes the whole reference clip to snapshot its codec state: **6.2–8.0 s, first audio byte at 5 530 ms**. Every later request for that voice restores the snapshot in **0.5 ms**. Registrations live in memory only — a restart forgets every voice — and re-registering does **not** warm a voice, only synthesizing does. So `frank-tts-engine.container` carries +**1. The registry law → the unit warms at boot.** The voice registry is **memory only**: a restart forgets every voice, and re-registering one does **not** warm it — only synthesizing does. A boot that registers nothing leaves the façade answering 400 for every name, which is why `frank-tts-engine.container` carries ``` ExecStartPost=-/usr/bin/podman exec frank-tts-engine /app/warm-voices.sh TimeoutStartSec=900 ``` -and the warm pass runs on every start including boot. Without it, whoever asks for audio first that day pays 5.5 s of silence. The `-` prefix is load-bearing: with `Restart=always`, a warmer that exits non-zero would restart the container and crash-loop the endpoint forever. The hook is `podman exec` rather than a host script **because this container publishes nothing** — there is no address the host could dial — and a consequence worth knowing: an exec inherits the unit's `EnvironmentFile`, so the warmer's knobs (`VOICES_DIR`, `HEALTH_WAIT`, `WARM_TEXT`, `REQUEST_TIMEOUT`) live in `scripts/frank-tts-engine.env` beside the engine's own. The deployer runs the same hook a second time so a failed warm becomes a deploy result rather than only a journal line, and so the per-voice numbers land on the console: +and the warm pass runs on every start including boot. What it buys is registration (mandatory) plus a measurably smaller first request — and the size of that saving is **not** the number the trial published. **Re-measured on the deployed build, 2026-10-18,** against an empty registry (register, then time the synthesis): the first request for `narrator1` cost **1 883 ms**, the second **1 283 ms**, the third **1 226 ms** — about **0.6 s** of saved silence. A ~110-word request cost **9.4 s** and then **9.8 s**, so request time tracks the audio generated, not snapshot state. The trial's **6.2–8.0 s / first audio byte 5 530 ms / later requests 0.5 ms** ladder does **not** reproduce here; the 0.5 ms figure is an internal state-restore cost that was never an end-to-end request time, and the honest claim is the ~0.6 s above. The `-` prefix is load-bearing: with `Restart=always`, a warmer that exits non-zero would restart the container and crash-loop the endpoint forever. The hook is `podman exec` rather than a host script **because this container publishes nothing** — there is no address the host could dial — and a consequence worth knowing: an exec inherits the unit's `EnvironmentFile`, so the warmer's knobs (`VOICES_DIR`, `HEALTH_WAIT`, `WARM_TEXT`, `REQUEST_TIMEOUT`) live in `scripts/frank-tts-engine.env` beside the engine's own. The deployer runs the same hook a second time so a failed warm becomes a deploy result rather than only a journal line, and so the per-voice numbers land on the console: ``` -frank-tts-engine-warm: voice narrator1 cold=6821ms warm=412ms +frank-tts-engine-warm: voice narrator1 cold=1402ms warm=1812ms frank-tts-engine-warm: 3 voice(s) registered and warm ``` @@ -122,7 +122,7 @@ Nothing below has been run. Steps 1–5 are this service; 6–7 are what unblock |---|---|---| | Unit restarts every 10 s, journal says `cannot read /models/…` | weights missing or the mount is empty | run the fetcher; re-deploy | | Unit `active`, but every voice name returns 400 from the façade | the engine restarted and lost its in-memory registry, or the warm pass failed | `journalctl --user -u frank-tts-engine` for the warmer's lines; `podman exec frank-tts-engine /app/warm-voices.sh` | -| First request of the day is slow, the rest are fast | the warm pass did not run (the hook carries `-`, so it cannot fail the unit) | look for `did not answer` in the journal — the warmer gave up while the model was still loading | +| Every voice logs `registration failed (HTTP no response)`, with `/usr/bin/curl: Argument list too long` on the line above it | the hook handed the registration body to curl as one argument: the kernel caps a single argument at 131 072 bytes (`MAX_ARG_STRLEN`), and a shipped `reference.wav` base64's to about 1.27 MB | the body goes through a temp file and `--data-binary @file` (2026-10-18). Any pack over ~96 KB trips this, so no small fixture can see it — the seam ships a 300 KB pack for exactly that reason | | Everything is ~2× slower after a systemd tweak | a cpuset/cap pin changed `hardware_concurrency()` and the engine resized its thread pools | remove the pin (law 2) | | `502` from the façade with `frank-tts upstream request failed` | the façade cannot resolve or reach the engine | `podman network inspect systemd-frank-tts-engine` and `podman inspect frank-tts --format '{{json .NetworkSettings.Networks}}'` — the façade must be attached to the bridge; a `127.0.0.1:8880` or `host.containers.internal:8880` upstream in `scripts/frank-tts.env` is the old shape and cannot work | | A voice is missing from the list | its directory has no `reference.wav`, or the name is not lowercase | fix the pack; both are refused loudly by design | diff --git a/serverAppConfigs-docs/Dankclaw/Voice Packs.md b/serverAppConfigs-docs/Dankclaw/Voice Packs.md index 979b935..5ed466f 100644 --- a/serverAppConfigs-docs/Dankclaw/Voice Packs.md +++ b/serverAppConfigs-docs/Dankclaw/Voice Packs.md @@ -169,10 +169,14 @@ invariant to every crop in the tracked pack list. curl -s localhost:7772/v1/audio/voices # the façade's list, not the engine's ``` - The hook walks `/voices`, registers any pack with a `reference.wav` it has not - seen, and logs `voice cold=…ms warm=…ms` to the unit's journal. The - rootless unit restart — `systemctl --user restart frank-tts-engine.service` — - is for when the unit or its env changed, not for a new pack. + The hook walks `/voices` and registers **every** pack that has a + `reference.wav`, on every run — the engine replaces a name it already holds + (`tools/tts-server.cpp:148` at the pinned commit: *"Re-registering a name + replaces the previous entry."*) — then synthesizes twice per voice and logs + `voice cold=…ms warm=…ms` to the unit's journal. The two numbers should + be close; see law 1 of [[Frank TTS Engine]] for what they measure. The rootless + unit restart — `systemctl --user restart frank-tts-engine.service` — is for when + the unit or its env changed, not for a new pack. ## When the extractor's output size changes on an engine bump diff --git a/serverAppConfigs-docs/Research/Local TTS and Voice Cloning.md b/serverAppConfigs-docs/Research/Local TTS and Voice Cloning.md index 836b68f..93fe561 100644 --- a/serverAppConfigs-docs/Research/Local TTS and Voice Cloning.md +++ b/serverAppConfigs-docs/Research/Local TTS and Voice Cloning.md @@ -258,11 +258,17 @@ HTTP path at all — that is what the façade's CLI backend exists for. Four laws came out of the trial, each worth a line of configuration: -- **Boot-warm every voice.** The first request for a registered voice decodes - the whole reference to snapshot codec state: 6.2–8.0 s, first audio byte at - 5,530 ms. Later requests restore that state in 0.5 ms. An engine unit must - run one throwaway synthesis per voice at start, or the first request of the - day pays five seconds. +- **Boot-warm every voice.** The trial measured the first request for a + registered voice at 6.2–8.0 s (first audio byte 5,530 ms) and later requests + restoring the snapshot in 0.5 ms. **That ladder did not survive deployment:** + measured 2026-10-18 on the built container against an empty registry — register, + then time — the first synthesis of a freshly registered voice cost **1,883 ms** + against **1,226–1,283 ms** for the requests after it, and a ~110-word request + cost **9.4 s** then **9.8 s**, so request time tracks the audio generated rather + than snapshot state. What still holds is the reason to warm at all: the registry + is memory-only and re-registering does not warm, so an engine unit must run one + throwaway synthesis per voice at start. It buys registration and about 0.6 s, + not five seconds. - **Speaker artifacts are reusable and exact.** `qwen-codec` turns a reference WAV into `.spk` (8,192 B x-vector) + `.rvq` (5,456–6,864 B, 16 codebooks × T, 11-bit packed) in 1.8–2.1 s at 1.3–1.5 GiB peak; replaying them yields a diff --git a/serverAppConfigs-docs/log.md b/serverAppConfigs-docs/log.md index 1f73602..388b71d 100644 --- a/serverAppConfigs-docs/log.md +++ b/serverAppConfigs-docs/log.md @@ -2643,3 +2643,42 @@ every restart since the cutover. and a new unit on a host that serves people — Frank's call), no voice pack built on the host, no Playwright run (this branch publishes no host port and touches no vhost), and Gitea **#2** phase 1 stays open until 7772 answers. + +## 2026-10-18 — frank-tts-engine deployed: it builds, it serves, and the warm hook died on the kernel's argument cap + +- Merged as **#14** (`747c665`) and deployed `--service frank-tts-engine`. The + payload took 18 s to fetch for the 1.4 GiB pair (byte count + Hub LFS sha256, + `--verify-only` OK), and the image built clean — the first real proof of the + `useradd` fix: `STEP 3/13: RUN useradd --system --uid 10000 …` passed with only + the documented cosmetic `SYS_UID_MAX` warning, and the unit was `healthy` after + 31 s. +- Then the deploy reported `0 voice(s) warm`: + `/app/warm-voices.sh: line 152: /usr/bin/curl: Argument list too long`. The hook + handed the registration body to curl as ONE argument; the kernel caps a single + argument at 131,072 bytes (`MAX_ARG_STRLEN`) and a shipped 955 KB + `reference.wav` base64's to about 1.27 MB, so curl never executed and the hook + logged that as "HTTP no response". Fourth silent fault of the voice stack, same + species as the other three: `/health` stayed green throughout. +- Fix: the body goes to a `mktemp` file and curl gets `@file`. The regression test + is the honest shape — the seam's stub engine is a real listener, `curl` is the + real binary, the fixture pack is 300 KB — so it fails with the same kernel error + the host produced (proved red first, green after). Every shipped pack sits well + over the ~96 KB cliff, so no small fixture could ever have seen this. +- Live proof on the deployed container with the three real packs: + `narrator1 cold=1175ms warm=1874ms`, `narrator2 1373/1892`, `narrator3 + 1816/1487`, exit 0, 15.2 s for the whole pass. +- **A trial law did not survive deployment.** Against an empty registry — + register only, then time — the FIRST synthesis of a freshly registered voice + cost **1 883 ms**, the next two **1 283** and **1 226 ms**; a ~110-word request + cost **9.4 s** then **9.8 s**. The trial's 6.2–8.0 s cold / 0.5 ms restore + ladder, which the warmer's own header repeated, is retired: warming buys + registration — mandatory, because the registry is memory-only — plus about + **0.6 s**. Corrected in [[Frank TTS Engine]] law 1 and its troubleshooting + table, in [[Local TTS and Voice Cloning]], and in [[Voice Packs]]. +- Host state, honestly: the engine is `active` with three voices warm, but the hook + fix is not in the image yet, so the next restart re-breaks it until this branch + is merged and re-deployed. `frank-tts` (the 7772 façade) is still undeployed. +- Deliberately NOT done in this window: the `frank-tts` deploy, any Prometheus + probe for the voice pair (neither container exposes `/metrics`; the blackbox + question is still open), and the podcast render — open-notebook is on `main` but + not installed. diff --git a/tests/frank_tts_engine.bats b/tests/frank_tts_engine.bats index 74d9d95..4290b31 100755 --- a/tests/frank_tts_engine.bats +++ b/tests/frank_tts_engine.bats @@ -394,6 +394,37 @@ assert {s["response_format"] for s in speech} == {"wav"}, speech PY } +@test "a reference clip too big for one argument still registers" { + # The kernel caps a SINGLE command-line string at 32 pages = 131,072 bytes + # (MAX_ARG_STRLEN), and a shipped reference clip base64's to about 1.27 MB — + # so a warmer that hands the registration body to curl as an argument dies + # before curl runs. That is exactly what the first real boot of 2026-10-18 + # hit with the real packs: `/usr/bin/curl: Argument list too long`, then the + # misleading "registration failed (HTTP no response)" for all three voices. + # The stub engine is a real listener and curl is the real binary, so this + # test fails for the same kernel reason the host did. + start_engine_stub + local big="$TEST_TMP/bigvoices/bigvoice" + mkdir -p "$big" + head -c 300000 /dev/zero | tr '\0' 'A' > "$big/reference.wav" + + run env ENGINE_URL="http://127.0.0.1:$STUB_PORT" VOICES_DIR="$TEST_TMP/bigvoices" \ + bash "$WARM" + [ "$status" -eq 0 ] || { echo "$output"; false; } + [[ "$output" == *"voice bigvoice cold="*"ms warm="*"ms"* ]] || { echo "$output"; false; } + + # And the audio that arrived is the audio that was on disk, whole. + python3 - "$TEST_TMP/journal.jsonl" "$big/reference.wav" <<'PY' +import hashlib, json, sys +entries = [json.loads(l) for l in open(sys.argv[1])] +regs = [e for e in entries if e["path"] == "/v1/audio/voices"] +assert len(regs) == 1, entries +assert regs[0]["name"] == "bigvoice", regs[0] +want = hashlib.sha256(open(sys.argv[2], "rb").read()).hexdigest() +assert regs[0]["wav_sha256"] == want, (len(regs[0]["wav_sha256"]), want) +PY +} + @test "the warm sentence survives both readers of the env file intact" { # scripts/frank-tts-engine.env is read two ways and they disagree about # quotes. bash (`set -a; source`, what the seams and a human do) strips the