feat(dankvoice): speak the cloned voices in the TUI (Refs #21)
Every speak-aloud path in the harness gates `--voice` against the bundled Kokoro enum, so the frank-tts cloned voices could only ever produce files. `agent/extensions/` auto-loads, so one tracked extension adds the missing transport: `/say <text>`, `/say on|off` (narrate the final answer at `agent_end`, silent when the harness schedules a continuation), `/say voice|voices|stop|status` — OpenAI speech POST plus playback through pw-play/paplay/ffplay. The endpoint and served id come from the `api: openai-speech` row of `agent/models.yml`, so the file carries no host and no secret; markdown is stripped, over-long utterances are cut at a sentence bound, and the temp WAV dir is always removed. Verified live in a real TUI session on the workstation (roster narrator1/narrator2/narrator3; 2.08 s and 1.44 s WAVs played; auto-speak spoke each finished answer, 1.20 s WAV; unknown voice answers with the roster). Black-box seam `tests/dankvoice.bats` (12 tests) drives the extension against a fixture speech endpoint and a stub player. cwd: /home/blank/.omp/wt/feat-21-dankvoice session: wt-20261017-181502-6d1fc1f
This commit is contained in:
Frank
@@ -0,0 +1,463 @@
|
||||
/**
|
||||
* dankvoice — cloned-voice vocalization for omp, served by frank-tts.
|
||||
*
|
||||
* The built-in speak-aloud feature (`speech.enabled` / `speech.voice`) and the
|
||||
* `omp say` CLI are gated to the bundled Kokoro voice enum, so the cloned
|
||||
* voices on frank-tts (Dankclaw :7772, the `speech` model role in
|
||||
* agent/models.yml) could only ever produce files. This extension adds the
|
||||
* missing path — the TUI speaks with the cloned voices:
|
||||
*
|
||||
* /say <text> speak once through the speakers
|
||||
* /say on | off auto-speak the assistant's final answer each turn
|
||||
* /say voice <name> choose a voice (empty = whatever the server defaults to)
|
||||
* /say voices list the voices frank-tts reports
|
||||
* /say stop cut off the current utterance
|
||||
* /say status voice, endpoint and player in use
|
||||
*
|
||||
* The endpoint comes from agent/models.yml — the provider block carrying
|
||||
* `api: openai-speech`, i.e. the same row the speech role resolves — so this
|
||||
* file carries no host address and no secret. Environment overrides:
|
||||
*
|
||||
* DANKVOICE_URL speech base URL (…/v1) instead of models.yml
|
||||
* DANKVOICE_MODEL served model id (default: the id in that block)
|
||||
* DANKVOICE_VOICE default voice name
|
||||
* DANKVOICE_AUTO=1 start with auto-speak armed
|
||||
* DANKVOICE_MAX spoken-character cap per utterance (default 700)
|
||||
* DANKVOICE_PLAYER player binary, skipping detection
|
||||
* DANKVOICE_MODELS path to models.yml (default ~/.omp/agent/models.yml)
|
||||
*
|
||||
* Utterances are synthesized to a temp WAV in $TMPDIR; the file is removed
|
||||
* when its player process exits. A new utterance interrupts the previous one
|
||||
* instead of queueing behind it.
|
||||
*/
|
||||
|
||||
import { spawn, spawnSync, type ChildProcess } from "node:child_process";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
||||
|
||||
const DEFAULT_MAX_CHARS = 700;
|
||||
const SYNTH_TIMEOUT_MS = 60_000;
|
||||
const VOICES_TIMEOUT_MS = 15_000;
|
||||
/** A voice name the shim must reject; its 400 body carries the real list. */
|
||||
const VOICE_PROBE = "_dankvoice_probe_";
|
||||
|
||||
// ---------------------------------------------------------------- pure logic
|
||||
|
||||
export interface SpeechTarget {
|
||||
/** POST target for OpenAI speech synthesis. */
|
||||
speechUrl: string;
|
||||
/** GET target for the voice roster (frank-tts answers a bare JSON array). */
|
||||
voicesUrl: string;
|
||||
/** GET target for the liveness route of the same host:port. */
|
||||
healthUrl: string;
|
||||
/** Served model id to name in the request body. */
|
||||
model: string;
|
||||
}
|
||||
|
||||
export interface Player {
|
||||
command: string;
|
||||
args: string[];
|
||||
}
|
||||
|
||||
export type SayAction =
|
||||
| { kind: "help" }
|
||||
| { kind: "speak"; text: string }
|
||||
| { kind: "on" }
|
||||
| { kind: "off" }
|
||||
| { kind: "stop" }
|
||||
| { kind: "voices" }
|
||||
| { kind: "status" }
|
||||
| { kind: "voice"; name: string };
|
||||
|
||||
/** `/say <args>` grammar: subcommands win, anything else is spoken text. */
|
||||
export function parseSayArgs(raw: string): SayAction {
|
||||
const args = raw.trim();
|
||||
const lower = args.toLowerCase();
|
||||
if (lower === "" || lower === "help") return { kind: "help" };
|
||||
if (lower === "on") return { kind: "on" };
|
||||
if (lower === "off") return { kind: "off" };
|
||||
if (lower === "stop") return { kind: "stop" };
|
||||
if (lower === "voices") return { kind: "voices" };
|
||||
if (lower === "status") return { kind: "status" };
|
||||
if (lower === "voice" || lower.startsWith("voice ")) return { kind: "voice", name: args.slice(5).trim() };
|
||||
return { kind: "speak", text: args };
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the speech endpoint out of agent/models.yml. The runner API lives on
|
||||
* the MODEL row (`api: openai-speech`), the base URL on its provider, so this
|
||||
* scan remembers the last provider base URL and model id seen and returns when
|
||||
* a model row claims the speech runner API. A new provider key (indent two)
|
||||
* resets both.
|
||||
*/
|
||||
export function resolveSpeechEndpoint(modelsYml: string): SpeechTarget | undefined {
|
||||
let base: string | undefined;
|
||||
let id: string | undefined;
|
||||
for (const line of modelsYml.split(/\r?\n/)) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith("#")) continue;
|
||||
if (/^ {2}\S.*:\s*$/.test(line)) {
|
||||
base = undefined;
|
||||
id = undefined;
|
||||
continue;
|
||||
}
|
||||
const providerUrl = trimmed.match(/^baseUrl:\s*(\S+)/);
|
||||
if (providerUrl) {
|
||||
base = providerUrl[1].replace(/\/+$/, "");
|
||||
continue;
|
||||
}
|
||||
const modelId = trimmed.match(/^-\s*id:\s*(\S+)/);
|
||||
if (modelId) {
|
||||
id = modelId[1];
|
||||
continue;
|
||||
}
|
||||
if (/^api:\s*openai-speech\s*$/.test(trimmed) && base) {
|
||||
return {
|
||||
speechUrl: `${base}/audio/speech`,
|
||||
voicesUrl: `${base}/audio/voices`,
|
||||
healthUrl: `${base.replace(/\/v1$/, "")}/health`,
|
||||
model: id ?? "frank-tts",
|
||||
};
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** OpenAI speech body; `voice` is sent only when named — the shim carries its own default. */
|
||||
export function speechBody(model: string, text: string, voice: string): Record<string, string> {
|
||||
const body: Record<string, string> = { model, input: text, response_format: "wav" };
|
||||
if (voice) body.voice = voice;
|
||||
return body;
|
||||
}
|
||||
|
||||
/** Voice names as frank-tts reports them: the invalid-voice 400 body carries `available_voices`. */
|
||||
export function parseVoices(payload: unknown): string[] {
|
||||
// frank-tts answers GET /v1/audio/voices with a bare array and an unknown voice with
|
||||
// {error:{available_voices:[…]}}; a flat body is accepted too.
|
||||
if (typeof payload !== "object" || payload === null) return [];
|
||||
let list: unknown;
|
||||
if (Array.isArray(payload)) {
|
||||
list = payload;
|
||||
} else if ("error" in payload && typeof payload.error === "object" && payload.error !== null && "available_voices" in payload.error) {
|
||||
list = payload.error.available_voices;
|
||||
} else if ("available_voices" in payload) {
|
||||
list = payload.available_voices;
|
||||
} else if ("voices" in payload) {
|
||||
list = payload.voices;
|
||||
} else if ("data" in payload) {
|
||||
list = payload.data;
|
||||
}
|
||||
if (!Array.isArray(list)) return [];
|
||||
const names: string[] = [];
|
||||
for (const entry of list) {
|
||||
if (typeof entry === "string") {
|
||||
names.push(entry);
|
||||
} else if (typeof entry === "object" && entry !== null) {
|
||||
const name = "id" in entry ? entry.id : "name" in entry ? entry.name : undefined;
|
||||
if (typeof name === "string") names.push(name);
|
||||
}
|
||||
}
|
||||
return names.filter((name) => name.length > 0);
|
||||
}
|
||||
|
||||
/** Player preference: PipeWire first (this fleet runs WirePlumber), then Pulse, then ffmpeg. */
|
||||
export function detectPlayer(lookup: (command: string) => string | null): Player | null {
|
||||
const candidates: Player[] = [
|
||||
{ command: "pw-play", args: [] },
|
||||
{ command: "paplay", args: [] },
|
||||
{ command: "ffplay", args: ["-nodisp", "-autoexit", "-loglevel", "quiet"] },
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
if (lookup(candidate.command)) return candidate;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** What a speech engine should say: prose only, no fences, no markup, bounded. */
|
||||
export function stripForSpeech(text: string, maxChars: number = DEFAULT_MAX_CHARS): string {
|
||||
let out = text;
|
||||
out = out.replace(/```[\s\S]*?```/g, " ");
|
||||
out = out.replace(/~~~[\s\S]*?~~~/g, " ");
|
||||
out = out.replace(/`([^`]*)`/g, "$1");
|
||||
out = out.replace(/!\[[^\]]*\]\([^)]*\)/g, " ");
|
||||
out = out.replace(/\[([^\]]*)\]\([^)]*\)/g, "$1");
|
||||
out = out.replace(/^\s{0,3}#{1,6}\s+/gm, "");
|
||||
out = out.replace(/^\s*[-*+]\s+/gm, "");
|
||||
out = out.replace(/^\s*\|[-: |]+\|\s*$/gm, " ");
|
||||
out = out.replace(/\|/g, " ");
|
||||
out = out.replace(/[*_]{1,3}([^*_]+)[*_]{1,3}/g, "$1");
|
||||
out = out.replace(/\s+/g, " ").trim();
|
||||
if (out.length <= maxChars) return out;
|
||||
const window = out.slice(0, maxChars);
|
||||
const cut = Math.max(window.lastIndexOf(". "), window.lastIndexOf("! "), window.lastIndexOf("? "));
|
||||
const clipped = cut > maxChars / 2 ? window.slice(0, cut + 1) : window.replace(/[,;:]\s*\S*$/, "");
|
||||
return `${clipped.trim()} …`;
|
||||
}
|
||||
|
||||
/** The text the assistant last actually said; tool-only messages do not count. */
|
||||
export function lastAssistantText(messages: readonly unknown[]): string {
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
const message = messages[i];
|
||||
if (!message || typeof message !== "object" || !("role" in message) || message.role !== "assistant") continue;
|
||||
if (!("content" in message)) continue;
|
||||
const content = message.content;
|
||||
if (typeof content === "string") return content;
|
||||
if (!Array.isArray(content)) continue;
|
||||
const parts: string[] = [];
|
||||
for (const part of content) {
|
||||
if (part && typeof part === "object" && "type" in part && part.type === "text" && "text" in part && typeof part.text === "string") {
|
||||
parts.push(part.text);
|
||||
}
|
||||
}
|
||||
const joined = parts.join("\n").trim();
|
||||
if (joined) return joined;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------- live wiring
|
||||
|
||||
interface Config {
|
||||
target?: SpeechTarget;
|
||||
player?: Player;
|
||||
maxChars: number;
|
||||
auto: boolean;
|
||||
voice: string;
|
||||
}
|
||||
|
||||
function lookupOnPath(command: string): string | null {
|
||||
const found = spawnSync("sh", ["-c", `command -v ${command}`], { encoding: "utf8" });
|
||||
return found.status === 0 && found.stdout.trim() ? found.stdout.trim() : null;
|
||||
}
|
||||
|
||||
function readModelsYml(): string {
|
||||
const path = process.env.DANKVOICE_MODELS ?? join(process.env.HOME ?? "", ".omp", "agent", "models.yml");
|
||||
try {
|
||||
return readFileSync(path, "utf8");
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function loadConfig(): Config {
|
||||
const overrideUrl = process.env.DANKVOICE_URL?.trim().replace(/\/+$/, "");
|
||||
const modelOverride = process.env.DANKVOICE_MODEL?.trim();
|
||||
const fromYml = overrideUrl ? undefined : resolveSpeechEndpoint(readModelsYml());
|
||||
const target: SpeechTarget | undefined = overrideUrl
|
||||
? {
|
||||
speechUrl: `${overrideUrl}/audio/speech`,
|
||||
voicesUrl: `${overrideUrl}/audio/voices`,
|
||||
healthUrl: `${overrideUrl.replace(/\/v1$/, "")}/health`,
|
||||
model: modelOverride || "frank-tts",
|
||||
}
|
||||
: fromYml
|
||||
? { ...fromYml, model: modelOverride || fromYml.model }
|
||||
: undefined;
|
||||
const playerOverride = process.env.DANKVOICE_PLAYER?.trim();
|
||||
const maxChars = Number(process.env.DANKVOICE_MAX);
|
||||
return {
|
||||
target,
|
||||
player: playerOverride ? { command: playerOverride, args: [] } : (detectPlayer(lookupOnPath) ?? undefined),
|
||||
maxChars: Number.isFinite(maxChars) && maxChars > 0 ? Math.floor(maxChars) : DEFAULT_MAX_CHARS,
|
||||
auto: process.env.DANKVOICE_AUTO === "1",
|
||||
voice: process.env.DANKVOICE_VOICE?.trim() ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
/** Synthesize one utterance into a WAV inside `dir`; throws with the endpoint's own message. */
|
||||
async function synthesize(target: SpeechTarget, body: Record<string, string>, dir: string, timeoutMs: number): Promise<string> {
|
||||
const res = await fetch(target.speechUrl, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
const contentType = res.headers.get("content-type") ?? "";
|
||||
if (!res.ok || contentType.includes("json")) {
|
||||
let message = `frank-tts HTTP ${res.status}`;
|
||||
try {
|
||||
const payload: unknown = await res.json();
|
||||
if (
|
||||
typeof payload === "object" &&
|
||||
payload !== null &&
|
||||
"error" in payload &&
|
||||
typeof payload.error === "object" &&
|
||||
payload.error !== null &&
|
||||
"message" in payload.error &&
|
||||
typeof payload.error.message === "string"
|
||||
) {
|
||||
message = payload.error.message;
|
||||
}
|
||||
} catch {
|
||||
// keep the status-line message
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
const bytes = new Uint8Array(await res.arrayBuffer());
|
||||
if (bytes.byteLength === 0) throw new Error("frank-tts returned no audio");
|
||||
const file = join(dir, `say-${Date.now()}.wav`);
|
||||
writeFileSync(file, bytes);
|
||||
return file;
|
||||
}
|
||||
|
||||
/** frank-tts's roster route first; if it ever answers nothing, ask with a voice it must reject. */
|
||||
async function fetchVoices(target: SpeechTarget): Promise<string[]> {
|
||||
const roster = await fetch(target.voicesUrl, { signal: AbortSignal.timeout(VOICES_TIMEOUT_MS) })
|
||||
.then((res) => res.json())
|
||||
.catch(() => undefined);
|
||||
const listed = parseVoices(roster);
|
||||
if (listed.length > 0) return listed;
|
||||
const probe = await fetch(target.speechUrl, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(speechBody(target.model, "voice list probe", VOICE_PROBE)),
|
||||
signal: AbortSignal.timeout(VOICES_TIMEOUT_MS),
|
||||
}).catch(() => undefined);
|
||||
if (!probe) return [];
|
||||
return parseVoices(await probe.json().catch(() => undefined));
|
||||
}
|
||||
|
||||
/** Play a WAV and clean its temp directory up when the player exits. */
|
||||
function startPlayback(player: Player, file: string, dir: string): ChildProcess {
|
||||
const child = spawn(player.command, [...player.args, file], { stdio: "ignore" });
|
||||
const clean = () => {
|
||||
try {
|
||||
rmSync(file, { force: true });
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// temp files are not worth a toast
|
||||
}
|
||||
};
|
||||
child.once("exit", clean);
|
||||
child.once("error", clean);
|
||||
return child;
|
||||
}
|
||||
|
||||
export default function dankvoice(pi: ExtensionAPI) {
|
||||
const cfg = loadConfig();
|
||||
const state: { auto: boolean; voice: string; child?: ChildProcess; lastCtx?: ExtensionContext } = {
|
||||
auto: cfg.auto,
|
||||
voice: cfg.voice,
|
||||
};
|
||||
const voiceLabel = () => state.voice || "server default";
|
||||
const reflectStatus = () => state.lastCtx?.ui.setStatus("dankvoice", state.auto ? `🔊 ${voiceLabel()}` : undefined);
|
||||
|
||||
const stop = () => {
|
||||
if (state.child && state.child.exitCode === null) state.child.kill("SIGTERM");
|
||||
state.child = undefined;
|
||||
};
|
||||
|
||||
const missingTarget = () => "dankvoice: no `api: openai-speech` provider in agent/models.yml (set DANKVOICE_URL)";
|
||||
|
||||
async function speak(ctx: ExtensionContext, text: string) {
|
||||
const spoken = stripForSpeech(text, cfg.maxChars);
|
||||
if (!spoken) return;
|
||||
if (!cfg.target) {
|
||||
ctx.ui.notify(missingTarget(), "error");
|
||||
return;
|
||||
}
|
||||
if (!cfg.player) {
|
||||
ctx.ui.notify("dankvoice: no audio player found (pw-play, paplay or ffplay)", "error");
|
||||
return;
|
||||
}
|
||||
stop();
|
||||
const dir = mkdtempSync(join(tmpdir(), "dankvoice-"));
|
||||
try {
|
||||
const file = await synthesize(cfg.target, speechBody(cfg.target.model, spoken, state.voice), dir, SYNTH_TIMEOUT_MS);
|
||||
state.child = startPlayback(cfg.player, file, dir);
|
||||
} catch (error) {
|
||||
stop();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
ctx.ui.notify(`dankvoice: ${error instanceof Error ? error.message : String(error)}`, "error");
|
||||
}
|
||||
}
|
||||
|
||||
pi.registerCommand("say", {
|
||||
description:
|
||||
"Speak with the fleet's cloned voices (frank-tts): /say <text> | on | off | voice <name> | voices | stop | status",
|
||||
handler: async (args: string, ctx) => {
|
||||
state.lastCtx = ctx;
|
||||
const action = parseSayArgs(args);
|
||||
switch (action.kind) {
|
||||
case "help":
|
||||
ctx.ui.notify(
|
||||
"dankvoice: /say <text> · /say on|off (auto-speak answers) · /say voice <name> · /say voices · /say stop · /say status",
|
||||
"info",
|
||||
);
|
||||
return;
|
||||
case "on":
|
||||
state.auto = true;
|
||||
reflectStatus();
|
||||
ctx.ui.notify(`dankvoice: auto-speak armed (${voiceLabel()} @ ${cfg.target?.speechUrl ?? "no endpoint"})`, "info");
|
||||
return;
|
||||
case "off":
|
||||
state.auto = false;
|
||||
stop();
|
||||
reflectStatus();
|
||||
ctx.ui.notify("dankvoice: auto-speak off", "info");
|
||||
return;
|
||||
case "stop":
|
||||
stop();
|
||||
return;
|
||||
case "voice":
|
||||
state.voice = action.name;
|
||||
reflectStatus();
|
||||
ctx.ui.notify(
|
||||
action.name ? `dankvoice: voice = ${action.name}` : "dankvoice: voice = whatever frank-tts defaults to",
|
||||
"info",
|
||||
);
|
||||
return;
|
||||
case "voices": {
|
||||
if (!cfg.target) {
|
||||
ctx.ui.notify(missingTarget(), "error");
|
||||
return;
|
||||
}
|
||||
const voices: string[] = [];
|
||||
try {
|
||||
voices.push(...(await fetchVoices(cfg.target)));
|
||||
} catch (error) {
|
||||
ctx.ui.notify(`dankvoice: ${error instanceof Error ? error.message : String(error)}`, "error");
|
||||
return;
|
||||
}
|
||||
ctx.ui.notify(
|
||||
voices.length
|
||||
? `dankvoice voices: ${voices.join(", ")} — using ${voiceLabel()}`
|
||||
: "dankvoice: frank-tts reported no voice list",
|
||||
voices.length ? "info" : "warning",
|
||||
);
|
||||
return;
|
||||
}
|
||||
case "status":
|
||||
ctx.ui.notify(
|
||||
`dankvoice: voice ${voiceLabel()} · ${cfg.target?.speechUrl ?? "NO ENDPOINT"} · player ${cfg.player ? cfg.player.command : "NONE"} · auto-speak ${state.auto ? "on" : "off"} · cap ${cfg.maxChars} chars`,
|
||||
"info",
|
||||
);
|
||||
return;
|
||||
case "speak":
|
||||
await speak(ctx, action.text);
|
||||
return;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
pi.on("agent_end", async (event, ctx) => {
|
||||
state.lastCtx = ctx;
|
||||
if (!state.auto) return;
|
||||
// `willContinue` means the harness already scheduled a continuation
|
||||
// (auto-retry, empty-stop retry): this is not the user-visible terminal
|
||||
// settle, so narrating it would speak an answer the model is about to redo.
|
||||
if (event.willContinue) return;
|
||||
if (ctx.mode === "print" || ctx.mode === "json") return;
|
||||
await speak(ctx, lastAssistantText(event.messages));
|
||||
});
|
||||
|
||||
pi.on("session_shutdown", () => {
|
||||
stop();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
---
|
||||
type: reference
|
||||
title: dankvoice
|
||||
description: omp extension that speaks assistant output with the fleet's cloned voices through frank-tts.
|
||||
tags:
|
||||
- omp
|
||||
- extension
|
||||
- tts
|
||||
- voice
|
||||
- frank-tts
|
||||
- dankclaw
|
||||
verified: 2026-10-18
|
||||
timestamp: 2026-10-18
|
||||
---
|
||||
|
||||
# dankvoice
|
||||
|
||||
`agent/extensions/dankvoice.ts` adds cloned-voice speech to the interactive
|
||||
harness: `/say <text>` speaks through the speakers and `/say on` narrates the
|
||||
assistant's final answer after every turn, both served by
|
||||
**frank-tts** (Dankclaw :7772, the `speech` model role in
|
||||
[agent/models.yml](/core/index.md)).
|
||||
|
||||
## Why it exists
|
||||
|
||||
Nothing built into the harness can pronounce Frank's cloned voices.
|
||||
|
||||
| Surface | Engine | Verdict |
|
||||
|---|---|---|
|
||||
| TUI speak-aloud (`speech.enabled`, `speech.voice`) | bundled Kokoro ONNX worker, `synthesizeStream(voice, speed, signal)` | Kokoro voice enum only; never consults `models.yml` or the speech role |
|
||||
| `omp say [TEXT]` | same local backend (`--model` = *local* model key) | `omp say --voice narrator1 "test"` → `error: Expected --voice to be one of: af_heart, af_bella, af_nicole, af_kore, af_sarah, am_michael, am_fenrir, am_puck, bf_emma, bm_george, bm_fable` (verified 2026-10-18) |
|
||||
| `tts` tool / `xd://tts` device (`modelRoles.speech`) | frank-tts over OpenAI `/v1/audio/speech` | cloned voices work, but only as a **file** — nothing plays them |
|
||||
|
||||
`dankvoice` is not a patch to that vocalizer (harness code, not ours); it is a
|
||||
second transport that happens to speak, so Kokoro stays available for quick
|
||||
local speech and the cloned voices carry the real narration.
|
||||
|
||||
## Commands
|
||||
|
||||
| Command | Effect |
|
||||
|---|---|
|
||||
| `/say <text>` | synthesize once and play; interrupts whatever is playing |
|
||||
| `/say on` \| `/say off` | arm/disarm auto-speak of the assistant's final text at `agent_end`; the footer shows `🔊 <voice>` while armed |
|
||||
| `/say voice <name>` | pick a voice; an empty name defers to the server's own default |
|
||||
| `/say voices` | list what frank-tts reports |
|
||||
| `/say stop` | cut the current utterance |
|
||||
| `/say status` \| `/say help` | endpoint, voice, player, cap |
|
||||
|
||||
Auto-speak is skipped in `print`/`json` mode so benches and pipelines stay
|
||||
silent. Markdown is stripped before synthesis — fenced code is dropped, links
|
||||
collapse to their label, bullets and table pipes disappear — and the utterance
|
||||
is truncated at a sentence boundary inside the cap.
|
||||
|
||||
## Endpoint resolution
|
||||
|
||||
The base URL is **read from `agent/models.yml`**, from the provider block whose
|
||||
model row carries `api: openai-speech` — the same row the `speech` role
|
||||
resolves, so no host address and no credential appears in this file. From
|
||||
`baseUrl: http://192.168.0.143:7772/v1` it derives
|
||||
`POST …/v1/audio/speech`, `GET …/v1/audio/voices` and `GET …/health`.
|
||||
|
||||
| Environment | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `DANKVOICE_URL` | models.yml | speech base URL (`…/v1`), bypasses the YAML read |
|
||||
| `DANKVOICE_MODEL` | the id in that block (`frank-tts`) | model named in the request body |
|
||||
| `DANKVOICE_VOICE` | empty (server default) | default voice name |
|
||||
| `DANKVOICE_AUTO=1` | off | start with auto-speak armed |
|
||||
| `DANKVOICE_MAX` | `700` | spoken characters per utterance |
|
||||
| `DANKVOICE_PLAYER` | detect | player binary, skipping detection |
|
||||
| `DANKVOICE_MODELS` | `~/.omp/agent/models.yml` | YAML to read the endpoint from |
|
||||
|
||||
Player detection order is `pw-play` → `paplay` → `ffplay -nodisp -autoexit`
|
||||
(PipeWire first: the workstations run WirePlumber). Each utterance is written
|
||||
to `$TMPDIR/dankvoice-*/say-<ms>.wav` and the file plus its directory are
|
||||
removed when the player process exits.
|
||||
|
||||
## Verified facts (2026-10-18)
|
||||
|
||||
* **Extension discovery** — any `.ts` file in `~/.omp/agent/extensions/` loads
|
||||
at startup: a probe file there printed `loaded 2026-10-19T02:36:55Z` from a
|
||||
fresh `omp -p` run. `omp --help` documents `--no-extensions` (discovery off,
|
||||
explicit `-e/--extension` and `--hook` still load) and `plugin install|link`.
|
||||
* **Endpoint** — resolving the live `agent/models.yml` yields
|
||||
`speechUrl http://192.168.0.143:7772/v1/audio/speech`,
|
||||
`voicesUrl …/v1/audio/voices`, `healthUrl …/health`, `model frank-tts`.
|
||||
* **Roster** — `GET /v1/audio/voices` answers the bare JSON array
|
||||
`["narrator1","narrator2","narrator3"]`; `/health` answers
|
||||
`{"status":"ok","backend":"omni-http","voices":[…]}`. `GET /v1/voices` and
|
||||
`GET /v1/models` are **404** on this shim, and an unknown voice answers
|
||||
`400 unknown voice 'zzz'; available voices: …` with `available_voices` in the
|
||||
body — which is the fallback the extension uses if the roster route ever
|
||||
disappears.
|
||||
* **Omitting `voice` is legal** — frank-tts fills its `DEFAULT_VOICE` since the
|
||||
2026-10-18 voice-pack work, so an unset voice synthesizes instead of failing.
|
||||
* **Speech** — `/say "dankvoice live smoke test"` produced a 99,884 B / 2.08 s
|
||||
WAV and `/say voice narrator2` + `/say "cloned voice check"` a 69,164 B /
|
||||
1.44 s WAV (durations from `ffprobe`), each handed to the player; temp
|
||||
directories were gone afterwards.
|
||||
* **Auto-speak** — verified in a real TUI session: `/say on` armed the footer
|
||||
(`🔊 narrator1`), the next finished turn synthesized and played the answer
|
||||
(1.20 s WAV), and an unknown voice name answers with the roster instead of
|
||||
speaking. An `agent_end` carrying `willContinue` (the harness scheduled an
|
||||
auto-retry) is never narrated — only the terminal settle speaks.
|
||||
|
||||
## Limits
|
||||
|
||||
* The TUI's own speak-aloud feature still narrates in Kokoro: its voice list is
|
||||
a compile-time enum in the harness bundle, and no setting routes it to
|
||||
frank-tts. Cloned voices live only on this file-producing/speaking path and
|
||||
on the `tts` tool. See [Handy](/tools/handy.md) for the STT half.
|
||||
* `agent_end` narrates the final message text only — mid-turn tool chatter is
|
||||
never spoken.
|
||||
* frank-tts is LAN-only: no Caddy vhost fronts :7772, so this extension is
|
||||
usable on the home network.
|
||||
* Deployment, packs and the voice registry are owned by the
|
||||
`serverAppConfigs` repo (`serverAppConfigs-docs/Dankclaw/Frank TTS.md`), not
|
||||
by this one.
|
||||
|
||||
## Citations
|
||||
|
||||
* Live probes on the workstation, 2026-10-18: `omp say --voice narrator1 …`,
|
||||
`curl http://192.168.0.143:7772/health`, `/v1/audio/voices`,
|
||||
`/v1/audio/…/speech` (invalid voice), `ffprobe` on the produced WAVs.
|
||||
* `omp --help` (extension discovery flags, `say`, `plugin`) and
|
||||
`@earendil-works/pi-coding-agent` `dist/core/extensions/types.d.ts`
|
||||
(`ExtensionAPI.on/registerCommand`, `ExtensionMode = "tui" | "rpc" | "json" | "print"`).
|
||||
* Gitea issue #21 on `dank/command-center` (the work item this page ships with).
|
||||
|
||||
## See Also
|
||||
|
||||
* [Core Documentation](/core/index.md) — the live harness configuration.
|
||||
* [Handy](/tools/handy.md) — offline speech-to-text, the input half of voice.
|
||||
* `serverAppConfigs-docs/Dankclaw/Frank TTS.md` — the frank-tts service runbook.
|
||||
@@ -13,6 +13,7 @@ These files define the structural conventions that govern the agent system itsel
|
||||
* [AGENTS.md Format Specification](agents-md-format.md) — Canonical structure and conventions for the system-level instruction file that governs agent behavior.
|
||||
* [AGENTS.md Index](index-doc.md) — Overview of the cached tool documentation system and how to add new entries.
|
||||
* [Pi Settings System](settings.md) — pi-era `settings.json` record (archive): providers, models, packages, subagent overrides.
|
||||
* [dankvoice](dankvoice.md) — Cloned-voice vocalization extension: `/say` and auto-speak over frank-tts, endpoint read from `agent/models.yml`.
|
||||
|
||||
## Change Tracking
|
||||
|
||||
|
||||
@@ -11,6 +11,10 @@ timestamp: 2026-08-28
|
||||
|
||||
# Bundle Change Log
|
||||
|
||||
## 2026-10-18 — dankvoice: the cloned voices finally speak in the TUI (Refs #21)
|
||||
|
||||
Nothing in the harness could pronounce them: the TUI vocalizer and `omp say` gate `--voice` against the bundled Kokoro enum (`omp say --voice narrator1 "test"` → `error: Expected --voice to be one of: af_heart, …`), so frank-tts was reachable only as a file. Added `agent/extensions/dankvoice.ts` — `/say <text>`, `/say on|off` (narrate the final answer at `agent_end`), `/say voice|voices|stop|status` — synthesizing over the OpenAI speech route and playing through `pw-play`/`paplay`/`ffplay`, with the endpoint read from the `api: openai-speech` row of `agent/models.yml` (no host, no secret in the file). Verified live in a real TUI session on the workstation: roster `GET /v1/audio/voices` → `narrator1, narrator2, narrator3`; `/say` produced 99,884 B / 2.08 s and 69,164 B / 1.44 s WAVs handed to the player, temp dirs cleaned afterwards; `/say on` armed the footer (`🔊 narrator1`) and each following finished turn spoke its answer (1.20 s WAV), an unknown voice name answered with the roster instead of speaking, and an `agent_end` carrying `willContinue` (the harness scheduled an auto-retry) is deliberately never narrated. Extension discovery itself proved with a probe file in `~/.omp/agent/extensions/` loaded by a fresh `omp -p` run. Reference page [dankvoice](core/dankvoice.md), registered in the core index; black-box seam `tests/dankvoice.bats` (12 tests).
|
||||
|
||||
## 2026-10-18 — Noctalia bar layout clobbered by trunk-sync stash, recovered from dangling objects
|
||||
|
||||
Frank's months of GUI-built bar customizations (cat/clock/clock-utc/weather start, taskbar center, capsule groups g1–g4, control-cheat-sheet, thickness/font-scale/opacity styling) vanished from the live bar at 16:46. Root cause: the layout existed **only in the working tree** — the repo copy of `settings.toml` carried a stale plain layout because GUI edits hadn't been committed inside the hourly heartbeat window — and a trunk-sync session clearing the dirty tree for `origin/main` stashed it; the stash entry was subsequently destroyed (three `WIP on main` stashes within 60 s, all later dropped). Recovery: `git fsck --lost-found` surfaced the destroyed stash as dangling commit `7fe3ed8` (16:46:09); candidate validated in a sandboxed `XDG_*_HOME` before restore, clobbered file preserved to `.AGENTTRASH/`, layout restored through the symlink target and proven by screenshot, committed as `c87b378` — the durable copy the layout never had. Lessons codified into managed skills: trunk-sync must preservation-**commit** every dirty deployment path (never stash, never `stash clear`), and GUI layout edits get committed the same session they're made.
|
||||
|
||||
Executable
+263
@@ -0,0 +1,263 @@
|
||||
#!/usr/bin/env bats
|
||||
# tests/dankvoice.bats — black-box tests for agent/extensions/dankvoice.ts
|
||||
#
|
||||
# The extension's contract, from outside: it registers a `say` command and an
|
||||
# `agent_end` listener; what is observable is (a) the JSON it POSTs to the
|
||||
# speech route, (b) the file it hands to the player, and (c) the notification
|
||||
# text it prints. A fixture HTTP server stands in for frank-tts (it answers the
|
||||
# speech route with a fixed WAV, the roster route with a JSON array, and
|
||||
# everything else with 404) and a stub player stands in for pw-play: it records
|
||||
# the path it was given and copies the bytes it was told to play.
|
||||
#
|
||||
# Driven by env: DV_EXT (extension under test), DV_STUB / DV_STUB_LOG / DV_COPY
|
||||
# (player fixture), DV_MODELS (models.yml fixture), DV_EMPTY_BIN (a PATH with no
|
||||
# players), DV_MODE (the harness mode reported to the extension).
|
||||
|
||||
setup() {
|
||||
REPO="$(cd "$BATS_TEST_DIRNAME/.." && pwd)"
|
||||
WORK="$(mktemp -d)"
|
||||
|
||||
export DV_EXT="$REPO/agent/extensions/dankvoice.ts"
|
||||
export DV_STUB="$WORK/player.sh"
|
||||
export DV_STUB_LOG="$WORK/player.log"
|
||||
export DV_COPY="$WORK/played.wav"
|
||||
export DV_EMPTY_BIN="$WORK/emptybin"
|
||||
mkdir -p "$DV_EMPTY_BIN"
|
||||
|
||||
cat > "$DV_STUB" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "$1" >> "$DV_STUB_LOG"
|
||||
cp "$1" "$DV_COPY"
|
||||
sleep 0.1
|
||||
SH
|
||||
chmod +x "$DV_STUB"
|
||||
|
||||
# models.yml fixture: a chat provider first, the speech provider second
|
||||
export DV_MODELS="$WORK/models.yml"
|
||||
cat > "$DV_MODELS" <<'YML'
|
||||
providers:
|
||||
"chat (box)":
|
||||
baseUrl: http://10.0.0.1:9999/v1
|
||||
models:
|
||||
- id: chatmodel
|
||||
api: openai-completions
|
||||
"speech (box)":
|
||||
baseUrl: http://127.0.0.1:1/v1
|
||||
auth: none
|
||||
models:
|
||||
- id: speechmodel
|
||||
api: openai-speech
|
||||
YML
|
||||
|
||||
cat > "$WORK/driver.ts" <<'TS'
|
||||
// Black-box driver: stand up the frank-tts fixture, load the extension against
|
||||
// it, invoke one scenario, print what the endpoint received and what got played.
|
||||
import { createServer } from "node:http";
|
||||
import { existsSync, readFileSync, statSync } from "node:fs";
|
||||
|
||||
const scenario = process.argv[2] ?? "speak";
|
||||
const AUDIO = Buffer.from([0x52, 0x49, 0x46, 0x46, 8, 0, 0, 0, 0x57, 0x41, 0x56, 0x45, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]);
|
||||
const received: Array<{ url: string | undefined; body: string }> = [];
|
||||
|
||||
const server = createServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (chunk: Buffer) => chunks.push(chunk));
|
||||
req.on("end", () => {
|
||||
const body = Buffer.concat(chunks).toString("utf8");
|
||||
received.push({ url: req.url, body });
|
||||
if (req.url?.endsWith("/audio/voices")) {
|
||||
res.setHeader("content-type", "application/json");
|
||||
res.end(JSON.stringify(["narrator1", "narrator2"]));
|
||||
return;
|
||||
}
|
||||
if (req.url?.endsWith("/audio/speech")) {
|
||||
res.setHeader("content-type", "audio/wav");
|
||||
res.end(AUDIO);
|
||||
return;
|
||||
}
|
||||
res.statusCode = 404;
|
||||
res.end("{}");
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const port = (server.address() as { port: number }).port;
|
||||
|
||||
if (scenario === "endpoint") {
|
||||
process.env.DANKVOICE_MODELS = process.env.DV_MODELS;
|
||||
} else {
|
||||
process.env.DANKVOICE_URL = `http://127.0.0.1:${port}/v1`;
|
||||
}
|
||||
if (scenario === "noplayer") process.env.PATH = process.env.DV_EMPTY_BIN ?? "/nonexistent-bin";
|
||||
process.env.DANKVOICE_PLAYER = scenario === "noplayer" ? "" : process.env.DV_STUB;
|
||||
|
||||
const extension = await import(process.env.DV_EXT as string);
|
||||
const commands = new Map<string, { handler: (args: string, ctx: unknown) => Promise<void> }>();
|
||||
const events = new Map<string, (event: unknown, ctx: unknown) => Promise<void> | void>();
|
||||
extension.default({
|
||||
registerCommand: (name: string, options: { handler: (args: string, ctx: unknown) => Promise<void> }) => commands.set(name, options),
|
||||
on: (event: string, handler: (event: unknown, ctx: unknown) => Promise<void> | void) => events.set(event, handler),
|
||||
});
|
||||
const ctx = {
|
||||
mode: process.env.DV_MODE ?? "tui",
|
||||
ui: {
|
||||
notify: (message: string, level?: string) => console.log(`NOTIFY ${level ?? "info"} ${message}`),
|
||||
setStatus: (key: string, text?: string) => console.log(`STATUS ${key} ${text ?? ""}`),
|
||||
},
|
||||
};
|
||||
const say = commands.get("say");
|
||||
const agentEnd = events.get("agent_end");
|
||||
const answer = (text: string) => ({ messages: [{ role: "assistant", content: [{ type: "text", text }] }] });
|
||||
const settle = () => new Promise((resolve) => setTimeout(resolve, 800));
|
||||
|
||||
if (!say) {
|
||||
console.log("COMMANDS none");
|
||||
} else {
|
||||
console.log(`COMMANDS ${[...commands.keys()].join(",")} EVENTS ${[...events.keys()].join(",")}`);
|
||||
switch (scenario) {
|
||||
case "speak":
|
||||
await say.handler("hello world", ctx);
|
||||
break;
|
||||
case "voice":
|
||||
await say.handler("voice narrator2", ctx);
|
||||
await say.handler("good morning", ctx);
|
||||
break;
|
||||
case "auto":
|
||||
await say.handler("on", ctx);
|
||||
await agentEnd?.(answer("the answer"), ctx);
|
||||
await settle();
|
||||
await say.handler("off", ctx);
|
||||
await agentEnd?.(answer("the second answer"), ctx);
|
||||
break;
|
||||
case "continuation":
|
||||
await say.handler("on", ctx);
|
||||
await agentEnd?.({ ...answer("not yet"), willContinue: true }, ctx);
|
||||
await settle();
|
||||
await agentEnd?.(answer("the settled answer"), ctx);
|
||||
break;
|
||||
case "printmode":
|
||||
await say.handler("on", ctx);
|
||||
await agentEnd?.(answer("quiet"), ctx);
|
||||
break;
|
||||
case "voices":
|
||||
await say.handler("voices", ctx);
|
||||
break;
|
||||
case "endpoint":
|
||||
await say.handler("status", ctx);
|
||||
break;
|
||||
case "markdown":
|
||||
await say.handler("## Title\n\nUse `pw-play` here.\n\n```bash\nnpm i x\n```\n\nsee [docs](http://example.test) and **bold**\n- a\n- b", ctx);
|
||||
break;
|
||||
case "truncate":
|
||||
await say.handler("Sentence one is here. ".repeat(40), ctx);
|
||||
break;
|
||||
case "noplayer":
|
||||
await say.handler("speak anyway", ctx);
|
||||
break;
|
||||
}
|
||||
await settle();
|
||||
}
|
||||
|
||||
for (const call of received) {
|
||||
const parsed: unknown = JSON.parse(call.body === "" ? "{}" : call.body);
|
||||
const field = (key: string): string => {
|
||||
if (parsed && typeof parsed === "object" && key in parsed) {
|
||||
const value = parsed[key];
|
||||
if (typeof value === "string") return value;
|
||||
}
|
||||
return "-";
|
||||
};
|
||||
console.log(`CALL ${call.url} model=${field("model")} format=${field("response_format")} voice=${field("voice")} input=${field("input")}`);
|
||||
}
|
||||
console.log(`CALLS ${received.length}`);
|
||||
const log = process.env.DV_STUB_LOG && existsSync(process.env.DV_STUB_LOG) ? readFileSync(process.env.DV_STUB_LOG, "utf8").trim() : "";
|
||||
console.log(`PLAYS ${log === "" ? 0 : log.split("\n").length}`);
|
||||
if (process.env.DV_COPY && existsSync(process.env.DV_COPY)) console.log(`COPIED ${statSync(process.env.DV_COPY).size}`);
|
||||
server.close();
|
||||
TS
|
||||
|
||||
export DV_DRIVER="$WORK/driver.ts"
|
||||
}
|
||||
|
||||
teardown() {
|
||||
rm -rf "$WORK"
|
||||
}
|
||||
|
||||
say_run() {
|
||||
run bun "$DV_DRIVER" "$1"
|
||||
}
|
||||
|
||||
@test "the extension registers the say command and an agent_end listener" {
|
||||
say_run speak
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"COMMANDS say EVENTS agent_end"* ]]
|
||||
}
|
||||
|
||||
@test "speaking posts the text to the speech route" {
|
||||
say_run speak
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"CALL /v1/audio/speech model=frank-tts format=wav voice=- input=hello world"* ]]
|
||||
}
|
||||
|
||||
@test "one utterance is played once, as exactly the bytes the endpoint sent" {
|
||||
say_run speak
|
||||
[[ "$output" == *"PLAYS 1"* ]]
|
||||
[[ "$output" == *"COPIED 24"* ]]
|
||||
}
|
||||
|
||||
@test "the voice is named in the request only when the user chose one" {
|
||||
say_run speak
|
||||
[[ "$output" == *"voice=- input=hello world"* ]]
|
||||
say_run voice
|
||||
[[ "$output" == *"voice=narrator2 input=good morning"* ]]
|
||||
[[ "$output" == *"NOTIFY info dankvoice: voice = narrator2"* ]]
|
||||
}
|
||||
|
||||
@test "auto-speak narrates the final answer once and stays silent after it is disarmed" {
|
||||
say_run auto
|
||||
[[ "$output" == *"PLAYS 1"* ]]
|
||||
[[ "$output" == *"input=the answer"* ]]
|
||||
[[ "$output" != *"the second answer"* ]]
|
||||
[[ "$output" == *"STATUS dankvoice 🔊"* ]]
|
||||
}
|
||||
|
||||
@test "auto-speak stays silent outside the interactive modes" {
|
||||
DV_MODE=print say_run printmode
|
||||
[[ "$output" == *"PLAYS 0"* ]]
|
||||
[[ "$output" != *"input=quiet"* ]]
|
||||
}
|
||||
|
||||
@test "the voice list comes from the endpoint roster route" {
|
||||
say_run voices
|
||||
[[ "$output" == *"CALL /v1/audio/voices"* ]]
|
||||
[[ "$output" == *"dankvoice voices: narrator1, narrator2"* ]]
|
||||
}
|
||||
|
||||
@test "the endpoint is taken from the speech row of the models file, never the chat provider" {
|
||||
say_run endpoint
|
||||
[[ "$output" == *"http://127.0.0.1:1/v1/audio/speech"* ]]
|
||||
[[ "$output" != *"10.0.0.1"* ]]
|
||||
}
|
||||
|
||||
@test "markdown never reaches the speech engine" {
|
||||
say_run markdown
|
||||
[[ "$output" == *"input=Title Use pw-play here. see docs and bold a b"* ]]
|
||||
}
|
||||
|
||||
@test "an over-long utterance is bounded and cut at a sentence boundary" {
|
||||
DANKVOICE_MAX=60 say_run truncate
|
||||
[[ "$output" == *"here. …"* ]]
|
||||
[ "$(grep -o 'Sentence one is here\.' <<<"$output" | wc -l)" -le 4 ]
|
||||
}
|
||||
|
||||
@test "a missing player is reported, not swallowed" {
|
||||
say_run noplayer
|
||||
[[ "$output" == *"NOTIFY error dankvoice: no audio player found"* ]]
|
||||
[[ "$output" == *"PLAYS 0"* ]]
|
||||
}
|
||||
|
||||
@test "a scheduled continuation is never narrated as the final answer" {
|
||||
say_run continuation
|
||||
[[ "$output" == *"PLAYS 1"* ]]
|
||||
[[ "$output" == *"input=the settled answer"* ]]
|
||||
[[ "$output" != *"input=not yet"* ]]
|
||||
}
|
||||
Reference in New Issue
Block a user