/** * dankvoice — cloned-voice vocalization for omp, served by frank-tts. * * The built-in speak-aloud feature (`speech.enabled` / `speech.voice`) and the * `omp say` CLI are gated to the bundled Kokoro voice enum, so the cloned * voices on frank-tts (Dankclaw :7772, the `speech` model role in * agent/models.yml) could only ever produce files. This extension adds the * missing path — the TUI speaks with the cloned voices: * * /say speak once through the speakers * /say on | off auto-speak the assistant's final answer each turn * /say voice choose a voice (empty = whatever the server defaults to) * /say voices list the voices frank-tts reports * /say stop cut off the current utterance * /say status voice, endpoint and player in use * * The endpoint comes from agent/models.yml — the provider block carrying * `api: openai-speech`, i.e. the same row the speech role resolves — so this * file carries no host address and no secret. Environment overrides: * * DANKVOICE_URL speech base URL (…/v1) instead of models.yml * DANKVOICE_MODEL served model id (default: the id in that block) * DANKVOICE_VOICE default voice name * DANKVOICE_AUTO=1 start with auto-speak armed * DANKVOICE_MAX spoken-character cap per utterance (default 700) * DANKVOICE_PLAYER player binary, skipping detection * DANKVOICE_MODELS path to models.yml (default ~/.omp/agent/models.yml) * * Utterances are synthesized to a temp WAV in $TMPDIR; the file is removed * when its player process exits. A new utterance interrupts the previous one * instead of queueing behind it. */ import { spawn, spawnSync, type ChildProcess } from "node:child_process"; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; const DEFAULT_MAX_CHARS = 700; const SYNTH_TIMEOUT_MS = 60_000; const VOICES_TIMEOUT_MS = 15_000; /** A voice name the shim must reject; its 400 body carries the real list. */ const VOICE_PROBE = "_dankvoice_probe_"; // ---------------------------------------------------------------- pure logic export interface SpeechTarget { /** POST target for OpenAI speech synthesis. */ speechUrl: string; /** GET target for the voice roster (frank-tts answers a bare JSON array). */ voicesUrl: string; /** GET target for the liveness route of the same host:port. */ healthUrl: string; /** Served model id to name in the request body. */ model: string; } export interface Player { command: string; args: string[]; } export type SayAction = | { kind: "help" } | { kind: "speak"; text: string } | { kind: "on" } | { kind: "off" } | { kind: "stop" } | { kind: "voices" } | { kind: "status" } | { kind: "voice"; name: string }; /** `/say ` grammar: subcommands win, anything else is spoken text. */ export function parseSayArgs(raw: string): SayAction { const args = raw.trim(); const lower = args.toLowerCase(); if (lower === "" || lower === "help") return { kind: "help" }; if (lower === "on") return { kind: "on" }; if (lower === "off") return { kind: "off" }; if (lower === "stop") return { kind: "stop" }; if (lower === "voices") return { kind: "voices" }; if (lower === "status") return { kind: "status" }; if (lower === "voice" || lower.startsWith("voice ")) return { kind: "voice", name: args.slice(5).trim() }; return { kind: "speak", text: args }; } /** * Pull the speech endpoint out of agent/models.yml. The runner API lives on * the MODEL row (`api: openai-speech`), the base URL on its provider, so this * scan remembers the last provider base URL and model id seen and returns when * a model row claims the speech runner API. A new provider key (indent two) * resets both. */ export function resolveSpeechEndpoint(modelsYml: string): SpeechTarget | undefined { let base: string | undefined; let id: string | undefined; for (const line of modelsYml.split(/\r?\n/)) { const trimmed = line.trim(); if (!trimmed || trimmed.startsWith("#")) continue; if (/^ {2}\S.*:\s*$/.test(line)) { base = undefined; id = undefined; continue; } const providerUrl = trimmed.match(/^baseUrl:\s*(\S+)/); if (providerUrl) { base = providerUrl[1].replace(/\/+$/, ""); continue; } const modelId = trimmed.match(/^-\s*id:\s*(\S+)/); if (modelId) { id = modelId[1]; continue; } if (/^api:\s*openai-speech\s*$/.test(trimmed) && base) { return { speechUrl: `${base}/audio/speech`, voicesUrl: `${base}/audio/voices`, healthUrl: `${base.replace(/\/v1$/, "")}/health`, model: id ?? "frank-tts", }; } } return undefined; } /** OpenAI speech body; `voice` is sent only when named — the shim carries its own default. */ export function speechBody(model: string, text: string, voice: string): Record { const body: Record = { model, input: text, response_format: "wav" }; if (voice) body.voice = voice; return body; } /** Voice names as frank-tts reports them: the invalid-voice 400 body carries `available_voices`. */ export function parseVoices(payload: unknown): string[] { // frank-tts answers GET /v1/audio/voices with a bare array and an unknown voice with // {error:{available_voices:[…]}}; a flat body is accepted too. if (typeof payload !== "object" || payload === null) return []; let list: unknown; if (Array.isArray(payload)) { list = payload; } else if ("error" in payload && typeof payload.error === "object" && payload.error !== null && "available_voices" in payload.error) { list = payload.error.available_voices; } else if ("available_voices" in payload) { list = payload.available_voices; } else if ("voices" in payload) { list = payload.voices; } else if ("data" in payload) { list = payload.data; } if (!Array.isArray(list)) return []; const names: string[] = []; for (const entry of list) { if (typeof entry === "string") { names.push(entry); } else if (typeof entry === "object" && entry !== null) { const name = "id" in entry ? entry.id : "name" in entry ? entry.name : undefined; if (typeof name === "string") names.push(name); } } return names.filter((name) => name.length > 0); } /** Player preference: PipeWire first (this fleet runs WirePlumber), then Pulse, then ffmpeg. */ export function detectPlayer(lookup: (command: string) => string | null): Player | null { const candidates: Player[] = [ { command: "pw-play", args: [] }, { command: "paplay", args: [] }, { command: "ffplay", args: ["-nodisp", "-autoexit", "-loglevel", "quiet"] }, ]; for (const candidate of candidates) { if (lookup(candidate.command)) return candidate; } return null; } /** What a speech engine should say: prose only, no fences, no markup, bounded. */ export function stripForSpeech(text: string, maxChars: number = DEFAULT_MAX_CHARS): string { let out = text; out = out.replace(/```[\s\S]*?```/g, " "); out = out.replace(/~~~[\s\S]*?~~~/g, " "); out = out.replace(/`([^`]*)`/g, "$1"); out = out.replace(/!\[[^\]]*\]\([^)]*\)/g, " "); out = out.replace(/\[([^\]]*)\]\([^)]*\)/g, "$1"); out = out.replace(/^\s{0,3}#{1,6}\s+/gm, ""); out = out.replace(/^\s*[-*+]\s+/gm, ""); out = out.replace(/^\s*\|[-: |]+\|\s*$/gm, " "); out = out.replace(/\|/g, " "); out = out.replace(/[*_]{1,3}([^*_]+)[*_]{1,3}/g, "$1"); out = out.replace(/\s+/g, " ").trim(); if (out.length <= maxChars) return out; const window = out.slice(0, maxChars); const cut = Math.max(window.lastIndexOf(". "), window.lastIndexOf("! "), window.lastIndexOf("? ")); const clipped = cut > maxChars / 2 ? window.slice(0, cut + 1) : window.replace(/[,;:]\s*\S*$/, ""); return `${clipped.trim()} …`; } /** The text the assistant last actually said; tool-only messages do not count. */ export function lastAssistantText(messages: readonly unknown[]): string { for (let i = messages.length - 1; i >= 0; i--) { const message = messages[i]; if (!message || typeof message !== "object" || !("role" in message) || message.role !== "assistant") continue; if (!("content" in message)) continue; const content = message.content; if (typeof content === "string") return content; if (!Array.isArray(content)) continue; const parts: string[] = []; for (const part of content) { if (part && typeof part === "object" && "type" in part && part.type === "text" && "text" in part && typeof part.text === "string") { parts.push(part.text); } } const joined = parts.join("\n").trim(); if (joined) return joined; } return ""; } // ------------------------------------------------------------- live wiring interface Config { target?: SpeechTarget; player?: Player; maxChars: number; auto: boolean; voice: string; } function lookupOnPath(command: string): string | null { const found = spawnSync("sh", ["-c", `command -v ${command}`], { encoding: "utf8" }); return found.status === 0 && found.stdout.trim() ? found.stdout.trim() : null; } function readModelsYml(): string { const path = process.env.DANKVOICE_MODELS ?? join(process.env.HOME ?? "", ".omp", "agent", "models.yml"); try { return readFileSync(path, "utf8"); } catch { return ""; } } function loadConfig(): Config { const overrideUrl = process.env.DANKVOICE_URL?.trim().replace(/\/+$/, ""); const modelOverride = process.env.DANKVOICE_MODEL?.trim(); const fromYml = overrideUrl ? undefined : resolveSpeechEndpoint(readModelsYml()); const target: SpeechTarget | undefined = overrideUrl ? { speechUrl: `${overrideUrl}/audio/speech`, voicesUrl: `${overrideUrl}/audio/voices`, healthUrl: `${overrideUrl.replace(/\/v1$/, "")}/health`, model: modelOverride || "frank-tts", } : fromYml ? { ...fromYml, model: modelOverride || fromYml.model } : undefined; const playerOverride = process.env.DANKVOICE_PLAYER?.trim(); const maxChars = Number(process.env.DANKVOICE_MAX); return { target, player: playerOverride ? { command: playerOverride, args: [] } : (detectPlayer(lookupOnPath) ?? undefined), maxChars: Number.isFinite(maxChars) && maxChars > 0 ? Math.floor(maxChars) : DEFAULT_MAX_CHARS, auto: process.env.DANKVOICE_AUTO === "1", voice: process.env.DANKVOICE_VOICE?.trim() ?? "", }; } /** Synthesize one utterance into a WAV inside `dir`; throws with the endpoint's own message. */ async function synthesize(target: SpeechTarget, body: Record, dir: string, timeoutMs: number): Promise { const res = await fetch(target.speechUrl, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body), signal: AbortSignal.timeout(timeoutMs), }); const contentType = res.headers.get("content-type") ?? ""; if (!res.ok || contentType.includes("json")) { let message = `frank-tts HTTP ${res.status}`; try { const payload: unknown = await res.json(); if ( typeof payload === "object" && payload !== null && "error" in payload && typeof payload.error === "object" && payload.error !== null && "message" in payload.error && typeof payload.error.message === "string" ) { message = payload.error.message; } } catch { // keep the status-line message } throw new Error(message); } const bytes = new Uint8Array(await res.arrayBuffer()); if (bytes.byteLength === 0) throw new Error("frank-tts returned no audio"); const file = join(dir, `say-${Date.now()}.wav`); writeFileSync(file, bytes); return file; } /** frank-tts's roster route first; if it ever answers nothing, ask with a voice it must reject. */ async function fetchVoices(target: SpeechTarget): Promise { const roster = await fetch(target.voicesUrl, { signal: AbortSignal.timeout(VOICES_TIMEOUT_MS) }) .then((res) => res.json()) .catch(() => undefined); const listed = parseVoices(roster); if (listed.length > 0) return listed; const probe = await fetch(target.speechUrl, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(speechBody(target.model, "voice list probe", VOICE_PROBE)), signal: AbortSignal.timeout(VOICES_TIMEOUT_MS), }).catch(() => undefined); if (!probe) return []; return parseVoices(await probe.json().catch(() => undefined)); } /** Play a WAV and clean its temp directory up when the player exits. */ function startPlayback(player: Player, file: string, dir: string): ChildProcess { const child = spawn(player.command, [...player.args, file], { stdio: "ignore" }); const clean = () => { try { rmSync(file, { force: true }); rmSync(dir, { recursive: true, force: true }); } catch { // temp files are not worth a toast } }; child.once("exit", clean); child.once("error", clean); return child; } export default function dankvoice(pi: ExtensionAPI) { const cfg = loadConfig(); const state: { auto: boolean; voice: string; child?: ChildProcess; lastCtx?: ExtensionContext } = { auto: cfg.auto, voice: cfg.voice, }; const voiceLabel = () => state.voice || "server default"; const reflectStatus = () => state.lastCtx?.ui.setStatus("dankvoice", state.auto ? `🔊 ${voiceLabel()}` : undefined); const stop = () => { if (state.child && state.child.exitCode === null) state.child.kill("SIGTERM"); state.child = undefined; }; const missingTarget = () => "dankvoice: no `api: openai-speech` provider in agent/models.yml (set DANKVOICE_URL)"; async function speak(ctx: ExtensionContext, text: string) { const spoken = stripForSpeech(text, cfg.maxChars); if (!spoken) return; if (!cfg.target) { ctx.ui.notify(missingTarget(), "error"); return; } if (!cfg.player) { ctx.ui.notify("dankvoice: no audio player found (pw-play, paplay or ffplay)", "error"); return; } stop(); const dir = mkdtempSync(join(tmpdir(), "dankvoice-")); try { const file = await synthesize(cfg.target, speechBody(cfg.target.model, spoken, state.voice), dir, SYNTH_TIMEOUT_MS); state.child = startPlayback(cfg.player, file, dir); } catch (error) { stop(); try { rmSync(dir, { recursive: true, force: true }); } catch { // ignore } ctx.ui.notify(`dankvoice: ${error instanceof Error ? error.message : String(error)}`, "error"); } } pi.registerCommand("say", { description: "Speak with the fleet's cloned voices (frank-tts): /say | on | off | voice | voices | stop | status", handler: async (args: string, ctx) => { state.lastCtx = ctx; const action = parseSayArgs(args); switch (action.kind) { case "help": ctx.ui.notify( "dankvoice: /say · /say on|off (auto-speak answers) · /say voice · /say voices · /say stop · /say status", "info", ); return; case "on": state.auto = true; reflectStatus(); ctx.ui.notify(`dankvoice: auto-speak armed (${voiceLabel()} @ ${cfg.target?.speechUrl ?? "no endpoint"})`, "info"); return; case "off": state.auto = false; stop(); reflectStatus(); ctx.ui.notify("dankvoice: auto-speak off", "info"); return; case "stop": stop(); return; case "voice": state.voice = action.name; reflectStatus(); ctx.ui.notify( action.name ? `dankvoice: voice = ${action.name}` : "dankvoice: voice = whatever frank-tts defaults to", "info", ); return; case "voices": { if (!cfg.target) { ctx.ui.notify(missingTarget(), "error"); return; } const voices: string[] = []; try { voices.push(...(await fetchVoices(cfg.target))); } catch (error) { ctx.ui.notify(`dankvoice: ${error instanceof Error ? error.message : String(error)}`, "error"); return; } ctx.ui.notify( voices.length ? `dankvoice voices: ${voices.join(", ")} — using ${voiceLabel()}` : "dankvoice: frank-tts reported no voice list", voices.length ? "info" : "warning", ); return; } case "status": ctx.ui.notify( `dankvoice: voice ${voiceLabel()} · ${cfg.target?.speechUrl ?? "NO ENDPOINT"} · player ${cfg.player ? cfg.player.command : "NONE"} · auto-speak ${state.auto ? "on" : "off"} · cap ${cfg.maxChars} chars`, "info", ); return; case "speak": await speak(ctx, action.text); return; } }, }); pi.on("agent_end", async (event, ctx) => { state.lastCtx = ctx; if (!state.auto) return; // `willContinue` means the harness already scheduled a continuation // (auto-retry, empty-stop retry): this is not the user-visible terminal // settle, so narrating it would speak an answer the model is about to redo. if (event.willContinue) return; if (ctx.mode === "print" || ctx.mode === "json") return; await speak(ctx, lastAssistantText(event.messages)); }); pi.on("session_shutdown", () => { stop(); }); }