feat(dankvoice): speak the cloned voices in the TUI (Refs #21)
Every speak-aloud path in the harness gates `--voice` against the bundled Kokoro enum, so the frank-tts cloned voices could only ever produce files. `agent/extensions/` auto-loads, so one tracked extension adds the missing transport: `/say <text>`, `/say on|off` (narrate the final answer at `agent_end`, silent when the harness schedules a continuation), `/say voice|voices|stop|status` — OpenAI speech POST plus playback through pw-play/paplay/ffplay. The endpoint and served id come from the `api: openai-speech` row of `agent/models.yml`, so the file carries no host and no secret; markdown is stripped, over-long utterances are cut at a sentence bound, and the temp WAV dir is always removed. Verified live in a real TUI session on the workstation (roster narrator1/narrator2/narrator3; 2.08 s and 1.44 s WAVs played; auto-speak spoke each finished answer, 1.20 s WAV; unknown voice answers with the roster). Black-box seam `tests/dankvoice.bats` (12 tests) drives the extension against a fixture speech endpoint and a stub player. cwd: /home/blank/.omp/wt/feat-21-dankvoice session: wt-20261017-181502-6d1fc1f
This commit is contained in:
Frank
@@ -0,0 +1,463 @@
|
||||
/**
|
||||
* dankvoice — cloned-voice vocalization for omp, served by frank-tts.
|
||||
*
|
||||
* The built-in speak-aloud feature (`speech.enabled` / `speech.voice`) and the
|
||||
* `omp say` CLI are gated to the bundled Kokoro voice enum, so the cloned
|
||||
* voices on frank-tts (Dankclaw :7772, the `speech` model role in
|
||||
* agent/models.yml) could only ever produce files. This extension adds the
|
||||
* missing path — the TUI speaks with the cloned voices:
|
||||
*
|
||||
* /say <text> speak once through the speakers
|
||||
* /say on | off auto-speak the assistant's final answer each turn
|
||||
* /say voice <name> choose a voice (empty = whatever the server defaults to)
|
||||
* /say voices list the voices frank-tts reports
|
||||
* /say stop cut off the current utterance
|
||||
* /say status voice, endpoint and player in use
|
||||
*
|
||||
* The endpoint comes from agent/models.yml — the provider block carrying
|
||||
* `api: openai-speech`, i.e. the same row the speech role resolves — so this
|
||||
* file carries no host address and no secret. Environment overrides:
|
||||
*
|
||||
* DANKVOICE_URL speech base URL (…/v1) instead of models.yml
|
||||
* DANKVOICE_MODEL served model id (default: the id in that block)
|
||||
* DANKVOICE_VOICE default voice name
|
||||
* DANKVOICE_AUTO=1 start with auto-speak armed
|
||||
* DANKVOICE_MAX spoken-character cap per utterance (default 700)
|
||||
* DANKVOICE_PLAYER player binary, skipping detection
|
||||
* DANKVOICE_MODELS path to models.yml (default ~/.omp/agent/models.yml)
|
||||
*
|
||||
* Utterances are synthesized to a temp WAV in $TMPDIR; the file is removed
|
||||
* when its player process exits. A new utterance interrupts the previous one
|
||||
* instead of queueing behind it.
|
||||
*/
|
||||
|
||||
import { spawn, spawnSync, type ChildProcess } from "node:child_process";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
||||
|
||||
const DEFAULT_MAX_CHARS = 700;
|
||||
const SYNTH_TIMEOUT_MS = 60_000;
|
||||
const VOICES_TIMEOUT_MS = 15_000;
|
||||
/** A voice name the shim must reject; its 400 body carries the real list. */
|
||||
const VOICE_PROBE = "_dankvoice_probe_";
|
||||
|
||||
// ---------------------------------------------------------------- pure logic
|
||||
|
||||
export interface SpeechTarget {
|
||||
/** POST target for OpenAI speech synthesis. */
|
||||
speechUrl: string;
|
||||
/** GET target for the voice roster (frank-tts answers a bare JSON array). */
|
||||
voicesUrl: string;
|
||||
/** GET target for the liveness route of the same host:port. */
|
||||
healthUrl: string;
|
||||
/** Served model id to name in the request body. */
|
||||
model: string;
|
||||
}
|
||||
|
||||
export interface Player {
|
||||
command: string;
|
||||
args: string[];
|
||||
}
|
||||
|
||||
export type SayAction =
|
||||
| { kind: "help" }
|
||||
| { kind: "speak"; text: string }
|
||||
| { kind: "on" }
|
||||
| { kind: "off" }
|
||||
| { kind: "stop" }
|
||||
| { kind: "voices" }
|
||||
| { kind: "status" }
|
||||
| { kind: "voice"; name: string };
|
||||
|
||||
/** `/say <args>` grammar: subcommands win, anything else is spoken text. */
|
||||
export function parseSayArgs(raw: string): SayAction {
|
||||
const args = raw.trim();
|
||||
const lower = args.toLowerCase();
|
||||
if (lower === "" || lower === "help") return { kind: "help" };
|
||||
if (lower === "on") return { kind: "on" };
|
||||
if (lower === "off") return { kind: "off" };
|
||||
if (lower === "stop") return { kind: "stop" };
|
||||
if (lower === "voices") return { kind: "voices" };
|
||||
if (lower === "status") return { kind: "status" };
|
||||
if (lower === "voice" || lower.startsWith("voice ")) return { kind: "voice", name: args.slice(5).trim() };
|
||||
return { kind: "speak", text: args };
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the speech endpoint out of agent/models.yml. The runner API lives on
|
||||
* the MODEL row (`api: openai-speech`), the base URL on its provider, so this
|
||||
* scan remembers the last provider base URL and model id seen and returns when
|
||||
* a model row claims the speech runner API. A new provider key (indent two)
|
||||
* resets both.
|
||||
*/
|
||||
export function resolveSpeechEndpoint(modelsYml: string): SpeechTarget | undefined {
|
||||
let base: string | undefined;
|
||||
let id: string | undefined;
|
||||
for (const line of modelsYml.split(/\r?\n/)) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith("#")) continue;
|
||||
if (/^ {2}\S.*:\s*$/.test(line)) {
|
||||
base = undefined;
|
||||
id = undefined;
|
||||
continue;
|
||||
}
|
||||
const providerUrl = trimmed.match(/^baseUrl:\s*(\S+)/);
|
||||
if (providerUrl) {
|
||||
base = providerUrl[1].replace(/\/+$/, "");
|
||||
continue;
|
||||
}
|
||||
const modelId = trimmed.match(/^-\s*id:\s*(\S+)/);
|
||||
if (modelId) {
|
||||
id = modelId[1];
|
||||
continue;
|
||||
}
|
||||
if (/^api:\s*openai-speech\s*$/.test(trimmed) && base) {
|
||||
return {
|
||||
speechUrl: `${base}/audio/speech`,
|
||||
voicesUrl: `${base}/audio/voices`,
|
||||
healthUrl: `${base.replace(/\/v1$/, "")}/health`,
|
||||
model: id ?? "frank-tts",
|
||||
};
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** OpenAI speech body; `voice` is sent only when named — the shim carries its own default. */
|
||||
export function speechBody(model: string, text: string, voice: string): Record<string, string> {
|
||||
const body: Record<string, string> = { model, input: text, response_format: "wav" };
|
||||
if (voice) body.voice = voice;
|
||||
return body;
|
||||
}
|
||||
|
||||
/** Voice names as frank-tts reports them: the invalid-voice 400 body carries `available_voices`. */
|
||||
export function parseVoices(payload: unknown): string[] {
|
||||
// frank-tts answers GET /v1/audio/voices with a bare array and an unknown voice with
|
||||
// {error:{available_voices:[…]}}; a flat body is accepted too.
|
||||
if (typeof payload !== "object" || payload === null) return [];
|
||||
let list: unknown;
|
||||
if (Array.isArray(payload)) {
|
||||
list = payload;
|
||||
} else if ("error" in payload && typeof payload.error === "object" && payload.error !== null && "available_voices" in payload.error) {
|
||||
list = payload.error.available_voices;
|
||||
} else if ("available_voices" in payload) {
|
||||
list = payload.available_voices;
|
||||
} else if ("voices" in payload) {
|
||||
list = payload.voices;
|
||||
} else if ("data" in payload) {
|
||||
list = payload.data;
|
||||
}
|
||||
if (!Array.isArray(list)) return [];
|
||||
const names: string[] = [];
|
||||
for (const entry of list) {
|
||||
if (typeof entry === "string") {
|
||||
names.push(entry);
|
||||
} else if (typeof entry === "object" && entry !== null) {
|
||||
const name = "id" in entry ? entry.id : "name" in entry ? entry.name : undefined;
|
||||
if (typeof name === "string") names.push(name);
|
||||
}
|
||||
}
|
||||
return names.filter((name) => name.length > 0);
|
||||
}
|
||||
|
||||
/** Player preference: PipeWire first (this fleet runs WirePlumber), then Pulse, then ffmpeg. */
|
||||
export function detectPlayer(lookup: (command: string) => string | null): Player | null {
|
||||
const candidates: Player[] = [
|
||||
{ command: "pw-play", args: [] },
|
||||
{ command: "paplay", args: [] },
|
||||
{ command: "ffplay", args: ["-nodisp", "-autoexit", "-loglevel", "quiet"] },
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
if (lookup(candidate.command)) return candidate;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** What a speech engine should say: prose only, no fences, no markup, bounded. */
|
||||
export function stripForSpeech(text: string, maxChars: number = DEFAULT_MAX_CHARS): string {
|
||||
let out = text;
|
||||
out = out.replace(/```[\s\S]*?```/g, " ");
|
||||
out = out.replace(/~~~[\s\S]*?~~~/g, " ");
|
||||
out = out.replace(/`([^`]*)`/g, "$1");
|
||||
out = out.replace(/!\[[^\]]*\]\([^)]*\)/g, " ");
|
||||
out = out.replace(/\[([^\]]*)\]\([^)]*\)/g, "$1");
|
||||
out = out.replace(/^\s{0,3}#{1,6}\s+/gm, "");
|
||||
out = out.replace(/^\s*[-*+]\s+/gm, "");
|
||||
out = out.replace(/^\s*\|[-: |]+\|\s*$/gm, " ");
|
||||
out = out.replace(/\|/g, " ");
|
||||
out = out.replace(/[*_]{1,3}([^*_]+)[*_]{1,3}/g, "$1");
|
||||
out = out.replace(/\s+/g, " ").trim();
|
||||
if (out.length <= maxChars) return out;
|
||||
const window = out.slice(0, maxChars);
|
||||
const cut = Math.max(window.lastIndexOf(". "), window.lastIndexOf("! "), window.lastIndexOf("? "));
|
||||
const clipped = cut > maxChars / 2 ? window.slice(0, cut + 1) : window.replace(/[,;:]\s*\S*$/, "");
|
||||
return `${clipped.trim()} …`;
|
||||
}
|
||||
|
||||
/** The text the assistant last actually said; tool-only messages do not count. */
|
||||
export function lastAssistantText(messages: readonly unknown[]): string {
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
const message = messages[i];
|
||||
if (!message || typeof message !== "object" || !("role" in message) || message.role !== "assistant") continue;
|
||||
if (!("content" in message)) continue;
|
||||
const content = message.content;
|
||||
if (typeof content === "string") return content;
|
||||
if (!Array.isArray(content)) continue;
|
||||
const parts: string[] = [];
|
||||
for (const part of content) {
|
||||
if (part && typeof part === "object" && "type" in part && part.type === "text" && "text" in part && typeof part.text === "string") {
|
||||
parts.push(part.text);
|
||||
}
|
||||
}
|
||||
const joined = parts.join("\n").trim();
|
||||
if (joined) return joined;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------- live wiring
|
||||
|
||||
interface Config {
|
||||
target?: SpeechTarget;
|
||||
player?: Player;
|
||||
maxChars: number;
|
||||
auto: boolean;
|
||||
voice: string;
|
||||
}
|
||||
|
||||
function lookupOnPath(command: string): string | null {
|
||||
const found = spawnSync("sh", ["-c", `command -v ${command}`], { encoding: "utf8" });
|
||||
return found.status === 0 && found.stdout.trim() ? found.stdout.trim() : null;
|
||||
}
|
||||
|
||||
function readModelsYml(): string {
|
||||
const path = process.env.DANKVOICE_MODELS ?? join(process.env.HOME ?? "", ".omp", "agent", "models.yml");
|
||||
try {
|
||||
return readFileSync(path, "utf8");
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function loadConfig(): Config {
|
||||
const overrideUrl = process.env.DANKVOICE_URL?.trim().replace(/\/+$/, "");
|
||||
const modelOverride = process.env.DANKVOICE_MODEL?.trim();
|
||||
const fromYml = overrideUrl ? undefined : resolveSpeechEndpoint(readModelsYml());
|
||||
const target: SpeechTarget | undefined = overrideUrl
|
||||
? {
|
||||
speechUrl: `${overrideUrl}/audio/speech`,
|
||||
voicesUrl: `${overrideUrl}/audio/voices`,
|
||||
healthUrl: `${overrideUrl.replace(/\/v1$/, "")}/health`,
|
||||
model: modelOverride || "frank-tts",
|
||||
}
|
||||
: fromYml
|
||||
? { ...fromYml, model: modelOverride || fromYml.model }
|
||||
: undefined;
|
||||
const playerOverride = process.env.DANKVOICE_PLAYER?.trim();
|
||||
const maxChars = Number(process.env.DANKVOICE_MAX);
|
||||
return {
|
||||
target,
|
||||
player: playerOverride ? { command: playerOverride, args: [] } : (detectPlayer(lookupOnPath) ?? undefined),
|
||||
maxChars: Number.isFinite(maxChars) && maxChars > 0 ? Math.floor(maxChars) : DEFAULT_MAX_CHARS,
|
||||
auto: process.env.DANKVOICE_AUTO === "1",
|
||||
voice: process.env.DANKVOICE_VOICE?.trim() ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
/** Synthesize one utterance into a WAV inside `dir`; throws with the endpoint's own message. */
|
||||
async function synthesize(target: SpeechTarget, body: Record<string, string>, dir: string, timeoutMs: number): Promise<string> {
|
||||
const res = await fetch(target.speechUrl, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
const contentType = res.headers.get("content-type") ?? "";
|
||||
if (!res.ok || contentType.includes("json")) {
|
||||
let message = `frank-tts HTTP ${res.status}`;
|
||||
try {
|
||||
const payload: unknown = await res.json();
|
||||
if (
|
||||
typeof payload === "object" &&
|
||||
payload !== null &&
|
||||
"error" in payload &&
|
||||
typeof payload.error === "object" &&
|
||||
payload.error !== null &&
|
||||
"message" in payload.error &&
|
||||
typeof payload.error.message === "string"
|
||||
) {
|
||||
message = payload.error.message;
|
||||
}
|
||||
} catch {
|
||||
// keep the status-line message
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
const bytes = new Uint8Array(await res.arrayBuffer());
|
||||
if (bytes.byteLength === 0) throw new Error("frank-tts returned no audio");
|
||||
const file = join(dir, `say-${Date.now()}.wav`);
|
||||
writeFileSync(file, bytes);
|
||||
return file;
|
||||
}
|
||||
|
||||
/** frank-tts's roster route first; if it ever answers nothing, ask with a voice it must reject. */
|
||||
async function fetchVoices(target: SpeechTarget): Promise<string[]> {
|
||||
const roster = await fetch(target.voicesUrl, { signal: AbortSignal.timeout(VOICES_TIMEOUT_MS) })
|
||||
.then((res) => res.json())
|
||||
.catch(() => undefined);
|
||||
const listed = parseVoices(roster);
|
||||
if (listed.length > 0) return listed;
|
||||
const probe = await fetch(target.speechUrl, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(speechBody(target.model, "voice list probe", VOICE_PROBE)),
|
||||
signal: AbortSignal.timeout(VOICES_TIMEOUT_MS),
|
||||
}).catch(() => undefined);
|
||||
if (!probe) return [];
|
||||
return parseVoices(await probe.json().catch(() => undefined));
|
||||
}
|
||||
|
||||
/** Play a WAV and clean its temp directory up when the player exits. */
|
||||
function startPlayback(player: Player, file: string, dir: string): ChildProcess {
|
||||
const child = spawn(player.command, [...player.args, file], { stdio: "ignore" });
|
||||
const clean = () => {
|
||||
try {
|
||||
rmSync(file, { force: true });
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// temp files are not worth a toast
|
||||
}
|
||||
};
|
||||
child.once("exit", clean);
|
||||
child.once("error", clean);
|
||||
return child;
|
||||
}
|
||||
|
||||
export default function dankvoice(pi: ExtensionAPI) {
|
||||
const cfg = loadConfig();
|
||||
const state: { auto: boolean; voice: string; child?: ChildProcess; lastCtx?: ExtensionContext } = {
|
||||
auto: cfg.auto,
|
||||
voice: cfg.voice,
|
||||
};
|
||||
const voiceLabel = () => state.voice || "server default";
|
||||
const reflectStatus = () => state.lastCtx?.ui.setStatus("dankvoice", state.auto ? `🔊 ${voiceLabel()}` : undefined);
|
||||
|
||||
const stop = () => {
|
||||
if (state.child && state.child.exitCode === null) state.child.kill("SIGTERM");
|
||||
state.child = undefined;
|
||||
};
|
||||
|
||||
const missingTarget = () => "dankvoice: no `api: openai-speech` provider in agent/models.yml (set DANKVOICE_URL)";
|
||||
|
||||
async function speak(ctx: ExtensionContext, text: string) {
|
||||
const spoken = stripForSpeech(text, cfg.maxChars);
|
||||
if (!spoken) return;
|
||||
if (!cfg.target) {
|
||||
ctx.ui.notify(missingTarget(), "error");
|
||||
return;
|
||||
}
|
||||
if (!cfg.player) {
|
||||
ctx.ui.notify("dankvoice: no audio player found (pw-play, paplay or ffplay)", "error");
|
||||
return;
|
||||
}
|
||||
stop();
|
||||
const dir = mkdtempSync(join(tmpdir(), "dankvoice-"));
|
||||
try {
|
||||
const file = await synthesize(cfg.target, speechBody(cfg.target.model, spoken, state.voice), dir, SYNTH_TIMEOUT_MS);
|
||||
state.child = startPlayback(cfg.player, file, dir);
|
||||
} catch (error) {
|
||||
stop();
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
ctx.ui.notify(`dankvoice: ${error instanceof Error ? error.message : String(error)}`, "error");
|
||||
}
|
||||
}
|
||||
|
||||
pi.registerCommand("say", {
|
||||
description:
|
||||
"Speak with the fleet's cloned voices (frank-tts): /say <text> | on | off | voice <name> | voices | stop | status",
|
||||
handler: async (args: string, ctx) => {
|
||||
state.lastCtx = ctx;
|
||||
const action = parseSayArgs(args);
|
||||
switch (action.kind) {
|
||||
case "help":
|
||||
ctx.ui.notify(
|
||||
"dankvoice: /say <text> · /say on|off (auto-speak answers) · /say voice <name> · /say voices · /say stop · /say status",
|
||||
"info",
|
||||
);
|
||||
return;
|
||||
case "on":
|
||||
state.auto = true;
|
||||
reflectStatus();
|
||||
ctx.ui.notify(`dankvoice: auto-speak armed (${voiceLabel()} @ ${cfg.target?.speechUrl ?? "no endpoint"})`, "info");
|
||||
return;
|
||||
case "off":
|
||||
state.auto = false;
|
||||
stop();
|
||||
reflectStatus();
|
||||
ctx.ui.notify("dankvoice: auto-speak off", "info");
|
||||
return;
|
||||
case "stop":
|
||||
stop();
|
||||
return;
|
||||
case "voice":
|
||||
state.voice = action.name;
|
||||
reflectStatus();
|
||||
ctx.ui.notify(
|
||||
action.name ? `dankvoice: voice = ${action.name}` : "dankvoice: voice = whatever frank-tts defaults to",
|
||||
"info",
|
||||
);
|
||||
return;
|
||||
case "voices": {
|
||||
if (!cfg.target) {
|
||||
ctx.ui.notify(missingTarget(), "error");
|
||||
return;
|
||||
}
|
||||
const voices: string[] = [];
|
||||
try {
|
||||
voices.push(...(await fetchVoices(cfg.target)));
|
||||
} catch (error) {
|
||||
ctx.ui.notify(`dankvoice: ${error instanceof Error ? error.message : String(error)}`, "error");
|
||||
return;
|
||||
}
|
||||
ctx.ui.notify(
|
||||
voices.length
|
||||
? `dankvoice voices: ${voices.join(", ")} — using ${voiceLabel()}`
|
||||
: "dankvoice: frank-tts reported no voice list",
|
||||
voices.length ? "info" : "warning",
|
||||
);
|
||||
return;
|
||||
}
|
||||
case "status":
|
||||
ctx.ui.notify(
|
||||
`dankvoice: voice ${voiceLabel()} · ${cfg.target?.speechUrl ?? "NO ENDPOINT"} · player ${cfg.player ? cfg.player.command : "NONE"} · auto-speak ${state.auto ? "on" : "off"} · cap ${cfg.maxChars} chars`,
|
||||
"info",
|
||||
);
|
||||
return;
|
||||
case "speak":
|
||||
await speak(ctx, action.text);
|
||||
return;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
pi.on("agent_end", async (event, ctx) => {
|
||||
state.lastCtx = ctx;
|
||||
if (!state.auto) return;
|
||||
// `willContinue` means the harness already scheduled a continuation
|
||||
// (auto-retry, empty-stop retry): this is not the user-visible terminal
|
||||
// settle, so narrating it would speak an answer the model is about to redo.
|
||||
if (event.willContinue) return;
|
||||
if (ctx.mode === "print" || ctx.mode === "json") return;
|
||||
await speak(ctx, lastAssistantText(event.messages));
|
||||
});
|
||||
|
||||
pi.on("session_shutdown", () => {
|
||||
stop();
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user