Frank
bcd52bb0f4
agent_end awaited one whole-answer synthesis; long answers hit the harness 30s extension-handler timeout (handler timed out after 30000ms) and the single frank-tts request was the stall. Port the built-in speak-aloud vocalizer's utterance segmenter (fHe in dist/cli.js) so segmentation matches the known-working stock path: [.!?…]-run boundaries, abbreviation/decimal/inline-code guards, fenced code and tables swallowed, markup cleaned. Play segment N to player exit before synthesizing N+1 (audio never stacks); fire-and-forget the chain so utterances outlive the handler timeout; a run token retires superseded chains. DANKVOICE_MAX becomes a per-answer budget of whole segments; stripForSpeech is gone. Refs #23 follow-up.
941 lines
34 KiB
TypeScript
941 lines
34 KiB
TypeScript
/**
|
|
* dankvoice — cloned-voice vocalization for omp, served by frank-tts.
|
|
*
|
|
* The built-in speak-aloud feature (`speech.enabled` / `speech.voice`) and the
|
|
* `omp say` CLI are gated to the bundled Kokoro voice enum, so the cloned
|
|
* voices on frank-tts (Dankclaw :7772, the `speech` model role in
|
|
* agent/models.yml) could only ever produce files. This extension adds the
|
|
* missing path — the TUI speaks with the cloned voices:
|
|
*
|
|
* /say open the options menu
|
|
* /say <text> speak once through the speakers
|
|
* /say on | off arm/disarm auto-speak of the assistant's final answer
|
|
* /say menu the options menu (voice, output device, player, cap)
|
|
* /say voice <name> choose a voice (empty = whatever the server defaults to)
|
|
* /say voices list the voices frank-tts reports
|
|
* /say stop cut off the current utterance
|
|
* /say status voice, endpoint, player and device in use
|
|
* /say help the command list
|
|
*
|
|
* The endpoint comes from agent/models.yml — the provider block carrying
|
|
* `api: openai-speech`, i.e. the same row the speech role resolves — so this
|
|
* file carries no host address and no secret. Menu choices persist to
|
|
* ~/.omp/cache/dankvoice.json (runtime state, never tracked). Environment
|
|
* overrides win over the file, which wins over defaults:
|
|
*
|
|
* DANKVOICE_URL speech base URL (…/v1) instead of models.yml
|
|
* DANKVOICE_MODEL served model id (default: the id in that block)
|
|
* DANKVOICE_CONFIG settings file (default ~/.omp/cache/dankvoice.json)
|
|
* DANKVOICE_PLAYER player binary, skipping detection
|
|
* DANKVOICE_VOICE voice name ("" = server default)
|
|
* DANKVOICE_MAX spoken-character budget per answer (default 700)
|
|
* DANKVOICE_AUTO=1 start with auto-speak armed
|
|
*
|
|
* Long answers are cut into speakable segments by a state machine ported
|
|
* from omp's built-in vocalizer: strong boundaries on [.!?…] runs followed
|
|
* by whitespace (so "!!!!!" and "..." stay one boundary), guards for
|
|
* abbreviations and decimals, code fences and tables swallowed, markdown
|
|
* cleaned. Each segment is synthesized to a temp WAV in $TMPDIR and played
|
|
* to completion before the next request goes out — requests are strictly
|
|
* sequential, so audio never stacks. The chain runs detached from the event
|
|
* handler, so an utterance may outlive the harness handler timeout. A new
|
|
* utterance interrupts the previous one. Output routing: a saved sink
|
|
* becomes `pw-play --target <sink>` or `paplay --device=<sink>` (ffplay has
|
|
* no device concept and ignores it); with no saved sink the player's own
|
|
* default applies.
|
|
*/
|
|
|
|
import { spawn, spawnSync, type ChildProcess } from "node:child_process";
|
|
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
|
|
const DEFAULT_MAX_CHARS = 700;
|
|
const SYNTH_TIMEOUT_MS = 60_000;
|
|
const VOICES_TIMEOUT_MS = 15_000;
|
|
/** A voice name the shim must reject; its 400 body carries the real list. */
|
|
const VOICE_PROBE = "_dankvoice_probe_";
|
|
|
|
// ---------------------------------------------------------------- pure logic
|
|
|
|
export interface SpeechTarget {
|
|
/** POST target for OpenAI speech synthesis. */
|
|
speechUrl: string;
|
|
/** GET target for the voice roster (frank-tts answers a bare JSON array). */
|
|
voicesUrl: string;
|
|
/** GET target for the liveness route of the same host:port. */
|
|
healthUrl: string;
|
|
/** Served model id to name in the request body. */
|
|
model: string;
|
|
}
|
|
|
|
export interface Player {
|
|
command: string;
|
|
args: string[];
|
|
}
|
|
|
|
export type SayAction =
|
|
| { kind: "menu" }
|
|
| { kind: "help" }
|
|
| { kind: "speak"; text: string }
|
|
| { kind: "on" }
|
|
| { kind: "off" }
|
|
| { kind: "stop" }
|
|
| { kind: "voices" }
|
|
| { kind: "status" }
|
|
| { kind: "voice"; name: string };
|
|
|
|
/** `/say <args>` grammar: subcommands win, anything else is spoken text. */
|
|
export function parseSayArgs(raw: string): SayAction {
|
|
const args = raw.trim();
|
|
const lower = args.toLowerCase();
|
|
if (lower === "" || lower === "menu") return { kind: "menu" };
|
|
if (lower === "help") return { kind: "help" };
|
|
if (lower === "on") return { kind: "on" };
|
|
if (lower === "off") return { kind: "off" };
|
|
if (lower === "stop") return { kind: "stop" };
|
|
if (lower === "voices") return { kind: "voices" };
|
|
if (lower === "status") return { kind: "status" };
|
|
if (lower === "voice" || lower.startsWith("voice ")) return { kind: "voice", name: args.slice(5).trim() };
|
|
return { kind: "speak", text: args };
|
|
}
|
|
|
|
/**
|
|
* Pull the speech endpoint out of agent/models.yml. The runner API lives on
|
|
* the MODEL row (`api: openai-speech`), the base URL on its provider, so this
|
|
* scan remembers the last provider base URL and model id seen and returns when
|
|
* a model row claims the speech runner API. A new provider key (indent two)
|
|
* resets both.
|
|
*/
|
|
export function resolveSpeechEndpoint(modelsYml: string): SpeechTarget | undefined {
|
|
let base: string | undefined;
|
|
let id: string | undefined;
|
|
for (const line of modelsYml.split(/\r?\n/)) {
|
|
const trimmed = line.trim();
|
|
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
if (/^ {2}\S.*:\s*$/.test(line)) {
|
|
base = undefined;
|
|
id = undefined;
|
|
continue;
|
|
}
|
|
const providerUrl = trimmed.match(/^baseUrl:\s*(\S+)/);
|
|
if (providerUrl) {
|
|
base = providerUrl[1].replace(/\/+$/, "");
|
|
continue;
|
|
}
|
|
const modelId = trimmed.match(/^-\s*id:\s*(\S+)/);
|
|
if (modelId) {
|
|
id = modelId[1];
|
|
continue;
|
|
}
|
|
if (/^api:\s*openai-speech\s*$/.test(trimmed) && base) {
|
|
return {
|
|
speechUrl: `${base}/audio/speech`,
|
|
voicesUrl: `${base}/audio/voices`,
|
|
healthUrl: `${base.replace(/\/v1$/, "")}/health`,
|
|
model: id ?? "frank-tts",
|
|
};
|
|
}
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
/** OpenAI speech body; `voice` is sent only when named — the shim carries its own default. */
|
|
export function speechBody(model: string, text: string, voice: string): Record<string, string> {
|
|
const body: Record<string, string> = { model, input: text, response_format: "wav" };
|
|
if (voice) body.voice = voice;
|
|
return body;
|
|
}
|
|
|
|
/** Voice names as frank-tts reports them: the invalid-voice 400 body carries `available_voices`. */
|
|
export function parseVoices(payload: unknown): string[] {
|
|
// frank-tts answers GET /v1/audio/voices with a bare array and an unknown voice with
|
|
// {error:{available_voices:[…]}}; a flat body is accepted too.
|
|
if (typeof payload !== "object" || payload === null) return [];
|
|
let list: unknown;
|
|
if (Array.isArray(payload)) {
|
|
list = payload;
|
|
} else if ("error" in payload && typeof payload.error === "object" && payload.error !== null && "available_voices" in payload.error) {
|
|
list = payload.error.available_voices;
|
|
} else if ("available_voices" in payload) {
|
|
list = payload.available_voices;
|
|
} else if ("voices" in payload) {
|
|
list = payload.voices;
|
|
} else if ("data" in payload) {
|
|
list = payload.data;
|
|
}
|
|
if (!Array.isArray(list)) return [];
|
|
const names: string[] = [];
|
|
for (const entry of list) {
|
|
if (typeof entry === "string") {
|
|
names.push(entry);
|
|
} else if (typeof entry === "object" && entry !== null) {
|
|
const name = "id" in entry ? entry.id : "name" in entry ? entry.name : undefined;
|
|
if (typeof name === "string") names.push(name);
|
|
}
|
|
}
|
|
return names.filter((name) => name.length > 0);
|
|
}
|
|
|
|
/** Known players, PipeWire first (the workstations run WirePlumber); first existing wins. */
|
|
export function availablePlayers(lookup: (command: string) => string | null): string[] {
|
|
return ["pw-play", "paplay", "ffplay"].filter((command) => lookup(command) !== null);
|
|
}
|
|
|
|
/** One player invocation: the saved sink becomes each player's routing dialect. */
|
|
export function playerFor(command: string, sink: string): Player {
|
|
const base = command.split("/").pop() ?? command;
|
|
if (base === "ffplay") return { command, args: ["-nodisp", "-autoexit", "-loglevel", "quiet"] };
|
|
if (!sink) return { command, args: [] };
|
|
if (base === "paplay") return { command, args: [`--device=${sink}`] };
|
|
return { command, args: ["--target", sink] };
|
|
}
|
|
|
|
// ------------------------------------------------------------ segmenting
|
|
//
|
|
// Ported from omp's built-in speak-aloud vocalizer (the `fHe` segmenter and
|
|
// its helpers in dist/cli.js). The same boundary rules as the stock path,
|
|
// kept deliberately: push() takes text and returns the segments that are
|
|
// complete, flush() ends the message.
|
|
|
|
const SENTENCE_END = /[.!?\u2026]+[)\]"'\u00BB\u201D\u2019]*\s/g;
|
|
const WEAK_BREAK = /[,;:\u2014\u2013]\s/g;
|
|
const ABBREVIATION = /(?:^|\s)(?:e\.g|i\.e|etc|vs|Mr|Mrs|Ms|Dr|St|No)\.$/i;
|
|
const LINE_HEAD = /^(?:#{1,6}|[-*+]|-{2,}|\*{2,}|_{2,}|\d{1,3}|\d{1,3}[.)]|>+|`{1,2}|~{1,2})$/;
|
|
const HORIZONTAL_RULE = /^(?:-{3,}|\*{3,}|_{3,})\s*$/;
|
|
|
|
const IMAGE_LINK = /!\[([^\]]*)\]\(([^()]*)\)/g;
|
|
const LINK = /\[([^\]]+)\]\(([^()]*)\)/g;
|
|
const BRACKETED_URL = /<(https?:\/\/[^\s>]+)>/g;
|
|
const BARE_URL = /\bhttps?:\/\/[^\s<>()"'\]]+|\bwww\.[\w-]+(?:\.[\w-]+)+[^\s<>()"'\]]*/g;
|
|
const INLINE_CODE = /`{1,2}([^`]+)`{1,2}/g;
|
|
const BOLD_MARKUP = /\*\*|__|~~/g;
|
|
const STAR = /\*(?=\S)|(?<=\S)\*/g;
|
|
const UNDERSCORE = /(^|\s)_+|_+(?=\s|$)/g;
|
|
const HTML_TAG = /<\/?[a-zA-Z][^<>]*>/g;
|
|
const EMPHASIS_RUN = /(^|\s)[-*_]{3,}(?=\s|$)/g;
|
|
const DEEP_PATH = /(^|[\s("'`])((?:~|\.{1,2})?\/?[\w.@+-]+(?:\/[\w.@+-]+){2,}\/?)/g;
|
|
const HAS_WORD = /[\p{L}\p{N}]/u;
|
|
|
|
/** Speak a URL by host and a file path by its last segment. */
|
|
export function spokenDomain(url: string): string {
|
|
return url.replace(/^[a-z][\w+.-]*:\/\//i, "").replace(/^www\./i, "").replace(/[/?#].*$/, "");
|
|
}
|
|
|
|
/** Final per-segment cleanup; empty when nothing speakable survived. */
|
|
export function cleanForSpeech(text: string): string {
|
|
const out = text
|
|
.replace(IMAGE_LINK, "$1")
|
|
.replace(LINK, "$1")
|
|
.replace(BRACKETED_URL, (_match, url: string) => spokenDomain(url))
|
|
.replace(BARE_URL, (url: string) => spokenDomain(url))
|
|
.replace(INLINE_CODE, "$1")
|
|
.replace(BOLD_MARKUP, "")
|
|
.replace(STAR, "")
|
|
.replace(UNDERSCORE, "$1")
|
|
.replace(HTML_TAG, " ")
|
|
.replace(EMPHASIS_RUN, "$1")
|
|
.replace(DEEP_PATH, (_match, pre: string, path: string) => {
|
|
const parts = path.split("/").filter((part) => part.length > 0);
|
|
return pre + (parts[parts.length - 1] ?? path);
|
|
})
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
return HAS_WORD.test(out) ? out : "";
|
|
}
|
|
|
|
function sentenceBoundary(text: string, minChars: number): number {
|
|
SENTENCE_END.lastIndex = 0;
|
|
for (let match = SENTENCE_END.exec(text); match; match = SENTENCE_END.exec(text)) {
|
|
const end = match.index + match[0].length;
|
|
if (end < minChars) continue;
|
|
if (ABBREVIATION.test(text.slice(0, end).trimEnd())) continue;
|
|
if ((text.slice(0, end).match(/`/g)?.length ?? 0) % 2 !== 0) continue;
|
|
return end;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
function firstWeakBreak(text: string, minChars: number): number {
|
|
WEAK_BREAK.lastIndex = 0;
|
|
for (let match = WEAK_BREAK.exec(text); match; match = WEAK_BREAK.exec(text)) {
|
|
const end = match.index + match[0].length;
|
|
if (end >= minChars) return end;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
function lastWeakBreakWithin(text: string, minChars: number, maxChars: number): number {
|
|
WEAK_BREAK.lastIndex = 0;
|
|
let found = -1;
|
|
for (let match = WEAK_BREAK.exec(text); match; match = WEAK_BREAK.exec(text)) {
|
|
const end = match.index + match[0].length;
|
|
if (end > maxChars) break;
|
|
if (end >= minChars) found = end;
|
|
}
|
|
return found;
|
|
}
|
|
|
|
function wordWrapCut(text: string, maxChars: number): number {
|
|
const space = text.lastIndexOf(" ", maxChars);
|
|
return space > 0 ? space + 1 : Math.min(maxChars, text.length);
|
|
}
|
|
|
|
type LineHead =
|
|
| { kind: "swallow" }
|
|
| { kind: "fence"; fence: string }
|
|
| { kind: "marker"; spoken: string }
|
|
| { kind: "prose"; text: string }
|
|
| { kind: "undecided" };
|
|
|
|
/** Classify a line start: tables and fences swallow, list markers get spoken. */
|
|
export function classifyLineHead(head: string): LineHead {
|
|
if (head === "|") return { kind: "swallow" };
|
|
if (/^(?:`{3}|~{3})/.test(head)) return { kind: "fence", fence: head.slice(0, 3) };
|
|
if (/^#{1,6}[ \t]/.test(head)) return { kind: "marker", spoken: "" };
|
|
if (/^[-*+][ \t]/.test(head)) return { kind: "marker", spoken: "" };
|
|
const ordered = /^(\d{1,3})[.)][ \t]/.exec(head);
|
|
if (ordered) return { kind: "marker", spoken: `${ordered[1]}, ` };
|
|
if (/^>+/.test(head) && !/^>+$/.test(head)) return { kind: "prose", text: head.replace(/^>+[ \t]?/, "") };
|
|
if (LINE_HEAD.test(head)) return { kind: "undecided" };
|
|
return { kind: "prose", text: head };
|
|
}
|
|
|
|
type SegmenterState = "linestart" | "prose" | "swallow" | "code";
|
|
|
|
/** The built-in vocalizer's segmenter: streamed text in, speakable segments out. */
|
|
export class SpeechSegmenter {
|
|
#state: SegmenterState = "linestart";
|
|
#head = "";
|
|
#fence = "";
|
|
#fenceLine = "";
|
|
#resume: SegmenterState = "linestart";
|
|
#buffer = "";
|
|
#started = false;
|
|
|
|
push(chunk: string): string[] {
|
|
const out: string[] = [];
|
|
for (const char of chunk) this.#feed(char, out);
|
|
this.#split(out);
|
|
return out;
|
|
}
|
|
|
|
flush(): string[] {
|
|
const out: string[] = [];
|
|
if (this.#state === "linestart" && this.#head.length > 0 && !HORIZONTAL_RULE.test(this.#head)) this.#buffer += this.#head;
|
|
this.#head = "";
|
|
this.#state = "linestart";
|
|
this.#emit(out);
|
|
return out;
|
|
}
|
|
|
|
/** Streaming idle flush: only cut a tail that already looks finished. */
|
|
flushIdle(): string[] {
|
|
const out: string[] = [];
|
|
const tail = this.#buffer.trimEnd();
|
|
if (!/[.!?\u2026][)\]"'\u00BB\u201D\u2019]*$/.test(tail) && tail.length < 24) return out;
|
|
this.#emit(out);
|
|
return out;
|
|
}
|
|
|
|
#feed(char: string, out: string[]): void {
|
|
switch (this.#state) {
|
|
case "linestart":
|
|
this.#feedLineStart(char, out);
|
|
return;
|
|
case "prose":
|
|
if (char === "\n") this.#endLine(out);
|
|
else this.#buffer += char;
|
|
return;
|
|
case "swallow":
|
|
if (char === "\n") this.#state = this.#resume;
|
|
return;
|
|
case "code":
|
|
this.#feedCode(char);
|
|
return;
|
|
}
|
|
}
|
|
|
|
#feedLineStart(char: string, out: string[]): void {
|
|
if (char === "\n") {
|
|
const head = this.#head;
|
|
this.#head = "";
|
|
if (head.length > 0 && !HORIZONTAL_RULE.test(head)) this.#buffer += head;
|
|
this.#endLine(out);
|
|
return;
|
|
}
|
|
this.#head += char;
|
|
const classified = classifyLineHead(this.#head);
|
|
if (classified.kind === "undecided") {
|
|
if (this.#head.length > 8) {
|
|
this.#buffer += this.#head;
|
|
this.#head = "";
|
|
this.#state = "prose";
|
|
}
|
|
return;
|
|
}
|
|
this.#head = "";
|
|
switch (classified.kind) {
|
|
case "prose":
|
|
this.#buffer += classified.text;
|
|
this.#state = "prose";
|
|
return;
|
|
case "marker":
|
|
this.#buffer += classified.spoken;
|
|
this.#state = "prose";
|
|
return;
|
|
case "swallow":
|
|
this.#state = "swallow";
|
|
this.#resume = "linestart";
|
|
return;
|
|
case "fence":
|
|
this.#fence = classified.fence;
|
|
this.#fenceLine = "";
|
|
this.#state = "swallow";
|
|
this.#resume = "code";
|
|
return;
|
|
}
|
|
}
|
|
|
|
#feedCode(char: string): void {
|
|
if (char === "\n") {
|
|
this.#fenceLine = "";
|
|
return;
|
|
}
|
|
if (this.#fenceLine.length < 3) {
|
|
this.#fenceLine += char;
|
|
if (this.#fenceLine === this.#fence) {
|
|
this.#state = "swallow";
|
|
this.#resume = "linestart";
|
|
}
|
|
}
|
|
}
|
|
|
|
#endLine(out: string[]): void {
|
|
this.#state = "linestart";
|
|
this.#emit(out);
|
|
}
|
|
|
|
#emit(out: string[]): void {
|
|
this.#split(out);
|
|
const segment = this.#buffer;
|
|
this.#buffer = "";
|
|
this.#pushSegment(segment, out);
|
|
}
|
|
|
|
#split(out: string[]): void {
|
|
for (;;) {
|
|
const buffer = this.#buffer;
|
|
const boundary = sentenceBoundary(buffer, this.#started ? 24 : 12);
|
|
if (boundary !== -1 && boundary <= 280) {
|
|
this.#cut(boundary, out);
|
|
continue;
|
|
}
|
|
if (!this.#started && buffer.length >= 40) {
|
|
const weak = firstWeakBreak(buffer, 12);
|
|
if (weak !== -1 && weak <= 140) {
|
|
this.#cut(weak, out);
|
|
continue;
|
|
}
|
|
if (buffer.length >= 140) {
|
|
this.#cut(wordWrapCut(buffer, 140), out);
|
|
continue;
|
|
}
|
|
}
|
|
if (this.#started && buffer.length >= 160) {
|
|
const weak = lastWeakBreakWithin(buffer, 24, 160);
|
|
if (weak !== -1) {
|
|
this.#cut(weak, out);
|
|
continue;
|
|
}
|
|
}
|
|
if (buffer.length > 280) {
|
|
const weak = lastWeakBreakWithin(buffer, 24, 280);
|
|
this.#cut(weak !== -1 ? weak : wordWrapCut(buffer, 280), out);
|
|
continue;
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
|
|
#cut(end: number, out: string[]): void {
|
|
const head = this.#buffer.slice(0, end);
|
|
this.#buffer = this.#buffer.slice(end);
|
|
this.#pushSegment(head, out);
|
|
}
|
|
|
|
#pushSegment(raw: string, out: string[]): void {
|
|
const cleaned = cleanForSpeech(raw);
|
|
if (!cleaned) return;
|
|
out.push(cleaned);
|
|
this.#started = true;
|
|
}
|
|
}
|
|
|
|
/** A whole message to speakable segments: the segmenter run start to end. */
|
|
export function segmentsForSpeech(text: string): string[] {
|
|
const segmenter = new SpeechSegmenter();
|
|
const segments = segmenter.push(text);
|
|
for (const tail of segmenter.flush()) segments.push(tail);
|
|
return segments;
|
|
}
|
|
|
|
/** The leading whole segments that fit the user's spoken-character budget. */
|
|
export function budgetSegments(segments: string[], maxChars: number): string[] {
|
|
const chosen: string[] = [];
|
|
let used = 0;
|
|
for (const segment of segments) {
|
|
if (used + segment.length > maxChars) break;
|
|
used += segment.length;
|
|
chosen.push(segment);
|
|
}
|
|
return chosen;
|
|
}
|
|
|
|
/** The text the assistant last actually said; tool-only messages do not count. */
|
|
export function lastAssistantText(messages: readonly unknown[]): string {
|
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
const message = messages[i];
|
|
if (!message || typeof message !== "object" || !("role" in message) || message.role !== "assistant") continue;
|
|
if (!("content" in message)) continue;
|
|
const content = message.content;
|
|
if (typeof content === "string") return content;
|
|
if (!Array.isArray(content)) continue;
|
|
const parts: string[] = [];
|
|
for (const part of content) {
|
|
if (part && typeof part === "object" && "type" in part && part.type === "text" && "text" in part && typeof part.text === "string") {
|
|
parts.push(part.text);
|
|
}
|
|
}
|
|
const joined = parts.join("\n").trim();
|
|
if (joined) return joined;
|
|
}
|
|
return "";
|
|
}
|
|
|
|
/** Menu choices, persisted between sessions; every field optional. */
|
|
export interface Settings {
|
|
voice?: string;
|
|
player?: string;
|
|
sink?: string;
|
|
maxChars?: number;
|
|
auto?: boolean;
|
|
}
|
|
|
|
/** Where menu choices live: DANKVOICE_CONFIG, else ~/.omp/cache/dankvoice.json. */
|
|
export function settingsPath(env: NodeJS.ProcessEnv): string {
|
|
return env.DANKVOICE_CONFIG?.trim() || join(env.HOME ?? "", ".omp", "cache", "dankvoice.json");
|
|
}
|
|
|
|
export function readSettings(path: string): Settings {
|
|
try {
|
|
const parsed: unknown = JSON.parse(readFileSync(path, "utf8"));
|
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {};
|
|
const raw = parsed as Record<string, unknown>;
|
|
const settings: Settings = {};
|
|
if (typeof raw.voice === "string") settings.voice = raw.voice;
|
|
if (typeof raw.player === "string") settings.player = raw.player;
|
|
if (typeof raw.sink === "string") settings.sink = raw.sink;
|
|
if (typeof raw.maxChars === "number" && Number.isFinite(raw.maxChars) && raw.maxChars > 0) settings.maxChars = Math.floor(raw.maxChars);
|
|
if (typeof raw.auto === "boolean") settings.auto = raw.auto;
|
|
return settings;
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
/** A sink row from `wpctl list audio sinks`: `116<TAB>alsa_output…<TAB>audio/sink<TAB>*`. */
|
|
export interface SinkRow {
|
|
id: string;
|
|
name: string;
|
|
}
|
|
|
|
export function parseSinkList(output: string): SinkRow[] {
|
|
const rows: SinkRow[] = [];
|
|
for (const line of output.split("\n")) {
|
|
const cols = line.split("\t").map((col) => col.trim());
|
|
if (cols.length < 3 || !cols[2]?.includes("audio/sink")) continue;
|
|
const id = cols[0] ?? "";
|
|
const name = cols[1] ?? "";
|
|
if (!id || !name) continue;
|
|
rows.push({ id, name });
|
|
}
|
|
return rows;
|
|
}
|
|
|
|
// ------------------------------------------------------------- live wiring
|
|
|
|
interface Config {
|
|
target?: SpeechTarget;
|
|
player?: Player;
|
|
sink: string;
|
|
maxChars: number;
|
|
auto: boolean;
|
|
voice: string;
|
|
}
|
|
|
|
function lookupOnPath(command: string): string | null {
|
|
const found = spawnSync("sh", ["-c", `command -v ${command}`], { encoding: "utf8" });
|
|
return found.status === 0 && found.stdout.trim() ? found.stdout.trim() : null;
|
|
}
|
|
|
|
function readModelsYml(): string {
|
|
const path = process.env.DANKVOICE_MODELS ?? join(process.env.HOME ?? "", ".omp", "agent", "models.yml");
|
|
try {
|
|
return readFileSync(path, "utf8");
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
function loadConfig(): Config {
|
|
const env = process.env;
|
|
const settings = readSettings(settingsPath(env));
|
|
const overrideUrl = env.DANKVOICE_URL?.trim().replace(/\/+$/, "");
|
|
const modelOverride = env.DANKVOICE_MODEL?.trim();
|
|
const fromYml = overrideUrl ? undefined : resolveSpeechEndpoint(readModelsYml());
|
|
const target: SpeechTarget | undefined = overrideUrl
|
|
? {
|
|
speechUrl: `${overrideUrl}/audio/speech`,
|
|
voicesUrl: `${overrideUrl}/audio/voices`,
|
|
healthUrl: `${overrideUrl.replace(/\/v1$/, "")}/health`,
|
|
model: modelOverride || "frank-tts",
|
|
}
|
|
: fromYml
|
|
? { ...fromYml, model: modelOverride || fromYml.model }
|
|
: undefined;
|
|
const playerCommand = env.DANKVOICE_PLAYER?.trim() || settings.player || availablePlayers(lookupOnPath)[0];
|
|
const sink = settings.sink ?? "";
|
|
const maxChars = Number(env.DANKVOICE_MAX);
|
|
return {
|
|
target,
|
|
player: playerCommand ? playerFor(playerCommand, sink) : undefined,
|
|
sink,
|
|
maxChars: Number.isFinite(maxChars) && maxChars > 0 ? Math.floor(maxChars) : (settings.maxChars ?? DEFAULT_MAX_CHARS),
|
|
auto: env.DANKVOICE_AUTO === "1" || (settings.auto ?? false),
|
|
voice: env.DANKVOICE_VOICE?.trim() ?? settings.voice ?? "",
|
|
};
|
|
}
|
|
|
|
/** PipeWire sinks via wpctl; empty when wireplumber is unreachable. */
|
|
function listSinks(): SinkRow[] {
|
|
const probe = spawnSync("wpctl", ["list", "audio", "sinks"], { encoding: "utf8" });
|
|
return probe.status === 0 ? parseSinkList(probe.stdout) : [];
|
|
}
|
|
|
|
/** Synthesize one utterance into a WAV inside `dir`; throws with the endpoint's own message. */
|
|
async function synthesize(target: SpeechTarget, body: Record<string, string>, dir: string, timeoutMs: number): Promise<string> {
|
|
const res = await fetch(target.speechUrl, {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify(body),
|
|
signal: AbortSignal.timeout(timeoutMs),
|
|
});
|
|
const contentType = res.headers.get("content-type") ?? "";
|
|
if (!res.ok || contentType.includes("json")) {
|
|
let message = `frank-tts HTTP ${res.status}`;
|
|
try {
|
|
const payload: unknown = await res.json();
|
|
if (
|
|
typeof payload === "object" &&
|
|
payload !== null &&
|
|
"error" in payload &&
|
|
typeof payload.error === "object" &&
|
|
payload.error !== null &&
|
|
"message" in payload.error &&
|
|
typeof payload.error.message === "string"
|
|
) {
|
|
message = payload.error.message;
|
|
}
|
|
} catch {
|
|
// keep the status-line message
|
|
}
|
|
throw new Error(message);
|
|
}
|
|
const bytes = new Uint8Array(await res.arrayBuffer());
|
|
if (bytes.byteLength === 0) throw new Error("frank-tts returned no audio");
|
|
const file = join(dir, `say-${Date.now()}.wav`);
|
|
writeFileSync(file, bytes);
|
|
return file;
|
|
}
|
|
|
|
/** frank-tts's roster route first; if it ever answers nothing, ask with a voice it must reject. */
|
|
async function fetchVoices(target: SpeechTarget): Promise<string[]> {
|
|
const roster = await fetch(target.voicesUrl, { signal: AbortSignal.timeout(VOICES_TIMEOUT_MS) })
|
|
.then((res) => res.json())
|
|
.catch(() => undefined);
|
|
const listed = parseVoices(roster);
|
|
if (listed.length > 0) return listed;
|
|
const probe = await fetch(target.speechUrl, {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify(speechBody(target.model, "voice list probe", VOICE_PROBE)),
|
|
signal: AbortSignal.timeout(VOICES_TIMEOUT_MS),
|
|
}).catch(() => undefined);
|
|
if (!probe) return [];
|
|
return parseVoices(await probe.json().catch(() => undefined));
|
|
}
|
|
|
|
/** Play a WAV and clean its temp directory up when the player exits. */
|
|
function startPlayback(player: Player, file: string, dir: string): ChildProcess {
|
|
const child = spawn(player.command, [...player.args, file], { stdio: "ignore" });
|
|
const clean = () => {
|
|
try {
|
|
rmSync(file, { force: true });
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// temp files are not worth a toast
|
|
}
|
|
};
|
|
child.once("exit", clean);
|
|
child.once("error", clean);
|
|
return child;
|
|
}
|
|
|
|
export default function dankvoice(pi: ExtensionAPI) {
|
|
const cfg = loadConfig();
|
|
const state: { auto: boolean; voice: string; child?: ChildProcess; run: number; lastCtx?: ExtensionContext } = {
|
|
auto: cfg.auto,
|
|
voice: cfg.voice,
|
|
run: 0,
|
|
};
|
|
const voiceLabel = () => state.voice || "server default";
|
|
const reflectStatus = () => state.lastCtx?.ui.setStatus("dankvoice", state.auto ? `🔊 ${voiceLabel()}` : undefined);
|
|
|
|
const stop = () => {
|
|
state.run += 1; // retire the live chain; its loop stops at the next gate
|
|
if (state.child && state.child.exitCode === null) state.child.kill("SIGTERM");
|
|
state.child = undefined;
|
|
};
|
|
|
|
const missingTarget = () => "dankvoice: no `api: openai-speech` provider in agent/models.yml (set DANKVOICE_URL)";
|
|
|
|
const persistSettings = (ctx: ExtensionContext) => {
|
|
const settings: Settings = {
|
|
voice: state.voice,
|
|
player: cfg.player?.command ?? "",
|
|
sink: cfg.sink,
|
|
maxChars: cfg.maxChars,
|
|
auto: state.auto,
|
|
};
|
|
try {
|
|
writeFileSync(settingsPath(process.env), `${JSON.stringify(settings, null, 2)}\n`);
|
|
} catch (error) {
|
|
ctx.ui.notify(`dankvoice: settings not saved: ${error instanceof Error ? error.message : String(error)}`, "warning");
|
|
}
|
|
};
|
|
|
|
const setVoice = (ctx: ExtensionContext, name: string) => {
|
|
state.voice = name;
|
|
reflectStatus();
|
|
ctx.ui.notify(name ? `dankvoice: voice = ${name}` : "dankvoice: voice = whatever frank-tts defaults to", "info");
|
|
persistSettings(ctx);
|
|
};
|
|
|
|
const setAuto = (ctx: ExtensionContext, on: boolean) => {
|
|
state.auto = on;
|
|
if (!on) stop();
|
|
reflectStatus();
|
|
ctx.ui.notify(
|
|
on ? `dankvoice: auto-speak armed (${voiceLabel()} @ ${cfg.target?.speechUrl ?? "no endpoint"})` : "dankvoice: auto-speak off",
|
|
"info",
|
|
);
|
|
persistSettings(ctx);
|
|
};
|
|
|
|
/**
|
|
* Speak one answer as strictly sequential requests: synthesize a segment,
|
|
* play it to the end, only then ask for the next — audio never stacks.
|
|
* Superseded chains (new utterance, /say stop, disarm) exit at the gates.
|
|
*/
|
|
async function speak(ctx: ExtensionContext, text: string) {
|
|
if (!cfg.target) {
|
|
ctx.ui.notify(missingTarget(), "error");
|
|
return;
|
|
}
|
|
if (!cfg.player) {
|
|
ctx.ui.notify("dankvoice: no audio player found (pw-play, paplay or ffplay)", "error");
|
|
return;
|
|
}
|
|
const target = cfg.target;
|
|
const player = cfg.player;
|
|
const segments = budgetSegments(segmentsForSpeech(text), cfg.maxChars);
|
|
if (segments.length === 0) return;
|
|
stop();
|
|
const run = state.run;
|
|
for (const segment of segments) {
|
|
if (state.run !== run) return;
|
|
const dir = mkdtempSync(join(tmpdir(), "dankvoice-"));
|
|
try {
|
|
const file = await synthesize(target, speechBody(target.model, segment, state.voice), dir, SYNTH_TIMEOUT_MS);
|
|
if (state.run !== run) {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
return;
|
|
}
|
|
await new Promise<void>((resolve) => {
|
|
const child = startPlayback(player, file, dir);
|
|
state.child = child;
|
|
child.once("exit", () => resolve());
|
|
child.once("error", () => resolve());
|
|
});
|
|
} catch (error) {
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// temp files are not worth a toast
|
|
}
|
|
ctx.ui.notify(`dankvoice: ${error instanceof Error ? error.message : String(error)}`, "error");
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
async function openMenu(ctx: ExtensionContext): Promise<void> {
|
|
const options = [
|
|
`Auto-speak: ${state.auto ? "on" : "off"} (toggle)`,
|
|
`Voice: ${voiceLabel()} (change)`,
|
|
`Output device: ${cfg.sink || "system default"} (change)`,
|
|
`Player: ${cfg.player?.command ?? "none"} (change)`,
|
|
`Spoken cap: ${cfg.maxChars} chars (change)`,
|
|
"Speak a test phrase",
|
|
];
|
|
const choice = await ctx.ui.select("dankvoice — options", options);
|
|
if (!choice) return;
|
|
if (choice === options[0]) {
|
|
setAuto(ctx, !state.auto);
|
|
return;
|
|
}
|
|
if (choice === options[1]) {
|
|
if (!cfg.target) {
|
|
ctx.ui.notify(missingTarget(), "error");
|
|
return;
|
|
}
|
|
let voices: string[];
|
|
try {
|
|
voices = await fetchVoices(cfg.target);
|
|
} catch (error) {
|
|
ctx.ui.notify(`dankvoice: ${error instanceof Error ? error.message : String(error)}`, "error");
|
|
return;
|
|
}
|
|
const picked = await ctx.ui.select("dankvoice — voice", ["server default", ...voices]);
|
|
if (picked === undefined) return;
|
|
setVoice(ctx, picked === "server default" ? "" : picked);
|
|
return;
|
|
}
|
|
if (choice === options[2]) {
|
|
const picked = await ctx.ui.select("dankvoice — output device", ["system default", ...listSinks().map((sink) => sink.name)]);
|
|
if (picked === undefined) return;
|
|
cfg.sink = picked === "system default" ? "" : picked;
|
|
if (cfg.player) cfg.player = playerFor(cfg.player.command, cfg.sink);
|
|
ctx.ui.notify(`dankvoice: output device = ${cfg.sink || "system default"}`, "info");
|
|
persistSettings(ctx);
|
|
return;
|
|
}
|
|
if (choice === options[3]) {
|
|
const choices = [...new Set([cfg.player?.command, ...availablePlayers(lookupOnPath)].filter((c): c is string => Boolean(c)))];
|
|
if (choices.length === 0) {
|
|
ctx.ui.notify("dankvoice: no audio player found (pw-play, paplay or ffplay)", "error");
|
|
return;
|
|
}
|
|
const picked = await ctx.ui.select("dankvoice — player", choices);
|
|
if (picked === undefined) return;
|
|
cfg.player = playerFor(picked, cfg.sink);
|
|
ctx.ui.notify(`dankvoice: player = ${picked}`, "info");
|
|
persistSettings(ctx);
|
|
return;
|
|
}
|
|
if (choice === options[4]) {
|
|
const picked = await ctx.ui.select("dankvoice — spoken cap", ["300", "700", "1500", "3000"]);
|
|
const next = Number(picked);
|
|
if (!Number.isFinite(next) || next <= 0) return;
|
|
cfg.maxChars = next;
|
|
ctx.ui.notify(`dankvoice: cap = ${next} chars`, "info");
|
|
persistSettings(ctx);
|
|
return;
|
|
}
|
|
await speak(ctx, `dankvoice: ${voiceLabel()} through ${cfg.player?.command ?? "no player"}, checking.`);
|
|
}
|
|
|
|
pi.registerCommand("say", {
|
|
description:
|
|
"Speak with the fleet's cloned voices (frank-tts): /say (menu) | <text> | on | off | menu | voice <name> | voices | stop | status",
|
|
handler: async (args: string, ctx) => {
|
|
state.lastCtx = ctx;
|
|
const action = parseSayArgs(args);
|
|
switch (action.kind) {
|
|
case "menu":
|
|
await openMenu(ctx);
|
|
return;
|
|
case "help":
|
|
ctx.ui.notify(
|
|
"dankvoice: /say (menu) · /say <text> · /say on|off (auto-speak answers) · /say voice <name> · /say voices · /say stop · /say status",
|
|
"info",
|
|
);
|
|
return;
|
|
case "on":
|
|
setAuto(ctx, true);
|
|
return;
|
|
case "off":
|
|
setAuto(ctx, false);
|
|
return;
|
|
case "stop":
|
|
stop();
|
|
return;
|
|
case "voice":
|
|
setVoice(ctx, action.name);
|
|
return;
|
|
case "voices": {
|
|
if (!cfg.target) {
|
|
ctx.ui.notify(missingTarget(), "error");
|
|
return;
|
|
}
|
|
const voices: string[] = [];
|
|
try {
|
|
voices.push(...(await fetchVoices(cfg.target)));
|
|
} catch (error) {
|
|
ctx.ui.notify(`dankvoice: ${error instanceof Error ? error.message : String(error)}`, "error");
|
|
return;
|
|
}
|
|
ctx.ui.notify(
|
|
voices.length
|
|
? `dankvoice voices: ${voices.join(", ")} — using ${voiceLabel()}`
|
|
: "dankvoice: frank-tts reported no voice list",
|
|
voices.length ? "info" : "warning",
|
|
);
|
|
return;
|
|
}
|
|
case "status":
|
|
ctx.ui.notify(
|
|
`dankvoice: voice ${voiceLabel()} · ${cfg.target?.speechUrl ?? "NO ENDPOINT"} · player ${cfg.player ? cfg.player.command : "NONE"} · device ${cfg.sink || "system default"} · auto-speak ${state.auto ? "on" : "off"} · cap ${cfg.maxChars} chars`,
|
|
"info",
|
|
);
|
|
return;
|
|
case "speak":
|
|
void speak(ctx, action.text).catch((error) => ctx.ui.notify(`dankvoice: ${error instanceof Error ? error.message : String(error)}`, "error"));
|
|
return;
|
|
}
|
|
},
|
|
});
|
|
|
|
pi.on("agent_end", (event, ctx) => {
|
|
state.lastCtx = ctx;
|
|
if (!state.auto) return;
|
|
// `willContinue` means the harness already scheduled a continuation
|
|
// (auto-retry, empty-stop retry): this is not the user-visible terminal
|
|
// settle, so narrating it would speak an answer the model is about to redo.
|
|
if (event.willContinue) return;
|
|
if (ctx.mode === "print" || ctx.mode === "json") return;
|
|
// Fire-and-forget: a multi-segment utterance legitimately outlives the
|
|
// harness's handler timeout, and blocking here would only produce a
|
|
// spurious "handler timed out" toast while playback continues anyway.
|
|
void speak(ctx, lastAssistantText(event.messages)).catch((error) =>
|
|
ctx.ui.notify(`dankvoice: ${error instanceof Error ? error.message : String(error)}`, "error"),
|
|
);
|
|
});
|
|
|
|
pi.on("session_shutdown", () => {
|
|
stop();
|
|
});
|
|
}
|