dankvoice: segmented sequential speech (ported bundle vocalizer) #26
+355
-41
@@ -28,14 +28,21 @@
|
|||||||
* DANKVOICE_CONFIG settings file (default ~/.omp/cache/dankvoice.json)
|
* DANKVOICE_CONFIG settings file (default ~/.omp/cache/dankvoice.json)
|
||||||
* DANKVOICE_PLAYER player binary, skipping detection
|
* DANKVOICE_PLAYER player binary, skipping detection
|
||||||
* DANKVOICE_VOICE voice name ("" = server default)
|
* DANKVOICE_VOICE voice name ("" = server default)
|
||||||
* DANKVOICE_MAX spoken-character cap per utterance (default 700)
|
* DANKVOICE_MAX spoken-character budget per answer (default 700)
|
||||||
* DANKVOICE_AUTO=1 start with auto-speak armed
|
* DANKVOICE_AUTO=1 start with auto-speak armed
|
||||||
*
|
*
|
||||||
* Utterances are synthesized to a temp WAV in $TMPDIR; the file is removed
|
* Long answers are cut into speakable segments by a state machine ported
|
||||||
* when its player process exits. A new utterance interrupts the previous one
|
* from omp's built-in vocalizer: strong boundaries on [.!?…] runs followed
|
||||||
* instead of queueing behind it. Output routing: a saved sink becomes
|
* by whitespace (so "!!!!!" and "..." stay one boundary), guards for
|
||||||
* `pw-play --target <sink>` or `paplay --device=<sink>` (ffplay has no device
|
* abbreviations and decimals, code fences and tables swallowed, markdown
|
||||||
* concept and ignores it); with no saved sink the player's own default applies.
|
* cleaned. Each segment is synthesized to a temp WAV in $TMPDIR and played
|
||||||
|
* to completion before the next request goes out — requests are strictly
|
||||||
|
* sequential, so audio never stacks. The chain runs detached from the event
|
||||||
|
* handler, so an utterance may outlive the harness handler timeout. A new
|
||||||
|
* utterance interrupts the previous one. Output routing: a saved sink
|
||||||
|
* becomes `pw-play --target <sink>` or `paplay --device=<sink>` (ffplay has
|
||||||
|
* no device concept and ignores it); with no saved sink the player's own
|
||||||
|
* default applies.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { spawn, spawnSync, type ChildProcess } from "node:child_process";
|
import { spawn, spawnSync, type ChildProcess } from "node:child_process";
|
||||||
@@ -185,25 +192,305 @@ export function playerFor(command: string, sink: string): Player {
|
|||||||
return { command, args: ["--target", sink] };
|
return { command, args: ["--target", sink] };
|
||||||
}
|
}
|
||||||
|
|
||||||
/** What a speech engine should say: prose only, no fences, no markup, bounded. */
|
// ------------------------------------------------------------ segmenting
|
||||||
export function stripForSpeech(text: string, maxChars: number = DEFAULT_MAX_CHARS): string {
|
//
|
||||||
let out = text;
|
// Ported from omp's built-in speak-aloud vocalizer (the `fHe` segmenter and
|
||||||
out = out.replace(/```[\s\S]*?```/g, " ");
|
// its helpers in dist/cli.js). The same boundary rules as the stock path,
|
||||||
out = out.replace(/~~~[\s\S]*?~~~/g, " ");
|
// kept deliberately: push() takes text and returns the segments that are
|
||||||
out = out.replace(/`([^`]*)`/g, "$1");
|
// complete, flush() ends the message.
|
||||||
out = out.replace(/!\[[^\]]*\]\([^)]*\)/g, " ");
|
|
||||||
out = out.replace(/\[([^\]]*)\]\([^)]*\)/g, "$1");
|
const SENTENCE_END = /[.!?\u2026]+[)\]"'\u00BB\u201D\u2019]*\s/g;
|
||||||
out = out.replace(/^\s{0,3}#{1,6}\s+/gm, "");
|
const WEAK_BREAK = /[,;:\u2014\u2013]\s/g;
|
||||||
out = out.replace(/^\s*[-*+]\s+/gm, "");
|
const ABBREVIATION = /(?:^|\s)(?:e\.g|i\.e|etc|vs|Mr|Mrs|Ms|Dr|St|No)\.$/i;
|
||||||
out = out.replace(/^\s*\|[-: |]+\|\s*$/gm, " ");
|
const LINE_HEAD = /^(?:#{1,6}|[-*+]|-{2,}|\*{2,}|_{2,}|\d{1,3}|\d{1,3}[.)]|>+|`{1,2}|~{1,2})$/;
|
||||||
out = out.replace(/\|/g, " ");
|
const HORIZONTAL_RULE = /^(?:-{3,}|\*{3,}|_{3,})\s*$/;
|
||||||
out = out.replace(/[*_]{1,3}([^*_]+)[*_]{1,3}/g, "$1");
|
|
||||||
out = out.replace(/\s+/g, " ").trim();
|
const IMAGE_LINK = /!\[([^\]]*)\]\(([^()]*)\)/g;
|
||||||
if (out.length <= maxChars) return out;
|
const LINK = /\[([^\]]+)\]\(([^()]*)\)/g;
|
||||||
const window = out.slice(0, maxChars);
|
const BRACKETED_URL = /<(https?:\/\/[^\s>]+)>/g;
|
||||||
const cut = Math.max(window.lastIndexOf(". "), window.lastIndexOf("! "), window.lastIndexOf("? "));
|
const BARE_URL = /\bhttps?:\/\/[^\s<>()"'\]]+|\bwww\.[\w-]+(?:\.[\w-]+)+[^\s<>()"'\]]*/g;
|
||||||
const clipped = cut > maxChars / 2 ? window.slice(0, cut + 1) : window.replace(/[,;:]\s*\S*$/, "");
|
const INLINE_CODE = /`{1,2}([^`]+)`{1,2}/g;
|
||||||
return `${clipped.trim()} …`;
|
const BOLD_MARKUP = /\*\*|__|~~/g;
|
||||||
|
const STAR = /\*(?=\S)|(?<=\S)\*/g;
|
||||||
|
const UNDERSCORE = /(^|\s)_+|_+(?=\s|$)/g;
|
||||||
|
const HTML_TAG = /<\/?[a-zA-Z][^<>]*>/g;
|
||||||
|
const EMPHASIS_RUN = /(^|\s)[-*_]{3,}(?=\s|$)/g;
|
||||||
|
const DEEP_PATH = /(^|[\s("'`])((?:~|\.{1,2})?\/?[\w.@+-]+(?:\/[\w.@+-]+){2,}\/?)/g;
|
||||||
|
const HAS_WORD = /[\p{L}\p{N}]/u;
|
||||||
|
|
||||||
|
/** Speak a URL by host and a file path by its last segment. */
|
||||||
|
export function spokenDomain(url: string): string {
|
||||||
|
return url.replace(/^[a-z][\w+.-]*:\/\//i, "").replace(/^www\./i, "").replace(/[/?#].*$/, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Final per-segment cleanup; empty when nothing speakable survived. */
|
||||||
|
export function cleanForSpeech(text: string): string {
|
||||||
|
const out = text
|
||||||
|
.replace(IMAGE_LINK, "$1")
|
||||||
|
.replace(LINK, "$1")
|
||||||
|
.replace(BRACKETED_URL, (_match, url: string) => spokenDomain(url))
|
||||||
|
.replace(BARE_URL, (url: string) => spokenDomain(url))
|
||||||
|
.replace(INLINE_CODE, "$1")
|
||||||
|
.replace(BOLD_MARKUP, "")
|
||||||
|
.replace(STAR, "")
|
||||||
|
.replace(UNDERSCORE, "$1")
|
||||||
|
.replace(HTML_TAG, " ")
|
||||||
|
.replace(EMPHASIS_RUN, "$1")
|
||||||
|
.replace(DEEP_PATH, (_match, pre: string, path: string) => {
|
||||||
|
const parts = path.split("/").filter((part) => part.length > 0);
|
||||||
|
return pre + (parts[parts.length - 1] ?? path);
|
||||||
|
})
|
||||||
|
.replace(/\s+/g, " ")
|
||||||
|
.trim();
|
||||||
|
return HAS_WORD.test(out) ? out : "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function sentenceBoundary(text: string, minChars: number): number {
|
||||||
|
SENTENCE_END.lastIndex = 0;
|
||||||
|
for (let match = SENTENCE_END.exec(text); match; match = SENTENCE_END.exec(text)) {
|
||||||
|
const end = match.index + match[0].length;
|
||||||
|
if (end < minChars) continue;
|
||||||
|
if (ABBREVIATION.test(text.slice(0, end).trimEnd())) continue;
|
||||||
|
if ((text.slice(0, end).match(/`/g)?.length ?? 0) % 2 !== 0) continue;
|
||||||
|
return end;
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
function firstWeakBreak(text: string, minChars: number): number {
|
||||||
|
WEAK_BREAK.lastIndex = 0;
|
||||||
|
for (let match = WEAK_BREAK.exec(text); match; match = WEAK_BREAK.exec(text)) {
|
||||||
|
const end = match.index + match[0].length;
|
||||||
|
if (end >= minChars) return end;
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
function lastWeakBreakWithin(text: string, minChars: number, maxChars: number): number {
|
||||||
|
WEAK_BREAK.lastIndex = 0;
|
||||||
|
let found = -1;
|
||||||
|
for (let match = WEAK_BREAK.exec(text); match; match = WEAK_BREAK.exec(text)) {
|
||||||
|
const end = match.index + match[0].length;
|
||||||
|
if (end > maxChars) break;
|
||||||
|
if (end >= minChars) found = end;
|
||||||
|
}
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
|
||||||
|
function wordWrapCut(text: string, maxChars: number): number {
|
||||||
|
const space = text.lastIndexOf(" ", maxChars);
|
||||||
|
return space > 0 ? space + 1 : Math.min(maxChars, text.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
type LineHead =
|
||||||
|
| { kind: "swallow" }
|
||||||
|
| { kind: "fence"; fence: string }
|
||||||
|
| { kind: "marker"; spoken: string }
|
||||||
|
| { kind: "prose"; text: string }
|
||||||
|
| { kind: "undecided" };
|
||||||
|
|
||||||
|
/** Classify a line start: tables and fences swallow, list markers get spoken. */
|
||||||
|
export function classifyLineHead(head: string): LineHead {
|
||||||
|
if (head === "|") return { kind: "swallow" };
|
||||||
|
if (/^(?:`{3}|~{3})/.test(head)) return { kind: "fence", fence: head.slice(0, 3) };
|
||||||
|
if (/^#{1,6}[ \t]/.test(head)) return { kind: "marker", spoken: "" };
|
||||||
|
if (/^[-*+][ \t]/.test(head)) return { kind: "marker", spoken: "" };
|
||||||
|
const ordered = /^(\d{1,3})[.)][ \t]/.exec(head);
|
||||||
|
if (ordered) return { kind: "marker", spoken: `${ordered[1]}, ` };
|
||||||
|
if (/^>+/.test(head) && !/^>+$/.test(head)) return { kind: "prose", text: head.replace(/^>+[ \t]?/, "") };
|
||||||
|
if (LINE_HEAD.test(head)) return { kind: "undecided" };
|
||||||
|
return { kind: "prose", text: head };
|
||||||
|
}
|
||||||
|
|
||||||
|
type SegmenterState = "linestart" | "prose" | "swallow" | "code";
|
||||||
|
|
||||||
|
/** The built-in vocalizer's segmenter: streamed text in, speakable segments out. */
|
||||||
|
export class SpeechSegmenter {
|
||||||
|
#state: SegmenterState = "linestart";
|
||||||
|
#head = "";
|
||||||
|
#fence = "";
|
||||||
|
#fenceLine = "";
|
||||||
|
#resume: SegmenterState = "linestart";
|
||||||
|
#buffer = "";
|
||||||
|
#started = false;
|
||||||
|
|
||||||
|
push(chunk: string): string[] {
|
||||||
|
const out: string[] = [];
|
||||||
|
for (const char of chunk) this.#feed(char, out);
|
||||||
|
this.#split(out);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
flush(): string[] {
|
||||||
|
const out: string[] = [];
|
||||||
|
if (this.#state === "linestart" && this.#head.length > 0 && !HORIZONTAL_RULE.test(this.#head)) this.#buffer += this.#head;
|
||||||
|
this.#head = "";
|
||||||
|
this.#state = "linestart";
|
||||||
|
this.#emit(out);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Streaming idle flush: only cut a tail that already looks finished. */
|
||||||
|
flushIdle(): string[] {
|
||||||
|
const out: string[] = [];
|
||||||
|
const tail = this.#buffer.trimEnd();
|
||||||
|
if (!/[.!?\u2026][)\]"'\u00BB\u201D\u2019]*$/.test(tail) && tail.length < 24) return out;
|
||||||
|
this.#emit(out);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
#feed(char: string, out: string[]): void {
|
||||||
|
switch (this.#state) {
|
||||||
|
case "linestart":
|
||||||
|
this.#feedLineStart(char, out);
|
||||||
|
return;
|
||||||
|
case "prose":
|
||||||
|
if (char === "\n") this.#endLine(out);
|
||||||
|
else this.#buffer += char;
|
||||||
|
return;
|
||||||
|
case "swallow":
|
||||||
|
if (char === "\n") this.#state = this.#resume;
|
||||||
|
return;
|
||||||
|
case "code":
|
||||||
|
this.#feedCode(char);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#feedLineStart(char: string, out: string[]): void {
|
||||||
|
if (char === "\n") {
|
||||||
|
const head = this.#head;
|
||||||
|
this.#head = "";
|
||||||
|
if (head.length > 0 && !HORIZONTAL_RULE.test(head)) this.#buffer += head;
|
||||||
|
this.#endLine(out);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.#head += char;
|
||||||
|
const classified = classifyLineHead(this.#head);
|
||||||
|
if (classified.kind === "undecided") {
|
||||||
|
if (this.#head.length > 8) {
|
||||||
|
this.#buffer += this.#head;
|
||||||
|
this.#head = "";
|
||||||
|
this.#state = "prose";
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.#head = "";
|
||||||
|
switch (classified.kind) {
|
||||||
|
case "prose":
|
||||||
|
this.#buffer += classified.text;
|
||||||
|
this.#state = "prose";
|
||||||
|
return;
|
||||||
|
case "marker":
|
||||||
|
this.#buffer += classified.spoken;
|
||||||
|
this.#state = "prose";
|
||||||
|
return;
|
||||||
|
case "swallow":
|
||||||
|
this.#state = "swallow";
|
||||||
|
this.#resume = "linestart";
|
||||||
|
return;
|
||||||
|
case "fence":
|
||||||
|
this.#fence = classified.fence;
|
||||||
|
this.#fenceLine = "";
|
||||||
|
this.#state = "swallow";
|
||||||
|
this.#resume = "code";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#feedCode(char: string): void {
|
||||||
|
if (char === "\n") {
|
||||||
|
this.#fenceLine = "";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (this.#fenceLine.length < 3) {
|
||||||
|
this.#fenceLine += char;
|
||||||
|
if (this.#fenceLine === this.#fence) {
|
||||||
|
this.#state = "swallow";
|
||||||
|
this.#resume = "linestart";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#endLine(out: string[]): void {
|
||||||
|
this.#state = "linestart";
|
||||||
|
this.#emit(out);
|
||||||
|
}
|
||||||
|
|
||||||
|
#emit(out: string[]): void {
|
||||||
|
this.#split(out);
|
||||||
|
const segment = this.#buffer;
|
||||||
|
this.#buffer = "";
|
||||||
|
this.#pushSegment(segment, out);
|
||||||
|
}
|
||||||
|
|
||||||
|
#split(out: string[]): void {
|
||||||
|
for (;;) {
|
||||||
|
const buffer = this.#buffer;
|
||||||
|
const boundary = sentenceBoundary(buffer, this.#started ? 24 : 12);
|
||||||
|
if (boundary !== -1 && boundary <= 280) {
|
||||||
|
this.#cut(boundary, out);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!this.#started && buffer.length >= 40) {
|
||||||
|
const weak = firstWeakBreak(buffer, 12);
|
||||||
|
if (weak !== -1 && weak <= 140) {
|
||||||
|
this.#cut(weak, out);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (buffer.length >= 140) {
|
||||||
|
this.#cut(wordWrapCut(buffer, 140), out);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (this.#started && buffer.length >= 160) {
|
||||||
|
const weak = lastWeakBreakWithin(buffer, 24, 160);
|
||||||
|
if (weak !== -1) {
|
||||||
|
this.#cut(weak, out);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (buffer.length > 280) {
|
||||||
|
const weak = lastWeakBreakWithin(buffer, 24, 280);
|
||||||
|
this.#cut(weak !== -1 ? weak : wordWrapCut(buffer, 280), out);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#cut(end: number, out: string[]): void {
|
||||||
|
const head = this.#buffer.slice(0, end);
|
||||||
|
this.#buffer = this.#buffer.slice(end);
|
||||||
|
this.#pushSegment(head, out);
|
||||||
|
}
|
||||||
|
|
||||||
|
#pushSegment(raw: string, out: string[]): void {
|
||||||
|
const cleaned = cleanForSpeech(raw);
|
||||||
|
if (!cleaned) return;
|
||||||
|
out.push(cleaned);
|
||||||
|
this.#started = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A whole message to speakable segments: the segmenter run start to end. */
|
||||||
|
export function segmentsForSpeech(text: string): string[] {
|
||||||
|
const segmenter = new SpeechSegmenter();
|
||||||
|
const segments = segmenter.push(text);
|
||||||
|
for (const tail of segmenter.flush()) segments.push(tail);
|
||||||
|
return segments;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The leading whole segments that fit the user's spoken-character budget. */
|
||||||
|
export function budgetSegments(segments: string[], maxChars: number): string[] {
|
||||||
|
const chosen: string[] = [];
|
||||||
|
let used = 0;
|
||||||
|
for (const segment of segments) {
|
||||||
|
if (used + segment.length > maxChars) break;
|
||||||
|
used += segment.length;
|
||||||
|
chosen.push(segment);
|
||||||
|
}
|
||||||
|
return chosen;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The text the assistant last actually said; tool-only messages do not count. */
|
/** The text the assistant last actually said; tool-only messages do not count. */
|
||||||
@@ -408,14 +695,16 @@ function startPlayback(player: Player, file: string, dir: string): ChildProcess
|
|||||||
|
|
||||||
export default function dankvoice(pi: ExtensionAPI) {
|
export default function dankvoice(pi: ExtensionAPI) {
|
||||||
const cfg = loadConfig();
|
const cfg = loadConfig();
|
||||||
const state: { auto: boolean; voice: string; child?: ChildProcess; lastCtx?: ExtensionContext } = {
|
const state: { auto: boolean; voice: string; child?: ChildProcess; run: number; lastCtx?: ExtensionContext } = {
|
||||||
auto: cfg.auto,
|
auto: cfg.auto,
|
||||||
voice: cfg.voice,
|
voice: cfg.voice,
|
||||||
|
run: 0,
|
||||||
};
|
};
|
||||||
const voiceLabel = () => state.voice || "server default";
|
const voiceLabel = () => state.voice || "server default";
|
||||||
const reflectStatus = () => state.lastCtx?.ui.setStatus("dankvoice", state.auto ? `🔊 ${voiceLabel()}` : undefined);
|
const reflectStatus = () => state.lastCtx?.ui.setStatus("dankvoice", state.auto ? `🔊 ${voiceLabel()}` : undefined);
|
||||||
|
|
||||||
const stop = () => {
|
const stop = () => {
|
||||||
|
state.run += 1; // retire the live chain; its loop stops at the next gate
|
||||||
if (state.child && state.child.exitCode === null) state.child.kill("SIGTERM");
|
if (state.child && state.child.exitCode === null) state.child.kill("SIGTERM");
|
||||||
state.child = undefined;
|
state.child = undefined;
|
||||||
};
|
};
|
||||||
@@ -455,9 +744,12 @@ export default function dankvoice(pi: ExtensionAPI) {
|
|||||||
persistSettings(ctx);
|
persistSettings(ctx);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Speak one answer as strictly sequential requests: synthesize a segment,
|
||||||
|
* play it to the end, only then ask for the next — audio never stacks.
|
||||||
|
* Superseded chains (new utterance, /say stop, disarm) exit at the gates.
|
||||||
|
*/
|
||||||
async function speak(ctx: ExtensionContext, text: string) {
|
async function speak(ctx: ExtensionContext, text: string) {
|
||||||
const spoken = stripForSpeech(text, cfg.maxChars);
|
|
||||||
if (!spoken) return;
|
|
||||||
if (!cfg.target) {
|
if (!cfg.target) {
|
||||||
ctx.ui.notify(missingTarget(), "error");
|
ctx.ui.notify(missingTarget(), "error");
|
||||||
return;
|
return;
|
||||||
@@ -466,19 +758,36 @@ export default function dankvoice(pi: ExtensionAPI) {
|
|||||||
ctx.ui.notify("dankvoice: no audio player found (pw-play, paplay or ffplay)", "error");
|
ctx.ui.notify("dankvoice: no audio player found (pw-play, paplay or ffplay)", "error");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
const target = cfg.target;
|
||||||
|
const player = cfg.player;
|
||||||
|
const segments = budgetSegments(segmentsForSpeech(text), cfg.maxChars);
|
||||||
|
if (segments.length === 0) return;
|
||||||
stop();
|
stop();
|
||||||
const dir = mkdtempSync(join(tmpdir(), "dankvoice-"));
|
const run = state.run;
|
||||||
try {
|
for (const segment of segments) {
|
||||||
const file = await synthesize(cfg.target, speechBody(cfg.target.model, spoken, state.voice), dir, SYNTH_TIMEOUT_MS);
|
if (state.run !== run) return;
|
||||||
state.child = startPlayback(cfg.player, file, dir);
|
const dir = mkdtempSync(join(tmpdir(), "dankvoice-"));
|
||||||
} catch (error) {
|
|
||||||
stop();
|
|
||||||
try {
|
try {
|
||||||
rmSync(dir, { recursive: true, force: true });
|
const file = await synthesize(target, speechBody(target.model, segment, state.voice), dir, SYNTH_TIMEOUT_MS);
|
||||||
} catch {
|
if (state.run !== run) {
|
||||||
// ignore
|
rmSync(dir, { recursive: true, force: true });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
const child = startPlayback(player, file, dir);
|
||||||
|
state.child = child;
|
||||||
|
child.once("exit", () => resolve());
|
||||||
|
child.once("error", () => resolve());
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
try {
|
||||||
|
rmSync(dir, { recursive: true, force: true });
|
||||||
|
} catch {
|
||||||
|
// temp files are not worth a toast
|
||||||
|
}
|
||||||
|
ctx.ui.notify(`dankvoice: ${error instanceof Error ? error.message : String(error)}`, "error");
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
ctx.ui.notify(`dankvoice: ${error instanceof Error ? error.message : String(error)}`, "error");
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -603,13 +912,13 @@ export default function dankvoice(pi: ExtensionAPI) {
|
|||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
case "speak":
|
case "speak":
|
||||||
await speak(ctx, action.text);
|
void speak(ctx, action.text).catch((error) => ctx.ui.notify(`dankvoice: ${error instanceof Error ? error.message : String(error)}`, "error"));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
pi.on("agent_end", async (event, ctx) => {
|
pi.on("agent_end", (event, ctx) => {
|
||||||
state.lastCtx = ctx;
|
state.lastCtx = ctx;
|
||||||
if (!state.auto) return;
|
if (!state.auto) return;
|
||||||
// `willContinue` means the harness already scheduled a continuation
|
// `willContinue` means the harness already scheduled a continuation
|
||||||
@@ -617,7 +926,12 @@ export default function dankvoice(pi: ExtensionAPI) {
|
|||||||
// settle, so narrating it would speak an answer the model is about to redo.
|
// settle, so narrating it would speak an answer the model is about to redo.
|
||||||
if (event.willContinue) return;
|
if (event.willContinue) return;
|
||||||
if (ctx.mode === "print" || ctx.mode === "json") return;
|
if (ctx.mode === "print" || ctx.mode === "json") return;
|
||||||
await speak(ctx, lastAssistantText(event.messages));
|
// Fire-and-forget: a multi-segment utterance legitimately outlives the
|
||||||
|
// harness's handler timeout, and blocking here would only produce a
|
||||||
|
// spurious "handler timed out" toast while playback continues anyway.
|
||||||
|
void speak(ctx, lastAssistantText(event.messages)).catch((error) =>
|
||||||
|
ctx.ui.notify(`dankvoice: ${error instanceof Error ? error.message : String(error)}`, "error"),
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
pi.on("session_shutdown", () => {
|
pi.on("session_shutdown", () => {
|
||||||
|
|||||||
@@ -9,8 +9,8 @@ tags:
|
|||||||
- voice
|
- voice
|
||||||
- frank-tts
|
- frank-tts
|
||||||
- dankclaw
|
- dankclaw
|
||||||
verified: 2026-10-18
|
verified: 2026-10-19
|
||||||
timestamp: 2026-10-18
|
timestamp: 2026-10-19
|
||||||
---
|
---
|
||||||
|
|
||||||
# dankvoice
|
# dankvoice
|
||||||
@@ -54,9 +54,38 @@ player and device — the audible proof whenever a session "has say on" but
|
|||||||
stays silent.
|
stays silent.
|
||||||
|
|
||||||
Auto-speak is skipped in `print`/`json` mode so benches and pipelines stay
|
Auto-speak is skipped in `print`/`json` mode so benches and pipelines stay
|
||||||
silent. Markdown is stripped before synthesis — fenced code is dropped, links
|
silent. A long answer is never one monolithic synthesis request; see
|
||||||
collapse to their label, bullets and table pipes disappear — and the utterance
|
[Segmentation](#segmentation) for how utterances are cut and played.
|
||||||
is truncated at a sentence boundary inside the cap.
|
|
||||||
|
## Segmentation
|
||||||
|
|
||||||
|
Utterances are cut by a state machine ported from the harness's own
|
||||||
|
speak-aloud vocalizer (the minified `fHe` class in the agent bundle's
|
||||||
|
`dist/cli.js`), kept deliberately in lockstep with the stock path:
|
||||||
|
|
||||||
|
* A strong boundary is a run of `[.!?…]` followed by whitespace — so
|
||||||
|
`!!!!!` and `...` are one boundary, never a pile of cuts.
|
||||||
|
* Boundaries after `e.g.`, `etc.`, `Mr.` and friends, or inside inline
|
||||||
|
code, don't count; decimals (`3.5`, `1,5`) survive because a boundary
|
||||||
|
requires trailing whitespace.
|
||||||
|
* Line starts are classified: table rows and fenced code are swallowed
|
||||||
|
whole (their content is never read aloud), heading and bullet markers
|
||||||
|
vanish, ordered-list markers speak their number ("1, …"), quote markers
|
||||||
|
strip.
|
||||||
|
* Per segment, markdown is cleaned: links collapse to their label, URLs
|
||||||
|
to their host, deep paths to their last segment; emphasis markup dies.
|
||||||
|
* Segments aim for one sentence; run-ons break at commas or word wraps
|
||||||
|
before 280 characters.
|
||||||
|
|
||||||
|
Playback is strictly sequential: segment N+1 is synthesized only after
|
||||||
|
segment N's player process exited, so audio never stacks. The chain runs
|
||||||
|
detached from `agent_end` (fire-and-forget) and may legitimately outlive
|
||||||
|
the harness's 30 s handler timeout — the old awaiting design produced a
|
||||||
|
"handler timed out after 30000ms" toast on long answers. A new utterance,
|
||||||
|
`/say stop`, or disarming auto-speak retires the live chain through a run
|
||||||
|
token: the current player is killed and pending segments are skipped.
|
||||||
|
`DANKVOICE_MAX` is a per-answer character budget: the leading whole
|
||||||
|
segments that fit are spoken, the rest are dropped.
|
||||||
|
|
||||||
## Endpoint resolution
|
## Endpoint resolution
|
||||||
|
|
||||||
@@ -73,7 +102,7 @@ resolves, so no host address and no credential appears in this file. From
|
|||||||
| `DANKVOICE_CONFIG` | `~/.omp/cache/dankvoice.json` | persisted menu choices (voice, player, sink, cap, auto) |
|
| `DANKVOICE_CONFIG` | `~/.omp/cache/dankvoice.json` | persisted menu choices (voice, player, sink, cap, auto) |
|
||||||
| `DANKVOICE_VOICE` | empty (server default) | default voice name |
|
| `DANKVOICE_VOICE` | empty (server default) | default voice name |
|
||||||
| `DANKVOICE_AUTO=1` | off | start with auto-speak armed |
|
| `DANKVOICE_AUTO=1` | off | start with auto-speak armed |
|
||||||
| `DANKVOICE_MAX` | `700` | spoken characters per utterance |
|
| `DANKVOICE_MAX` | `700` | spoken character budget per answer (whole leading segments) |
|
||||||
| `DANKVOICE_PLAYER` | detect | player binary, skipping detection |
|
| `DANKVOICE_PLAYER` | detect | player binary, skipping detection |
|
||||||
| `DANKVOICE_MODELS` | `~/.omp/agent/models.yml` | YAML to read the endpoint from |
|
| `DANKVOICE_MODELS` | `~/.omp/agent/models.yml` | YAML to read the endpoint from |
|
||||||
|
|
||||||
@@ -86,7 +115,7 @@ if wireplumber is unreachable the menu offers only "system default". Each
|
|||||||
utterance is written to `$TMPDIR/dankvoice-*/say-<ms>.wav` and the file plus
|
utterance is written to `$TMPDIR/dankvoice-*/say-<ms>.wav` and the file plus
|
||||||
its directory are removed when the player process exits.
|
its directory are removed when the player process exits.
|
||||||
|
|
||||||
## Verified facts (2026-10-18)
|
## Verified facts (2026-10-18, 2026-10-19)
|
||||||
|
|
||||||
* **Extension discovery** — any `.ts` file in `~/.omp/agent/extensions/` loads
|
* **Extension discovery** — any `.ts` file in `~/.omp/agent/extensions/` loads
|
||||||
at startup: a probe file there printed `loaded 2026-10-19T02:36:55Z` from a
|
at startup: a probe file there printed `loaded 2026-10-19T02:36:55Z` from a
|
||||||
@@ -118,6 +147,12 @@ its directory are removed when the player process exits.
|
|||||||
menu is used by later utterances and survives a fresh load via the settings
|
menu is used by later utterances and survives a fresh load via the settings
|
||||||
file; a device chosen in the menu appears in the player invocation as
|
file; a device chosen in the menu appears in the player invocation as
|
||||||
`--target <sink>`.
|
`--target <sink>`.
|
||||||
|
* **Segmented playback** — driver scenarios 2026-10-19: "First stop here.
|
||||||
|
Second one goes!!! Really? Third ... done" produced exactly 3 sequential
|
||||||
|
requests and 3 plays; "Model 3.5 ships today. Version 1,5 too." produced
|
||||||
|
exactly 2 (decimals never split); the markdown fixture's fenced code and
|
||||||
|
URL never reached the fixture endpoint; with `DANKVOICE_MAX=60` only the
|
||||||
|
leading segment was spoken. `bats tests/dankvoice.bats` 17/17.
|
||||||
|
|
||||||
## Limits
|
## Limits
|
||||||
|
|
||||||
@@ -141,6 +176,12 @@ its directory are removed when the player process exits.
|
|||||||
* `omp --help` (extension discovery flags, `say`, `plugin`) and
|
* `omp --help` (extension discovery flags, `say`, `plugin`) and
|
||||||
`@earendil-works/pi-coding-agent` `dist/core/extensions/types.d.ts`
|
`@earendil-works/pi-coding-agent` `dist/core/extensions/types.d.ts`
|
||||||
(`ExtensionAPI.on/registerCommand`, `ExtensionMode = "tui" | "rpc" | "json" | "print"`).
|
(`ExtensionAPI.on/registerCommand`, `ExtensionMode = "tui" | "rpc" | "json" | "print"`).
|
||||||
|
* The stock speak-aloud implementation — vocalizer pipeline `I6o` and
|
||||||
|
utterance segmenter `fHe` — in `@oh-my-pi/pi-coding-agent`'s
|
||||||
|
`dist/cli.js`, read off the minified bundle 2026-10-19; the enhanced
|
||||||
|
rewrite ("with the tiny/smol model before synthesis") is that bundle's
|
||||||
|
own `speech.enhanced` description. dankvoice ports the mechanical half
|
||||||
|
only: no LLM rewrite happens before frank-tts.
|
||||||
* Gitea issue #21 on `dank/command-center` (the work item this page ships with).
|
* Gitea issue #21 on `dank/command-center` (the work item this page ships with).
|
||||||
* Gitea issue #23 on `dank/command-center` (the options-menu work item).
|
* Gitea issue #23 on `dank/command-center` (the options-menu work item).
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,29 @@ timestamp: 2026-08-28
|
|||||||
# Bundle Change Log
|
# Bundle Change Log
|
||||||
|
|
||||||
|
|
||||||
|
## 2026-10-19 — dankvoice: segmented sequential speech, copied from the bundle's vocalizer
|
||||||
|
|
||||||
|
Silent-answer incident follow-up: `agent_end` awaiting one whole-answer
|
||||||
|
synthesis hit the harness's 30 s extension-handler timeout
|
||||||
|
(`handler timed out after 30000ms`) on long answers, and the single frank-tts
|
||||||
|
request for the entire answer was the stall. Instead of inventing a heuristic,
|
||||||
|
ported the utterance-splitting state machine out of the built-in speak-aloud
|
||||||
|
vocalizer (minified class `fHe` in `dist/cli.js`): strong boundaries are
|
||||||
|
`[.!?…]+` runs followed by whitespace — `!!!!!`/`...` stay one cut — with
|
||||||
|
abbreviation and inline-code guards; `3.5`/`1,5` survive because a boundary
|
||||||
|
needs trailing whitespace; fenced code and tables swallow whole; headings and
|
||||||
|
bullets vanish, ordered markers speak their number ("1, "); links collapse to
|
||||||
|
labels, URLs to hosts, paths to basenames; run-ons break at commas or word
|
||||||
|
wraps before 280 chars. `speak()` now synthesizes segment N+1 only after
|
||||||
|
segment N's player exited — strictly sequential, audio never stacks — and
|
||||||
|
`agent_end`/`/say <text>` fire-and-forget a chain retired through a run token
|
||||||
|
by `/say stop`, disarming, or a newer utterance. `DANKVOICE_MAX` became a
|
||||||
|
per-answer budget of whole leading segments; the hand-rolled `stripForSpeech`
|
||||||
|
truncator is gone. No LLM "humanization" rewrite happens (the bundle's
|
||||||
|
`speech.enhanced` rewrites with the tiny/smol model; dankvoice speaks
|
||||||
|
mechanically cleaned text through the cloned voices). Tests: two new driver
|
||||||
|
scenarios plus two rewrites, 17/17 in `tests/dankvoice.bats`.
|
||||||
|
|
||||||
## 2026-10-18 — dankvoice: `/say` options menu with output-device routing (Refs #23)
|
## 2026-10-18 — dankvoice: `/say` options menu with output-device routing (Refs #23)
|
||||||
|
|
||||||
Added `/say menu` (bare `/say` now opens it; `help` keeps the text listing), the
|
Added `/say menu` (bare `/say` now opens it; `help` keeps the text listing), the
|
||||||
|
|||||||
+31
-4
@@ -189,6 +189,12 @@ if (!say) {
|
|||||||
case "truncate":
|
case "truncate":
|
||||||
await say.handler("Sentence one is here. ".repeat(40), ctx);
|
await say.handler("Sentence one is here. ".repeat(40), ctx);
|
||||||
break;
|
break;
|
||||||
|
case "sentences":
|
||||||
|
await say.handler("First stop here. Second one goes!!! Really? Third ... done", ctx);
|
||||||
|
break;
|
||||||
|
case "decimals":
|
||||||
|
await say.handler("Model 3.5 ships today. Version 1,5 too.", ctx);
|
||||||
|
break;
|
||||||
case "noplayer":
|
case "noplayer":
|
||||||
await say.handler("speak anyway", ctx);
|
await say.handler("speak anyway", ctx);
|
||||||
break;
|
break;
|
||||||
@@ -280,13 +286,34 @@ say_run() {
|
|||||||
|
|
||||||
@test "markdown never reaches the speech engine" {
|
@test "markdown never reaches the speech engine" {
|
||||||
say_run markdown
|
say_run markdown
|
||||||
[[ "$output" == *"input=Title Use pw-play here. see docs and bold a b"* ]]
|
[[ "$output" == *"input=Title"* ]]
|
||||||
|
[[ "$output" == *"input=Use pw-play here."* ]]
|
||||||
|
[[ "$output" == *"input=see docs and bold"* ]]
|
||||||
|
[[ "$output" != *"npm i x"* ]]
|
||||||
|
[[ "$output" != *"example.test"* ]]
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "an over-long utterance is bounded and cut at a sentence boundary" {
|
@test "the spoken budget keeps only the leading whole segments" {
|
||||||
DANKVOICE_MAX=60 say_run truncate
|
DANKVOICE_MAX=60 say_run truncate
|
||||||
[[ "$output" == *"here. …"* ]]
|
[[ "$output" == *"CALLS 1"* ]]
|
||||||
[ "$(grep -o 'Sentence one is here\.' <<<"$output" | wc -l)" -le 4 ]
|
[[ "$output" == *"PLAYS 1"* ]]
|
||||||
|
[[ "$output" == *"input=Sentence one is here."* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "a long answer is spoken as sequential sentence-sized requests" {
|
||||||
|
say_run sentences
|
||||||
|
[[ "$output" == *"CALLS 3"* ]]
|
||||||
|
[[ "$output" == *"PLAYS 3"* ]]
|
||||||
|
[[ "$output" == *"input=First stop here."* ]]
|
||||||
|
[[ "$output" == *"input=Second one goes!!! Really?"* ]]
|
||||||
|
[[ "$output" == *"input=Third ... done"* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
@test "decimals and comma-decimals never split a sentence" {
|
||||||
|
say_run decimals
|
||||||
|
[[ "$output" == *"CALLS 2"* ]]
|
||||||
|
[[ "$output" == *"input=Model 3.5 ships today."* ]]
|
||||||
|
[[ "$output" == *"input=Version 1,5 too."* ]]
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "a missing player is reported, not swallowed" {
|
@test "a missing player is reported, not swallowed" {
|
||||||
|
|||||||
Reference in New Issue
Block a user