legop3/MultiRoombaRover
1
0
Fork 0
mirror of https://github.com/legop3/MultiRoombaRover.git synced 2026-09-15 17:12:59 -04:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

server generated scanner voices!

Browse Source
This commit is contained in:
legop3
2026-06-13 17:53:52 -04:00
parent bddff1e67f
commit 61ace6e7c4
18 changed files with 323 additions and 3282 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webui/package-lock.json
Generated
+4 -150
View File
@@ -8,7 +8,6 @@
"name": "webui",
"version": "0.0.0",
"dependencies": {
"@mintplex-labs/piper-tts-web": "^1.0.4",
"react": "^19.2.0",
"react-dom": "^19.2.0",
"react-icons": "^5.5.0",
@@ -1046,15 +1045,6 @@
"@jridgewell/sourcemap-codec": "^1.4.14"
}
},
"node_modules/@mintplex-labs/piper-tts-web": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@mintplex-labs/piper-tts-web/-/piper-tts-web-1.0.4.tgz",
"integrity": "sha512-Y24X+CJaGXoY5HFPSstHvJI6408OAtw3Pmq2OIYwpRpcwLLbgadWg8l1ODHNkgpB0Ps5fS9PAAQB60fHA3Bdag==",
"license": "MIT",
"peerDependencies": {
"onnxruntime-web": "^1.18.0"
}
},
"node_modules/@nodelib/fs.scandir": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -1104,72 +1094,6 @@
"node": ">=14"
}
},
"node_modules/@protobufjs/aspromise": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
"integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==",
"license": "BSD-3-Clause",
"peer": true
},
"node_modules/@protobufjs/base64": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
"integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==",
"license": "BSD-3-Clause",
"peer": true
},
"node_modules/@protobufjs/codegen": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz",
"integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==",
"license": "BSD-3-Clause",
"peer": true
},
"node_modules/@protobufjs/eventemitter": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz",
"integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==",
"license": "BSD-3-Clause",
"peer": true
},
"node_modules/@protobufjs/fetch": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz",
"integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==",
"license": "BSD-3-Clause",
"peer": true,
"dependencies": {
"@protobufjs/aspromise": "^1.1.1"
}
},
"node_modules/@protobufjs/float": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
"integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==",
"license": "BSD-3-Clause",
"peer": true
},
"node_modules/@protobufjs/path": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
"integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==",
"license": "BSD-3-Clause",
"peer": true
},
"node_modules/@protobufjs/pool": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
"integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==",
"license": "BSD-3-Clause",
"peer": true
},
"node_modules/@protobufjs/utf8": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz",
"integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==",
"license": "BSD-3-Clause",
"peer": true
},
"node_modules/@rolldown/pluginutils": {
"version": "1.0.0-beta.47",
"resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.47.tgz",
@@ -1555,7 +1479,9 @@
"version": "25.9.3",
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.3.tgz",
"integrity": "sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg==",
"dev": true,
"license": "MIT",
"optional": true,
"peer": true,
"dependencies": {
"undici-types": ">=7.24.0 <7.24.7"
@@ -2498,13 +2424,6 @@
"node": ">=16"
}
},
"node_modules/flatbuffers": {
"version": "25.9.23",
"resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz",
"integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==",
"license": "Apache-2.0",
"peer": true
},
"node_modules/flatted": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz",
@@ -2651,13 +2570,6 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/guid-typescript": {
"version": "1.0.9",
"resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz",
"integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==",
"license": "ISC",
"peer": true
},
"node_modules/has-flag": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
@@ -2971,13 +2883,6 @@
"dev": true,
"license": "MIT"
},
"node_modules/long": {
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz",
"integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==",
"license": "Apache-2.0",
"peer": true
},
"node_modules/lru-cache": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
@@ -3140,28 +3045,6 @@
"node": ">= 6"
}
},
"node_modules/onnxruntime-common": {
"version": "1.26.0",
"resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.26.0.tgz",
"integrity": "sha512-qVyMR4lcWgbkc4getFV+GQijsTnbg/siteoqcDwa3sI/LxbrMSNw4ePyvCq/ymdQaRomCA7YuWmhzsswxvymdw==",
"license": "MIT",
"peer": true
},
"node_modules/onnxruntime-web": {
"version": "1.26.0",
"resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.26.0.tgz",
"integrity": "sha512-LbRr/8zZt2xilI2smrVQGGKINo0U46i8qJp+UXyMBGfqN7KjnH1BiwCwLwyNIVV4i9CKFv7Sf4PwLKWnT8/bEA==",
"license": "MIT",
"peer": true,
"dependencies": {
"flatbuffers": "^25.1.24",
"guid-typescript": "^1.0.9",
"long": "^5.2.3",
"onnxruntime-common": "1.26.0",
"platform": "^1.3.6",
"protobufjs": "^7.2.4"
}
},
"node_modules/optionator": {
"version": "0.9.4",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
@@ -3323,13 +3206,6 @@
"node": ">= 6"
}
},
"node_modules/platform": {
"version": "1.3.6",
"resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz",
"integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==",
"license": "MIT",
"peer": true
},
"node_modules/postcss": {
"version": "8.5.6",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz",
@@ -3509,30 +3385,6 @@
"node": ">= 0.8.0"
}
},
"node_modules/protobufjs": {
"version": "7.6.4",
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.4.tgz",
"integrity": "sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==",
"hasInstallScript": true,
"license": "BSD-3-Clause",
"peer": true,
"dependencies": {
"@protobufjs/aspromise": "^1.1.2",
"@protobufjs/base64": "^1.1.2",
"@protobufjs/codegen": "^2.0.5",
"@protobufjs/eventemitter": "^1.1.1",
"@protobufjs/fetch": "^1.1.1",
"@protobufjs/float": "^1.0.2",
"@protobufjs/path": "^1.1.2",
"@protobufjs/pool": "^1.1.0",
"@protobufjs/utf8": "^1.1.1",
"@types/node": ">=13.7.0",
"long": "^5.3.2"
},
"engines": {
"node": ">=12.0.0"
}
},
"node_modules/punycode": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
@@ -4181,7 +4033,9 @@
"version": "7.24.6",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz",
"integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==",
"dev": true,
"license": "MIT",
"optional": true,
"peer": true
},
"node_modules/update-browserslist-db": {
webui/package.json
-1
View File
@@ -10,7 +10,6 @@
"preview": "vite preview"
},
"dependencies": {
"@mintplex-labs/piper-tts-web": "^1.0.4",
"react": "^19.2.0",
"react-dom": "^19.2.0",
"react-icons": "^5.5.0",
webui/src/scanner/ScannerApp/ScannerContent.jsx
+8 -7
View File
@@ -8,7 +8,6 @@ import useDefaultNickname from '../../hooks/useDefaultNickname.js';
import useUserIdentitySync from '../../hooks/useUserIdentitySync.js';
import SocketConnectionPill from '../../components/SocketConnectionPill/index.jsx';
import useScannerSpeech from './useScannerSpeech.js';
import ScannerVoiceSetup from './ScannerVoiceSetup.jsx';
const EMPTY_SCANNER_STATE = {
beepAllowed: false,
@@ -44,16 +43,16 @@ function playSubmitBeep() {
export default function ScannerContent() {
const socket = useSocket();
const setupMode = new URLSearchParams(window.location.search).get('setup') === 'voice';
const inputRef = useRef(null);
const flashTimerRef = useRef(null);
const [scannerState, setScannerState] = useState(EMPTY_SCANNER_STATE);
const [scanAudioEvent, setScanAudioEvent] = useState(null);
const [flashActive, setFlashActive] = useState(false);
useDefaultNickname();
useUserIdentitySync();
useSpectatorMode();
useScannerSpeech(scannerState.lastScan);
useScannerSpeech(scannerState.lastScan, scanAudioEvent);
const focusInput = useCallback(() => {
inputRef.current?.focus();
@@ -76,10 +75,16 @@ export default function ScannerContent() {
...(nextState && typeof nextState === 'object' ? nextState : {}),
});
}
function handleScanAudio(payload = null) {
setScanAudioEvent(payload && typeof payload === 'object' ? payload : null);
}
socket.emit('barcode:subscribe', {}, () => {});
socket.on('barcode:state', handleScannerState);
socket.on('barcode:scanAudio', handleScanAudio);
return () => {
socket.off('barcode:state', handleScannerState);
socket.off('barcode:scanAudio', handleScanAudio);
};
}, [socket]);
@@ -114,10 +119,6 @@ export default function ScannerContent() {
const lastScan = scannerState.lastScan;
const label = lastScan?.label || 'waiting';
if (setupMode) {
return <ScannerVoiceSetup />;
}
return (
<main
className={`flex min-h-screen cursor-default flex-col items-center justify-center overflow-hidden px-6 text-center ${
webui/src/scanner/ScannerApp/ScannerVoiceSetup.jsx
-157
View File
@@ -1,157 +0,0 @@
// Scanner Voice Setup
// Purpose: Lets the scanner computer pick one exact browser TTS voice and store it locally.
// Scope: Keeps voice selection out of the rover-facing scanner screen while avoiding unreliable voice-name guessing.
import { useCallback, useEffect, useState } from 'react';
import { SCANNER_VOICE_STORAGE_KEY } from './useScannerSpeech.js';
function describeVoice(voice) {
const parts = [voice?.name || 'unknown voice'];
if (voice?.lang) parts.push(voice.lang);
if (voice?.default) parts.push('default');
return parts.join(' · ');
}
function speakVoiceSample(voice) {
const synth = window.speechSynthesis;
if (!synth || typeof window.SpeechSynthesisUtterance !== 'function') return;
// Setup speech is intentionally direct and uncached. The operator is testing
// individual installed voices, so each button should cancel the previous
// sample and immediately speak with the candidate voice.
synth.cancel();
const utterance = new window.SpeechSynthesisUtterance('object 1');
utterance.voice = voice;
utterance.lang = voice?.lang || 'en-US';
utterance.rate = 1;
utterance.pitch = 1;
utterance.volume = 1;
synth.speak(utterance);
}
function readBrowserVoices() {
const synth = window.speechSynthesis;
return typeof synth?.getVoices === 'function' ? synth.getVoices() : [];
}
export default function ScannerVoiceSetup() {
const [voices, setVoices] = useState(() => readBrowserVoices());
const [selectedVoiceName, setSelectedVoiceName] = useState(() =>
String(window.localStorage.getItem(SCANNER_VOICE_STORAGE_KEY) || '').trim(),
);
const refreshVoices = useCallback(() => {
const nextVoices = readBrowserVoices();
setVoices(nextVoices);
// Logging the exact voice list is useful on Linux because Firefox exposes
// whatever the system speech stack provides, and those names are the only
// reliable identifiers we can save for later scanner sessions.
console.info('[scanner voice setup] available voices', nextVoices.map(describeVoice));
}, []);
useEffect(() => {
const refreshTimer = window.setTimeout(refreshVoices, 0);
const synth = window.speechSynthesis;
if (!synth) {
return () => {
window.clearTimeout(refreshTimer);
};
}
synth.addEventListener?.('voiceschanged', refreshVoices);
synth.onvoiceschanged = refreshVoices;
return () => {
window.clearTimeout(refreshTimer);
synth.removeEventListener?.('voiceschanged', refreshVoices);
if (synth.onvoiceschanged === refreshVoices) {
synth.onvoiceschanged = null;
}
};
}, [refreshVoices]);
const saveVoice = useCallback((voice) => {
const voiceName = String(voice?.name || '').trim();
if (!voiceName) return;
window.localStorage.setItem(SCANNER_VOICE_STORAGE_KEY, voiceName);
setSelectedVoiceName(voiceName);
speakVoiceSample(voice);
}, []);
const clearVoice = useCallback(() => {
window.localStorage.removeItem(SCANNER_VOICE_STORAGE_KEY);
setSelectedVoiceName('');
const synth = window.speechSynthesis;
if (synth) synth.cancel();
}, []);
return (
<main className="min-h-screen overflow-y-auto bg-black px-6 py-6 text-white">
<div className="mx-auto flex max-w-5xl flex-col gap-4">
<header className="flex flex-col gap-2 border-b border-white/30 pb-4">
<h1 className="text-5xl font-black leading-none tracking-normal">scanner voice</h1>
<p className="text-2xl leading-tight text-white">
Pick the exact voice this scanner computer should use.
</p>
<p className="text-xl leading-tight text-white/80">
Selected: {selectedVoiceName || 'automatic fallback'}
</p>
</header>
<div className="flex flex-wrap gap-3">
<button
type="button"
className="border-2 border-white bg-white px-4 py-3 text-2xl font-bold text-black"
onClick={refreshVoices}
>
refresh voices
</button>
<button
type="button"
className="border-2 border-white px-4 py-3 text-2xl font-bold text-white"
onClick={clearVoice}
>
use automatic fallback
</button>
<a
className="border-2 border-white px-4 py-3 text-2xl font-bold text-white"
href="/scanner"
>
back to scanner
</a>
</div>
<section className="grid gap-3">
{voices.length ? voices.map((voice) => {
const selected = String(voice?.name || '') === selectedVoiceName;
return (
<article
key={`${voice.name}-${voice.lang}`}
className={`grid gap-3 border-2 p-4 ${selected ? 'border-white bg-white text-black' : 'border-white/50 bg-black text-white'}`}
>
<div className="text-3xl font-black leading-tight tracking-normal">
{describeVoice(voice)}
</div>
<div className="flex flex-wrap gap-3">
<button
type="button"
className={`border-2 px-4 py-3 text-2xl font-bold ${selected ? 'border-black text-black' : 'border-white text-white'}`}
onClick={() => speakVoiceSample(voice)}
>
test
</button>
<button
type="button"
className={`border-2 px-4 py-3 text-2xl font-bold ${selected ? 'border-black bg-black text-white' : 'border-white bg-white text-black'}`}
onClick={() => saveVoice(voice)}
>
select
</button>
</div>
</article>
);
}) : (
<div className="border-2 border-white/50 p-4 text-3xl font-black">
no browser voices loaded
</div>
)}
</section>
</div>
</main>
);
}
webui/src/scanner/ScannerApp/scannerTts/piperBrowserTts.js
-117
View File
@@ -1,117 +0,0 @@
// Piper Browser TTS Adapter
// Purpose: Runs scanner speech through Piper locally in the browser when available.
// Scope: Keeps the neural TTS package behind a dynamic import so non-scanner pages do not load it.
const PIPER_VOICE_ID = 'en_US-hfc_female-medium';
const PIPER_LIVE_TIMEOUT_MS = 900;
let piperModulePromise = null;
let currentAudio = null;
let currentObjectUrl = null;
let activeRequestId = 0;
function stopCurrentAudio() {
if (currentAudio) {
// Scanner labels should never overlap. Stop the previous clip before the
// next scan speaks so audio always matches the current large display text.
currentAudio.pause();
currentAudio.removeAttribute('src');
currentAudio.load();
currentAudio = null;
}
if (currentObjectUrl) {
URL.revokeObjectURL(currentObjectUrl);
currentObjectUrl = null;
}
}
async function loadPiperModule() {
if (!piperModulePromise) {
piperModulePromise = import('@mintplex-labs/piper-tts-web')
.then((module) => {
console.info('[scanner tts] Piper module ready');
return module;
})
.catch((err) => {
// Reset after failures so later scans can try again after a transient
// model/CDN/cache issue. The hook falls back to browser speech for the
// current scan rather than blocking the scanner interaction.
piperModulePromise = null;
throw err;
});
}
return piperModulePromise;
}
function playBlob(blob) {
stopCurrentAudio();
currentObjectUrl = URL.createObjectURL(blob);
currentAudio = new Audio(currentObjectUrl);
currentAudio.volume = 1;
return new Promise((resolve, reject) => {
const audio = currentAudio;
audio.onended = () => {
if (currentAudio === audio) {
stopCurrentAudio();
}
resolve(true);
};
audio.onerror = () => {
if (currentAudio === audio) {
stopCurrentAudio();
}
reject(new Error('Piper audio playback failed'));
};
audio.play().catch((err) => {
if (currentAudio === audio) {
stopCurrentAudio();
}
reject(err);
});
});
}
export async function speakWithPiper(text) {
const cleanText = String(text || '').replace(/\s+/g, ' ').trim();
if (!cleanText) return false;
const requestId = activeRequestId + 1;
activeRequestId = requestId;
try {
const wavBlob = await Promise.race([
loadPiperModule().then((piper) =>
piper.predict(
{
text: cleanText,
voiceId: PIPER_VOICE_ID,
},
(progress) => {
// Piper stores downloaded models in the browser origin-private file
// system. Console progress is enough for setup/debugging while the
// rover-facing scanner screen stays uncluttered.
if (progress?.url) {
console.info('[scanner tts] Piper load', progress);
}
},
),
),
new Promise((resolve) => {
window.setTimeout(() => resolve(null), PIPER_LIVE_TIMEOUT_MS);
}),
]);
if (!wavBlob || requestId !== activeRequestId) {
return false;
}
await playBlob(wavBlob);
return true;
} catch (err) {
console.warn('[scanner tts] Piper unavailable; falling back to browser speech', err);
return false;
}
}
export function stopPiperSpeech() {
activeRequestId += 1;
stopCurrentAudio();
}
webui/src/scanner/ScannerApp/useScannerSpeech.js
+98 -20
View File
@@ -1,11 +1,11 @@
// Scanner Speech Hook
// Purpose: Speaks server-provided scan labels on the scanner computer.
// Scope: Keeps browser TTS as a local output device while leaving scan meaning and phrase selection on the server.
// Scope: Plays server-generated scanner audio when available, with browser TTS as the local fallback output device.
import { useEffect, useRef } from 'react';
import { speakWithPiper, stopPiperSpeech } from './scannerTts/piperBrowserTts.js';
const SPEECH_START_TIMEOUT_MS = 1500;
const SPEECH_RETRY_DELAY_MS = 700;
const SERVER_AUDIO_GRACE_MS = 350;
export const SCANNER_VOICE_STORAGE_KEY = 'scanner.preferredVoiceName';
const PREFERRED_VOICE_PATTERN = /female|samantha|victoria|allison|zira|karen|moira|serena|ava|susan|hazel|google (us|uk) english/i;
const NOVELTY_VOICE_PATTERN = /whisper|bubbles|bells|boing|bad news|bahh|cellos|deranged|good news|hysterical|pipe organ|trinoids|zarvox/i;
@@ -45,16 +45,57 @@ function pickScannerVoice(synth) {
);
}
export default function useScannerSpeech(scan) {
function normalizeAudioBuffer(buffer) {
if (!buffer) return null;
if (buffer instanceof ArrayBuffer) return buffer;
if (ArrayBuffer.isView(buffer)) {
return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
}
if (Array.isArray(buffer)) {
return new Uint8Array(buffer).buffer;
}
return null;
}
function playServerAudio(audio) {
const audioBuffer = normalizeAudioBuffer(audio?.buffer);
if (!audioBuffer) return null;
const mime = String(audio?.mime || 'audio/wav').trim() || 'audio/wav';
const blob = new Blob([audioBuffer], { type: mime });
const objectUrl = URL.createObjectURL(blob);
const player = new Audio(objectUrl);
player.volume = 1;
return {
play: () =>
player.play().finally(() => {
// The generated clip is only needed once per scan. Releasing the object
// URL after play starts/ends prevents repeated scans from leaking blobs
// on a long-running scanner page.
player.onended = () => URL.revokeObjectURL(objectUrl);
}),
stop: () => {
player.pause();
player.removeAttribute('src');
player.load();
URL.revokeObjectURL(objectUrl);
},
};
}
export default function useScannerSpeech(scan, serverAudioEvent = null) {
const retryTimerRef = useRef(null);
const startTimerRef = useRef(null);
const spokenScanKeyRef = useRef(null);
const fallbackTimerRef = useRef(null);
const serverAudioPlayerRef = useRef(null);
useEffect(() => {
return () => {
window.clearTimeout(retryTimerRef.current);
window.clearTimeout(startTimerRef.current);
stopPiperSpeech();
window.clearTimeout(fallbackTimerRef.current);
serverAudioPlayerRef.current?.stop();
};
}, []);
@@ -64,19 +105,20 @@ export default function useScannerSpeech(scan) {
if (!speechText || !scanKey || spokenScanKeyRef.current === scanKey) return undefined;
let cancelled = false;
spokenScanKeyRef.current = scanKey;
function clearSpeechTimers() {
window.clearTimeout(retryTimerRef.current);
window.clearTimeout(startTimerRef.current);
window.clearTimeout(fallbackTimerRef.current);
retryTimerRef.current = null;
startTimerRef.current = null;
fallbackTimerRef.current = null;
}
function retryLater() {
if (cancelled) return;
clearSpeechTimers();
retryTimerRef.current = window.setTimeout(() => speakOnce(), SPEECH_RETRY_DELAY_MS);
retryTimerRef.current = window.setTimeout(() => speakBrowserOnce(), SPEECH_RETRY_DELAY_MS);
}
function speakWithBrowserFallback() {
@@ -120,26 +162,62 @@ export default function useScannerSpeech(scan) {
}, SPEECH_START_TIMEOUT_MS);
}
function speakOnce() {
if (cancelled) return;
clearSpeechTimers();
stopPiperSpeech();
// Piper is attempted first because it is lighter than Kokoro while still
// giving the scanner page a consistent local model voice. The old browser
// TTS path remains the reliability fallback if Piper is still loading or
// fails on the scanner computer.
speakWithPiper(speechText).then((spoken) => {
if (cancelled || spoken) return;
speakWithBrowserFallback();
});
function markSpoken() {
spokenScanKeyRef.current = scanKey;
}
speakOnce();
function speakBrowserOnce() {
if (cancelled) return;
markSpoken();
clearSpeechTimers();
serverAudioPlayerRef.current?.stop();
speakWithBrowserFallback();
}
// Server-generated audio is preferred, but it arrives on a separate scanner
// socket event after the immediate scan state. Waiting briefly lets cached
// Kokoro clips play without delaying the visual scan result; if the clip is
// still being generated, the page falls back to Web Speech for this scan.
fallbackTimerRef.current = window.setTimeout(speakBrowserOnce, SERVER_AUDIO_GRACE_MS);
return () => {
cancelled = true;
clearSpeechTimers();
stopPiperSpeech();
serverAudioPlayerRef.current?.stop();
};
}, [scan]);
useEffect(() => {
const speechText = String(serverAudioEvent?.scan?.speechText || '').trim();
const scanKey = serverAudioEvent?.scan?.scannedAt ? `${serverAudioEvent.scan.scannedAt}:${speechText}` : '';
const currentSpeechText = String(scan?.speechText || '').trim();
const currentScanKey = scan?.scannedAt ? `${scan.scannedAt}:${currentSpeechText}` : '';
if (!scanKey || scanKey !== currentScanKey || spokenScanKeyRef.current === scanKey) return;
const player = playServerAudio(serverAudioEvent.audio);
if (!player) return;
window.clearTimeout(fallbackTimerRef.current);
fallbackTimerRef.current = null;
spokenScanKeyRef.current = scanKey;
serverAudioPlayerRef.current?.stop();
serverAudioPlayerRef.current = player;
player.play().catch(() => {
// If the generated clip cannot play for any browser reason, fall back to
// the selected Web Speech voice rather than leaving the scan silent.
spokenScanKeyRef.current = scanKey;
const synth = window.speechSynthesis;
if (!synth || typeof window.SpeechSynthesisUtterance !== 'function') return;
synth.cancel();
const utterance = new window.SpeechSynthesisUtterance(currentSpeechText);
const preferredVoice = pickScannerVoice(synth);
if (preferredVoice) {
utterance.voice = preferredVoice;
utterance.lang = preferredVoice.lang;
}
utterance.rate = 1;
utterance.pitch = 1;
utterance.volume = 1;
synth.speak(utterance);
});
}, [scan, serverAudioEvent]);
}