server generated scanner voices!

2026-09-16 01:21:20 -04:00 · 2026-06-13 17:53:52 -04:00
parent bddff1e67f
commit 61ace6e7c4
18 changed files with 323 additions and 3282 deletions
@@ -14,6 +14,7 @@
    "fuse.js": "^7.4.2",
    "home-assistant-js-websocket": "^3.1.2",
    "js-yaml": "^4.1.1",
+    "kokoro-js": "^1.2.1",
    "morgan": "^1.10.0",
    "obscenity": "^0.4.6",
    "ollama": "^0.6.3",
@@ -12,8 +12,8 @@
    <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
    <meta name="apple-mobile-web-app-title" content="Roomba Rover" />
    <title>Roomba Rover</title>
-    <script type="module" crossorigin src="/assets/index-h2u2QzDn.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-BVKa8Zph.css">
+    <script type="module" crossorigin src="/assets/index-Dmdzc1Vt.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-DSZycrrG.css">
  </head>
  <body>
    <div id="root"></div>
@@ -7,14 +7,17 @@ const logger = require('../../globals/logger').child('barcodeScannerService');
 const { resolveDataDir, resolveDataPath } = require('../../helpers/dataPaths');
 const { getMode, MODES, modeEvents } = require('../modeManager');
 const { sendAlert } = require('../alertService');
+const { ensureAudioForText, warmAudioForTexts } = require('./ttsCache');

 const DATA_DIR = resolveDataDir();
 const REGISTRY_PATH = resolveDataPath('barcode-registry.json');
 const RECENT_SCAN_LIMIT = 8;
 const VALID_CODE_PATTERN = /^[a-z][0-9]{3}$/;
+const SCANNER_SOCKET_ROOM = 'barcode-scanner';

 let lastKnownGoodRegistry = null;
 let lastRegistryError = null;
+let lastPrewarmKey = '';
 let state = {
  lastScan: null,
  recentScans: [],
@@ -103,6 +106,16 @@ function validateRegistry(rawRegistry) {
  return { codes: normalizedCodes };
 }

+function prewarmRegistryAudio(registry) {
+  const labels = Object.values(registry?.codes || {})
+    .map((entry) => entry?.label)
+    .filter(Boolean);
+  const nextPrewarmKey = labels.slice().sort().join('\n');
+  if (!nextPrewarmKey || nextPrewarmKey === lastPrewarmKey) return;
+  lastPrewarmKey = nextPrewarmKey;
+  warmAudioForTexts(labels);
+}
+
 function loadRegistryForScan() {
  try {
    ensureRegistryFile();
@@ -111,6 +124,7 @@ function loadRegistryForScan() {
    const registry = validateRegistry(parsed);
    lastKnownGoodRegistry = registry;
    lastRegistryError = null;
+    prewarmRegistryAudio(registry);
    return { registry, error: null };
  } catch (err) {
    lastRegistryError = err.message;
@@ -136,7 +150,10 @@ function buildStatePayload() {
 }

 function broadcastState() {
-  io.emit('barcode:state', buildStatePayload());
+  // Scanner state is scoped to scanner clients because scan-specific packets can
+  // include generated audio. Driver/spectator/display pages should not receive
+  // barcode audio or scanner-only state traffic.
+  io.to(SCANNER_SOCKET_ROOM).emit('barcode:state', buildStatePayload());
 }

 function sendBarcodeScanAlert(result) {
@@ -217,7 +234,19 @@ function resolveScan(rawCode) {
  };
 }

-function applyScan(rawCode) {
+async function buildScanAudio(result) {
+  const text = String(result?.speechText || '').trim();
+  if (!text) return null;
+  const audio = await ensureAudioForText(text);
+  if (!audio?.buffer) return null;
+  return {
+    cacheKey: audio.cacheKey,
+    mime: audio.mime,
+    buffer: audio.buffer,
+  };
+}
+
+async function applyScan(rawCode) {
  const result = resolveScan(rawCode);
  // The latest result is the canonical display state. Recent scans are retained
  // only for debugging and future scanner-page variants; the rover-facing first
@@ -228,15 +257,33 @@ function applyScan(rawCode) {
    registryError: result.registryError || null,
  };
  broadcastState();
+  buildScanAudio(result)
+    .then((audio) => {
+      io.to(SCANNER_SOCKET_ROOM).emit('barcode:scanAudio', {
+        scan: result,
+        audio,
+      });
+    })
+    .catch((err) => {
+      logger.warn('Barcode scan audio emission failed', {
+        code: result.code,
+        error: err.message,
+      });
+    });
  sendBarcodeScanAlert(result);
-  return result;
+  return { result };
 }

 io.on('connection', (socket) => {
-  socket.emit('barcode:state', buildStatePayload());
-  socket.on('barcode:scan', ({ code } = {}, cb = () => {}) => {
+  socket.on('barcode:subscribe', (_payload = {}, cb = () => {}) => {
+    socket.join(SCANNER_SOCKET_ROOM);
+    socket.emit('barcode:state', buildStatePayload());
+    cb({ success: true, state: buildStatePayload() });
+  });
+
+  socket.on('barcode:scan', async ({ code } = {}, cb = () => {}) => {
    try {
-      const result = applyScan(code);
+      const { result } = await applyScan(code);
      cb({ success: true, result, state: buildStatePayload() });
    } catch (err) {
      // Socket handlers should never let a malformed scan or registry edge case
@@ -0,0 +1,148 @@
+// Barcode Scanner TTS Cache
+// Purpose: Generates and caches scanner speech audio on the server with Kokoro.
+// Scope: Keeps neural TTS completely outside the browser UI so scanner/audio experiments cannot freeze other web pages.
+const crypto = require('crypto');
+const fs = require('fs');
+const fsp = require('fs/promises');
+const path = require('path');
+const logger = require('../../globals/logger').child('barcodeScannerTts');
+const { resolveDataDir } = require('../../helpers/dataPaths');
+
+const CACHE_DIR = path.join(resolveDataDir(), 'barcode-tts-cache');
+const MODEL_ID = 'onnx-community/Kokoro-82M-v1.0-ONNX';
+const KOKORO_OPTIONS = {
+  // q8 keeps generation and model footprint smaller while preserving the voice
+  // quality that worked well in the browser experiment. Server generation is
+  // allowed to be slower than the scanner page, because cached files are reused.
+  dtype: 'q8',
+  device: 'cpu',
+};
+const VOICE_ID = 'af_bella';
+const SPEED = 1;
+
+let modelPromise = null;
+const inFlightByText = new Map();
+
+function normalizeSpeechText(text) {
+  return String(text || '').replace(/\s+/g, ' ').trim();
+}
+
+function cacheKeyForText(text) {
+  return crypto.createHash('sha256').update(text).digest('hex').slice(0, 24);
+}
+
+function cachePathForText(text) {
+  return path.join(CACHE_DIR, `${cacheKeyForText(text)}.wav`);
+}
+
+async function loadModel() {
+  if (!modelPromise) {
+    modelPromise = import('kokoro-js')
+      .then(async ({ KokoroTTS }) => {
+        logger.info('Loading Kokoro scanner TTS model');
+        const model = await KokoroTTS.from_pretrained(MODEL_ID, KOKORO_OPTIONS);
+        logger.info('Kokoro scanner TTS model ready');
+        return model;
+      })
+      .catch((err) => {
+        // Reset after a failed load so a future scan can retry after a transient
+        // model download or environment problem. Scanner clients will simply use
+        // browser fallback when no server audio is available.
+        modelPromise = null;
+        throw err;
+      });
+  }
+  return modelPromise;
+}
+
+async function readCachedAudio(text) {
+  const filePath = cachePathForText(text);
+  try {
+    return await fsp.readFile(filePath);
+  } catch (err) {
+    if (err.code !== 'ENOENT') {
+      logger.warn('Failed to read scanner TTS cache file', { filePath, error: err.message });
+    }
+    return null;
+  }
+}
+
+async function generateAudio(text) {
+  await fsp.mkdir(CACHE_DIR, { recursive: true });
+  const model = await loadModel();
+  const audio = await model.generate(text, {
+    voice: VOICE_ID,
+    speed: SPEED,
+  });
+  const wavBuffer = Buffer.from(audio.toWav());
+  const filePath = cachePathForText(text);
+  await fsp.writeFile(filePath, wavBuffer);
+  return wavBuffer;
+}
+
+async function ensureAudioForText(text) {
+  const cleanText = normalizeSpeechText(text);
+  if (!cleanText) return null;
+
+  const cached = await readCachedAudio(cleanText);
+  if (cached) {
+    return {
+      cacheKey: cacheKeyForText(cleanText),
+      mime: 'audio/wav',
+      buffer: cached,
+    };
+  }
+
+  if (!inFlightByText.has(cleanText)) {
+    inFlightByText.set(
+      cleanText,
+      generateAudio(cleanText)
+        .catch((err) => {
+          logger.warn('Failed to generate scanner TTS audio', { text: cleanText, error: err.message });
+          return null;
+        })
+        .finally(() => {
+          inFlightByText.delete(cleanText);
+        }),
+    );
+  }
+
+  const generated = await inFlightByText.get(cleanText);
+  if (!generated) return null;
+  return {
+    cacheKey: cacheKeyForText(cleanText),
+    mime: 'audio/wav',
+    buffer: generated,
+  };
+}
+
+function warmAudioForTexts(texts = []) {
+  const uniqueTexts = Array.from(new Set(texts.map(normalizeSpeechText).filter(Boolean)));
+  if (!uniqueTexts.length) return;
+
+  // Prewarming should never block server startup or barcode resolution. It just
+  // fills the cache opportunistically so the scanner page usually receives an
+  // already-generated wav blob at scan time.
+  uniqueTexts.reduce(
+    (chain, text) =>
+      chain.then(async () => {
+        try {
+          await ensureAudioForText(text);
+        } catch (err) {
+          logger.warn('Scanner TTS prewarm failed', { text, error: err.message });
+        }
+      }),
+    Promise.resolve(),
+  );
+}
+
+function getCacheDir() {
+  fs.mkdirSync(CACHE_DIR, { recursive: true });
+  return CACHE_DIR;
+}
+
+module.exports = {
+  ensureAudioForText,
+  warmAudioForTexts,
+  getCacheDir,
+};