wasm slop tts

2026-09-15 17:12:59 -04:00 · 2026-06-13 17:24:34 -04:00
parent 880ebfefd2
commit d28d37575c
9 changed files with 4021 additions and 12 deletions
@@ -43,6 +43,11 @@
      "type": "object",
      "entityId": "laptop",
      "label": "Broken Laptop"
+    },
+    "o006": {
+      "type": "object",
+      "entityId": "radio",
+      "label": "S. I. M. card properly"
    }
  }
 }
@@ -12,7 +12,7 @@
    <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
    <meta name="apple-mobile-web-app-title" content="Roomba Rover" />
    <title>Roomba Rover</title>
-    <script type="module" crossorigin src="/assets/index-suSXs4PI.js"></script>
+    <script type="module" crossorigin src="/assets/index-DXvz_bZm.js"></script>
    <link rel="stylesheet" crossorigin href="/assets/index-BVKa8Zph.css">
  </head>
  <body>
@@ -10,6 +10,7 @@
    "preview": "vite preview"
  },
  "dependencies": {
+    "kokoro-js": "^1.2.1",
    "react": "^19.2.0",
    "react-dom": "^19.2.0",
    "react-icons": "^5.5.0",
@@ -0,0 +1,114 @@
+// Kokoro Browser TTS Adapter
+// Purpose: Runs scanner speech through Kokoro.js locally in the browser when available.
+// Scope: Keeps the heavy model dependency behind a dynamic import so non-scanner pages do not load it.
+
+const KOKORO_MODEL_ID = 'onnx-community/Kokoro-82M-v1.0-ONNX';
+const KOKORO_OPTIONS = {
+  // q8 is the practical first pass for a scanner station: much smaller than
+  // fp32/fp16, still good enough for short object labels, and it runs on plain
+  // WASM so Firefox does not need WebGPU support.
+  dtype: 'q8',
+  device: 'wasm',
+};
+const KOKORO_VOICE = 'af_bella';
+const KOKORO_SPEED = 1;
+
+let kokoroModelPromise = null;
+let currentAudio = null;
+let currentObjectUrl = null;
+
+function stopCurrentAudio() {
+  if (currentAudio) {
+    // Scanner scans can arrive close together. Stop the previous generated clip
+    // before starting the next one so the spoken label always matches the most
+    // recent server scan state.
+    currentAudio.pause();
+    currentAudio.removeAttribute('src');
+    currentAudio.load();
+    currentAudio = null;
+  }
+  if (currentObjectUrl) {
+    URL.revokeObjectURL(currentObjectUrl);
+    currentObjectUrl = null;
+  }
+}
+
+async function loadKokoroModel() {
+  if (!kokoroModelPromise) {
+    kokoroModelPromise = import('kokoro-js')
+      .then(async ({ KokoroTTS }) => {
+        console.info('[scanner tts] loading Kokoro model');
+        const model = await KokoroTTS.from_pretrained(KOKORO_MODEL_ID, {
+          ...KOKORO_OPTIONS,
+          progress_callback: (progress) => {
+            // The normal scanner page stays clean, but console progress is very
+            // useful when first-load model downloads are slow on the scanner PC.
+            if (progress?.status) {
+              console.info('[scanner tts] Kokoro load', progress);
+            }
+          },
+        });
+        console.info('[scanner tts] Kokoro model ready');
+        return model;
+      })
+      .catch((err) => {
+        // Reset the promise after a failure so a later scan can retry. This is
+        // important for first-load network hiccups while still letting the hook
+        // fall back to browser speech immediately.
+        kokoroModelPromise = null;
+        throw err;
+      });
+  }
+  return kokoroModelPromise;
+}
+
+function playBlob(blob) {
+  stopCurrentAudio();
+  currentObjectUrl = URL.createObjectURL(blob);
+  currentAudio = new Audio(currentObjectUrl);
+  currentAudio.volume = 1;
+
+  return new Promise((resolve, reject) => {
+    const audio = currentAudio;
+    audio.onended = () => {
+      if (currentAudio === audio) {
+        stopCurrentAudio();
+      }
+      resolve(true);
+    };
+    audio.onerror = () => {
+      if (currentAudio === audio) {
+        stopCurrentAudio();
+      }
+      reject(new Error('Kokoro audio playback failed'));
+    };
+    audio.play().catch((err) => {
+      if (currentAudio === audio) {
+        stopCurrentAudio();
+      }
+      reject(err);
+    });
+  });
+}
+
+export async function speakWithKokoro(text) {
+  const cleanText = String(text || '').replace(/\s+/g, ' ').trim();
+  if (!cleanText) return false;
+
+  try {
+    const model = await loadKokoroModel();
+    const audio = await model.generate(cleanText, {
+      voice: KOKORO_VOICE,
+      speed: KOKORO_SPEED,
+    });
+    await playBlob(audio.toBlob());
+    return true;
+  } catch (err) {
+    console.warn('[scanner tts] Kokoro unavailable; falling back to browser speech', err);
+    return false;
+  }
+}
+
+export function stopKokoroSpeech() {
+  stopCurrentAudio();
+}
@@ -2,6 +2,7 @@
 // Purpose: Speaks server-provided scan labels on the scanner computer.
 // Scope: Keeps browser TTS as a local output device while leaving scan meaning and phrase selection on the server.
 import { useEffect, useRef } from 'react';
+import { speakWithKokoro, stopKokoroSpeech } from './scannerTts/kokoroBrowserTts.js';

 const SPEECH_START_TIMEOUT_MS = 1500;
 const SPEECH_RETRY_DELAY_MS = 700;
@@ -53,6 +54,7 @@ export default function useScannerSpeech(scan) {
    return () => {
      window.clearTimeout(retryTimerRef.current);
      window.clearTimeout(startTimerRef.current);
+      stopKokoroSpeech();
    };
  }, []);

@@ -77,8 +79,7 @@ export default function useScannerSpeech(scan) {
      retryTimerRef.current = window.setTimeout(() => speakOnce(), SPEECH_RETRY_DELAY_MS);
    }

-    function speakOnce() {
-      if (cancelled) return;
+    function speakWithBrowserFallback() {
      const synth = window.speechSynthesis;
      if (!synth || typeof window.SpeechSynthesisUtterance !== 'function') {
        retryLater();
@@ -119,11 +120,25 @@ export default function useScannerSpeech(scan) {
      }, SPEECH_START_TIMEOUT_MS);
    }

+    function speakOnce() {
+      if (cancelled) return;
+      clearSpeechTimers();
+      stopKokoroSpeech();
+      // Kokoro is attempted first because it gives the scanner page a consistent
+      // local model voice instead of relying on the OS/browser Web Speech voice
+      // list. The old browser TTS path remains the reliability fallback.
+      speakWithKokoro(speechText).then((spoken) => {
+        if (cancelled || spoken) return;
+        speakWithBrowserFallback();
+      });
+    }
+
    speakOnce();

    return () => {
      cancelled = true;
      clearSpeechTimers();
+      stopKokoroSpeech();
    };
  }, [scan]);
 }