turning into redoing all of the webrtc reading and publishing client side lol

webui/package.json

@@ -4,7 +4,7 @@
   "version": "0.0.0",
   "type": "module",
   "scripts": {
-    "dev": "VITE_ROVERD_URL=https://rover.otter.land vite",
+    "dev": "VITE_ROVERD_URL=https://rovertest.otter.land vite",
     "build": "vite build",
     "lint": "eslint .",
     "test": "node --test src/controls/inputs/*.test.js src/components/GamepadMappingSettings/*.test.js",

webui/src/components/PtzLiveVideo/index.jsx

@@ -5,6 +5,7 @@ import { useEffect, useRef, useState } from 'react';
 import { useVideoRequests } from '../../hooks/useVideoRequests.js';
 import { startWhepPlayback } from '../../lib/whepPlayback.js';
 import useWhepRestart from '../../hooks/useWhepRestart.js';
+import useServerUrl from '../../hooks/useServerUrl.js';
 
 export const PTZ_CAMERA_ID = 'ptz-camera';
 
@@ -23,6 +24,7 @@ export default function PtzLiveVideo({
   statusClassName = 'pointer-events-none absolute left-1 top-1 z-20 font-medium text-slate-100 text-[0.65rem]',
   fallback = null,
 }) {
+  const serverUrl = useServerUrl();
   const videoRef = useRef(null);
   const playTimerRef = useRef(null);
   const [status, setStatus] = useState('idle');
@@ -44,7 +46,7 @@ export default function PtzLiveVideo({
     if (!enabled || shouldUseFallback || !source?.url || !videoRef.current) return undefined;
     const video = videoRef.current;
     const stop = startWhepPlayback({
-      url: source.url,
+      url: serverUrl(source.url),
       token: source.token,
       video: videoRef.current,
       startMuted,
@@ -76,7 +78,7 @@ export default function PtzLiveVideo({
       video.removeEventListener('error', handleError);
       stop();
     };
-  }, [enabled, scheduleRestart, shouldUseFallback, source?.token, source?.url, startMuted]);
+  }, [enabled, scheduleRestart, shouldUseFallback, source?.token, source?.url, startMuted, serverUrl]);
 
   useEffect(() => {
     const video = videoRef.current;

webui/src/components/RoomAudioPlayer/index.jsx

@@ -5,9 +5,11 @@ import { useSettingsNamespace } from '../../settings/index.js';
 import { AUDIO_SETTINGS_DEFAULTS } from '../../settings/namespaces.js';
 import { RESTART_DELAY_MS, startWhepPlayback } from '../../lib/whepPlayback.js';
 import useWhepRestart from '../../hooks/useWhepRestart.js';
+import useServerUrl from '../../hooks/useServerUrl.js';
 
 function StreamPlayer({ id, volume }) {
   const socket = useSocket();
+  const serverUrl = useServerUrl();
   const audio = useRef(null);
   const { restartToken, scheduleRestart } = useWhepRestart();
   useEffect(() => {
@@ -22,7 +24,7 @@ function StreamPlayer({ id, volume }) {
       token = response.token;
       if (error || response.error || !response.url) { scheduleRestart(); return; }
       stop = startWhepPlayback({
-        ...response, video: audio.current, audioOnly: true,
+        ...response, url: serverUrl(response.url), video: audio.current, audioOnly: true,
         onStatus: () => {}, onError: () => {}, scheduleRestart,
       });
     });
@@ -30,7 +32,7 @@ function StreamPlayer({ id, volume }) {
       active = false; stop?.();
       if (token && socket.connected) socket.emit('roomAudio:release', { token });
     };
-  }, [id, socket, restartToken, scheduleRestart]);
+  }, [id, socket, restartToken, scheduleRestart, serverUrl]);
   useEffect(() => { if (audio.current) audio.current.volume = Math.max(0, Math.min(1, volume)); }, [volume]);
   return <audio ref={audio} autoPlay />;
 }

webui/src/components/RoverMediaPlayer/index.jsx

@@ -1,6 +1,7 @@
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { startWhepPlayback } from '../../lib/whepPlayback.js';
 import useWhepRestart from '../../hooks/useWhepRestart.js';
+import useServerUrl from '../../hooks/useServerUrl.js';
 import { useTelemetrySelector } from '../../context/TelemetryContext.jsx';
 import {
   mainBrushAudioTelemetryEqual,
@@ -130,6 +131,7 @@ export default function RoverMediaPlayer({
   pauseVideoWhenHidden = null,
 }) {
   const assignedRoverId = useSessionSelector((state) => state.session?.assignment?.roverId ?? null);
+  const serverUrl = useServerUrl();
   const effectiveRoverId = roverId ?? assignedRoverId;
   const mode = useSessionSelector((state) => state.session?.mode || null);
   const configuredPauseHiddenRoverVideo = useSessionSelector(
@@ -415,7 +417,7 @@ export default function RoverMediaPlayer({
     }
     const resetMuteId = setTimeout(() => setMuted(true), 0);
     const stop = startWhepPlayback({
-      url: resolvedSessionInfo.url,
+      url: serverUrl(resolvedSessionInfo.url),
       token: resolvedSessionInfo.token,
       video: videoRef.current,
       receiveAudio: !hasDedicatedAudio,
@@ -446,6 +448,7 @@ export default function RoverMediaPlayer({
     hasDedicatedAudio,
     logAudio,
     effectiveRoverId,
+    serverUrl,
   ]);
 
   useEffect(() => {
@@ -510,7 +513,7 @@ export default function RoverMediaPlayer({
       return undefined;
     }
     return startWhepPlayback({
-      url: resolvedAudioSessionInfo.url,
+      url: serverUrl(resolvedAudioSessionInfo.url),
       token: resolvedAudioSessionInfo.token,
       video: audioRef.current,
       audioOnly: true,
@@ -533,6 +536,7 @@ export default function RoverMediaPlayer({
     audioRestartToken,
     scheduleAudioRestart,
     logAudio,
+    serverUrl,
   ]);
 
   useEffect(() => {

webui/src/components/vip/VipAudioUploadCard/VipAudioUploadCardContent.jsx

@@ -1,19 +1,16 @@
 // Vip Audio Upload Card Content
 // Purpose: Defines the Vip Audio Upload Card Content module and the local helpers/components used in this file.
 // Scope: Keeps behavior unchanged while isolating this concern into a clear, single-responsibility unit.
-import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react';
 import { fieldClass } from '../constants.js';
 import { useControlSelector } from '../../../controls/index.js';
 import ControlHint from '../../ControlHint/index.jsx';
 import { useSettingsNamespace } from '../../../settings/index.js';
-import { MAX_UPLOAD_BYTES, TARGET_SAMPLE_RATE, RTC_CONFIG } from './constants.js';
-import { bytesToBase64, buildAuthHeader } from './base64.js';
-import {
-  waitForIceGatheringComplete,
-  waitForPeerConnected,
-  configureSenderForLowLatency,
-  waitForOutboundAudioFlow,
-} from './whipTransport.js';
+import { MAX_UPLOAD_BYTES, TARGET_SAMPLE_RATE } from './constants.js';
+import { bytesToBase64 } from './base64.js';
+import { waitForOutboundAudioFlow } from './whipTransport.js';
+import MediaMTXWebRTCPublisher from '../../../lib/vendor/mediamtxPublisher.js';
+import useServerUrl from '../../../hooks/useServerUrl.js';
 import { mergeFloatChunks, encodeWavMono16 } from './audioCodec.js';
 import StatusIndicator from './StatusIndicator.jsx';
 import KeyPill from './KeyPill.jsx';
@@ -28,6 +25,7 @@ export default function VipAudioUploadCard({
   readyMicWhip,
   stopMicWhip,
 }) {
+  const serverUrl = useServerUrl();
   const pttActive = useControlSelector((control) => Boolean(control.state.mic?.pttActive));
   const { value: vipAudio, save: saveVipAudio } = useSettingsNamespace('vipAudio', {
     openMicEnabled: false,
@@ -47,7 +45,8 @@ export default function VipAudioUploadCard({
 
   const streamRef = useRef(null);
   const audioTrackRef = useRef(null);
-  const whipPcRef = useRef(null);
+  const micAttemptRef = useRef(null);
+  const micEnabledRef = useRef(false);
   const micActiveRef = useRef(false);
   const activeRoverRef = useRef('');
 
@@ -146,19 +145,12 @@ export default function VipAudioUploadCard({
       const target = String(targetRoverId || activeRoverRef.current || '').trim();
       micActiveRef.current = false;
       setMicState('idle');
-      if (whipPcRef.current) {
-        try {
-          whipPcRef.current.getSenders().forEach((sender) => sender.track?.stop());
-        } catch {
-          // noop
-        }
-        try {
-          whipPcRef.current.close();
-        } catch {
-          // noop
-        }
-      }
-      whipPcRef.current = null;
+      const publisher = micAttemptRef.current?.publisher;
+      micAttemptRef.current = null;
+      activeRoverRef.current = '';
+      publisher?.close();
+      // The upstream client stops transport retries; release its HTTP session separately.
+      if (publisher?.sessionUrl) fetch(publisher.sessionUrl, { method: 'DELETE', keepalive: true }).catch(() => {});
       if (streamRef.current) {
         try {
           streamRef.current.getTracks().forEach((track) => track.stop());
@@ -175,14 +167,15 @@ export default function VipAudioUploadCard({
           // noop
         }
       }
-      activeRoverRef.current = '';
     },
     [stopMicWhip],
   );
 
   const startWhipMic = useCallback(
-    async (target) => {
+    async (target, attempt) => {
+      const isCurrent = () => micAttemptRef.current === attempt && activeRoverRef.current === target;
       const startPayload = await startMicWhip?.(target);
+      if (!isCurrent()) return;
       const whipUrl = String(startPayload?.whipUrl || '').trim();
       const token = String(startPayload?.token || '').trim();
       if (!whipUrl || !token) {
@@ -198,12 +191,16 @@ export default function VipAudioUploadCard({
           autoGainControl: false,
         },
       });
+      if (!isCurrent()) {
+        stream.getTracks().forEach((track) => track.stop());
+        return;
+      }
       streamRef.current = stream;
 
       const track = stream.getAudioTracks()?.[0];
       audioTrackRef.current = track || null;
       if (track) {
-        track.enabled = Boolean(openMicEnabled || pttActive);
+        track.enabled = micEnabledRef.current;
       }
       if (track?.applyConstraints) {
         try {
@@ -219,47 +216,42 @@ export default function VipAudioUploadCard({
         }
       }
 
-      const pc = new RTCPeerConnection(RTC_CONFIG);
-      whipPcRef.current = pc;
-      pc.onconnectionstatechange = () => {
-        if (!micActiveRef.current) return;
-        const state = pc.connectionState;
-        if (state === 'connected') {
-          setMicState('live');
-          return;
-        }
-        if (state === 'failed' || state === 'disconnected' || state === 'closed') {
-          setMicState('error');
-          setMessage(`WHIP transport ${state}.`);
-        }
-      };
-      stream.getAudioTracks().forEach((audioTrack) => {
-        const sender = pc.addTrack(audioTrack, stream);
-        configureSenderForLowLatency(sender);
-      });
-
-      const offer = await pc.createOffer({ offerToReceiveAudio: false, offerToReceiveVideo: false });
-      await pc.setLocalDescription(offer);
-      await waitForIceGatheringComplete(pc, 1800);
-
-      const response = await fetch(whipUrl, {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/sdp',
-          ...buildAuthHeader(token),
+      if (!isCurrent()) return;
+      attempt.publisher = new MediaMTXWebRTCPublisher({
+        url: serverUrl(whipUrl),
+        user: token,
+        pass: token,
+        stream,
+        audioCodec: 'opus',
+        audioBitrate: 64,
+        audioVoice: true,
+        onConnected: async () => {
+          if (!isCurrent()) return;
+          const pc = attempt.publisher.pc;
+          try {
+            // Report ready only after audio is flowing, including on transport reconnection.
+            await waitForOutboundAudioFlow(pc);
+            if (!isCurrent() || attempt.publisher.pc !== pc) return;
+            await readyMicWhip?.(target);
+            if (!isCurrent() || attempt.publisher.pc !== pc) return;
+            setMicState('live');
+            setMessage('');
+          } catch (error) {
+            if (!isCurrent() || attempt.publisher.pc !== pc) return;
+            await stopMicCapture(target);
+            if (micAttemptRef.current) return;
+            setMicState('error');
+            setMessage(error.message || 'Failed to start mic forwarding.');
+          }
+        },
+        onError: (message) => {
+          if (!isCurrent()) return;
+          setMicState('starting');
+          setMessage(message);
         },
-        body: pc.localDescription?.sdp || offer.sdp,
       });
-      if (!response.ok) {
-        throw new Error(`WHIP request failed: ${response.status}`);
-      }
-      const answerSdp = await response.text();
-      await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp });
-      await waitForPeerConnected(pc, 10000);
-      await waitForOutboundAudioFlow(pc, 6000);
-      await readyMicWhip?.(target);
     },
-    [openMicEnabled, pttActive, readyMicWhip, startMicWhip],
+    [readyMicWhip, startMicWhip, stopMicCapture, serverUrl],
   );
 
   const teardownClipPipeline = useCallback(async () => {
@@ -410,8 +402,6 @@ export default function VipAudioUploadCard({
   }, [ensureClipPipeline, roverId]);
 
   useEffect(() => {
-    let cancelled = false;
-
     async function syncLiveMic() {
       if (clipMode) {
         await stopMicCapture(activeRoverRef.current || roverId);
@@ -425,34 +415,31 @@ export default function VipAudioUploadCard({
       if (micActiveRef.current && activeRoverRef.current === roverId) return;
       if (!openMicEnabled && !pttActive) return;
 
+      const attempt = {};
+      micAttemptRef.current = attempt;
       try {
         setMicState('starting');
         setMessage('');
         micActiveRef.current = true;
         activeRoverRef.current = roverId;
-        await startWhipMic(roverId);
-        if (!cancelled) {
-          setMicState('live');
-        }
+        await startWhipMic(roverId, attempt);
       } catch (err) {
-        if (!cancelled) {
+        if (micAttemptRef.current === attempt) {
+          await stopMicCapture(roverId);
+          if (micAttemptRef.current) return;
           setMicState('error');
           setMessage(err?.message || 'Failed to start mic forwarding.');
         }
-        await stopMicCapture(roverId);
       }
     }
 
     syncLiveMic();
-    return () => {
-      cancelled = true;
-    };
   }, [clipMode, openMicEnabled, pttActive, roverId, startWhipMic, stopMicCapture]);
 
-  useEffect(() => {
+  useLayoutEffect(() => {
+    micEnabledRef.current = Boolean(!clipMode && (openMicEnabled || pttActive));
     const track = audioTrackRef.current;
-    if (!track) return;
-    track.enabled = Boolean(!clipMode && (openMicEnabled || pttActive));
+    if (track) track.enabled = micEnabledRef.current;
   }, [clipMode, openMicEnabled, pttActive]);
 
   useEffect(() => {

webui/src/components/vip/VipAudioUploadCard/base64.js

@@ -10,14 +10,3 @@ export function bytesToBase64(bytes) {
   }
   return btoa(binary);
 }
-
-function encodeBase64(value) {
-  if (typeof btoa === 'function') return btoa(value);
-  return '';
-}
-
-export function buildAuthHeader(token) {
-  if (!token) return {};
-  const encoded = encodeBase64(`${token}:${token}`);
-  return encoded ? { Authorization: `Basic ${encoded}` } : {};
-}

webui/src/components/vip/VipAudioUploadCard/constants.js

@@ -3,8 +3,3 @@
 // Scope: Keeps behavior unchanged while isolating this concern into a clear, single-responsibility unit.
 export const MAX_UPLOAD_BYTES = 8 * 1024 * 1024;
 export const TARGET_SAMPLE_RATE = 16000;
-export const RTC_CONFIG = {
-  iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
-  bundlePolicy: 'max-bundle',
-  rtcpMuxPolicy: 'require',
-};

webui/src/components/vip/VipAudioUploadCard/whipTransport.js

@@ -1,91 +1,6 @@
 // whip Transport
 // Purpose: Defines the whip Transport module and the local helpers/components used in this file.
 // Scope: Keeps behavior unchanged while isolating this concern into a clear, single-responsibility unit.
-export function waitForIceGatheringComplete(pc, timeoutMs = 1500) {
-  return new Promise((resolve) => {
-    if (!pc || pc.iceGatheringState === 'complete') {
-      resolve();
-      return;
-    }
-    const timer = setTimeout(() => {
-      pc.removeEventListener('icegatheringstatechange', onChange);
-      resolve();
-    }, timeoutMs);
-    function onChange() {
-      if (pc.iceGatheringState === 'complete') {
-        clearTimeout(timer);
-        pc.removeEventListener('icegatheringstatechange', onChange);
-        resolve();
-      }
-    }
-    pc.addEventListener('icegatheringstatechange', onChange);
-  });
-}
-
-function isPeerTransportReady(pc) {
-  if (!pc) return false;
-  const conn = pc.connectionState;
-  const ice = pc.iceConnectionState;
-  if (conn === 'connected') return true;
-  if (ice === 'connected' || ice === 'completed') return true;
-  return false;
-}
-
-export function waitForPeerConnected(pc, timeoutMs = 10000) {
-  return new Promise((resolve, reject) => {
-    if (!pc) {
-      reject(new Error('Peer connection missing'));
-      return;
-    }
-    if (isPeerTransportReady(pc)) {
-      resolve();
-      return;
-    }
-    const timer = setTimeout(() => {
-      cleanup();
-      reject(new Error('Peer connection timeout'));
-    }, timeoutMs);
-    const onState = () => {
-      if (isPeerTransportReady(pc)) {
-        cleanup();
-        resolve();
-      } else if (
-        pc.connectionState === 'failed' ||
-        pc.connectionState === 'closed' ||
-        pc.iceConnectionState === 'failed'
-      ) {
-        cleanup();
-        reject(new Error(`Peer connection ${pc.connectionState || pc.iceConnectionState}`));
-      }
-    };
-    function cleanup() {
-      clearTimeout(timer);
-      pc.removeEventListener('connectionstatechange', onState);
-      pc.removeEventListener('iceconnectionstatechange', onState);
-    }
-    pc.addEventListener('connectionstatechange', onState);
-    pc.addEventListener('iceconnectionstatechange', onState);
-  });
-}
-
-export async function configureSenderForLowLatency(sender) {
-  if (!sender?.getParameters || !sender?.setParameters) return;
-  const params = sender.getParameters() || {};
-  const first = (params.encodings && params.encodings[0]) || {};
-  params.encodings = [
-    {
-      ...first,
-      maxBitrate: 64000,
-      dtx: 'disabled',
-    },
-  ];
-  try {
-    await sender.setParameters(params);
-  } catch {
-    // Browser support varies; keep defaults if rejected.
-  }
-}
-
 export function waitForOutboundAudioFlow(pc, timeoutMs = 6000) {
   return new Promise((resolve, reject) => {
     if (!pc) {

webui/src/hooks/useServerUrl.js

@@ -0,0 +1,8 @@
+import { useCallback } from 'react';
+import { useSessionSelector } from '../context/SessionContext.jsx';
+
+export default function useServerUrl() {
+  const publicUrl = useSessionSelector((state) => state.session?.interInstances?.profile?.publicUrl);
+  // The UI may be served by Vite while session data comes from the actual server.
+  return useCallback((url) => new URL(url, publicUrl || window.location.origin).href, [publicUrl]);
+}

webui/src/mic/MicApp.jsx

@@ -6,6 +6,7 @@ import { RESTART_DELAY_MS } from '../lib/whepPlayback.js';
 import { ControlSystemProvider, KeyboardInputManager, GamepadInputManager, useControlSelector } from '../controls/index.js';
 import ControlHint from '../components/ControlHint/index.jsx';
 import MediaMTXWebRTCPublisher from '../lib/vendor/mediamtxPublisher.js';
+import useServerUrl from '../hooks/useServerUrl.js';
 
 export default function MicApp() {
   return (
@@ -19,6 +20,7 @@ export default function MicApp() {
 
 function MicPage() {
   const socket = useSocket();
+  const serverUrl = useServerUrl();
   const connected = useSessionSelector((state) => state.connected);
   const local = useSessionSelector((state) => state.session?.isLocalNetwork);
   const enabled = useSessionSelector((state) => state.session?.features?.roomAudio);
@@ -109,7 +111,7 @@ function MicPage() {
         if (!isCurrent(current)) return;
         setStatus('Connecting audio…');
         current.publisher = new MediaMTXWebRTCPublisher({
-          url: new URL(source.url, window.location.href).href,
+          url: serverUrl(source.url),
           user: source.token,
           pass: source.token,
           stream: media,
@@ -135,7 +137,7 @@ function MicPage() {
       mediaRef.current = null;
       if (socket.connected) socket.emit('roomAudio:stop', {});
     };
-  }, [socket, connected, local, enabled]);
+  }, [socket, connected, local, enabled, serverUrl]);
   const message = !connected ? 'Connecting…' : local === false ? 'This page is available to local visitors only.' : !enabled ? 'Room audio is disabled.' : status;
   return (
     <main className="min-h-screen bg-neutral-950 p-4 text-slate-200">