pass 1

webui/src/components/DriverVideoPanel.jsx

@@ -5,6 +5,7 @@ import { useVideoRequests } from '../hooks/useVideoRequests.js';
 import { useRoverSnapshots } from '../hooks/useRoverSnapshots.js';
 import { useControlSystem } from '../controls/index.js';
 import VideoTile from './VideoTile.jsx';
+import { supportsAv1WebRtc } from '../lib/mediaSupport.js';
 
 export default function DriverVideoPanel({layoutFormat = 'desktop'}) {
   const { session } = useSession();
@@ -23,6 +24,7 @@ export default function DriverVideoPanel({layoutFormat = 'desktop'}) {
     return () => clearInterval(timer);
   }, [session?.mode]);
   const roverId = session?.assignment?.roverId;
+  const av1Supported = supportsAv1WebRtc();
   const rosterEntry =
     roverId && session?.roster ? session.roster.find((item) => String(item.id) === String(roverId)) : null;
   const hasAudio = Boolean(rosterEntry?.media?.audioPublishUrl);
@@ -69,6 +71,19 @@ export default function DriverVideoPanel({layoutFormat = 'desktop'}) {
   const sources = useVideoRequests(entries);
   const info = roverId && shouldShowVideo ? sources[roverId] : null;
   const audioInfo = roverId && hasAudio ? sources[`${roverId}-audio`] : null;
+  const previewEntries = roverId
+    ? [
+        {
+          type: 'rover',
+          id: roverId,
+          key: `rover:${roverId}:preview:av1`,
+          preview: true,
+          codec: 'av1',
+        },
+      ]
+    : [];
+  const previewSources = useVideoRequests(previewEntries, { enabled: Boolean(roverId) && av1Supported });
+  const previewSession = roverId ? previewSources[`rover:${roverId}:preview:av1`] || null : null;
   const snapshotFeeds = useRoverSnapshots(roverId ? [roverId] : [], {
     enabled: Boolean(roverId),
     version: session?.mode,
@@ -114,8 +129,8 @@ export default function DriverVideoPanel({layoutFormat = 'desktop'}) {
     <section className="panel">
       {roverId ? (
         <VideoTile
-          sessionInfo={info}
-          videoMode={shouldShowVideo ? 'whep' : 'snapshot'}
+          sessionInfo={shouldShowVideo ? info : previewSession?.url ? previewSession : null}
+          videoMode={shouldShowVideo ? 'whep' : previewSession?.url ? 'whep' : 'snapshot'}
           snapshotFeed={snapshotFeed}
           audioSessionInfo={audioInfo}
           label={roverLabel}
@@ -123,7 +138,13 @@ export default function DriverVideoPanel({layoutFormat = 'desktop'}) {
           batteryConfig={batteryConfig}
           layoutFormat={layoutFormat}
           songNote={song?.note}
-          qualityNotice={!shouldShowVideo ? 'Preview feed (low FPS) until your turn.' : null}
+          qualityNotice={
+            !shouldShowVideo
+              ? previewSession?.url
+                ? 'Preview feed (AV1) until your turn.'
+                : 'Preview feed (snapshots) until your turn.'
+              : null
+          }
           showTurnCue={turnCueVisible}
           turnTimerText={turnTimerText}
           turnSeconds={turnSeconds}

webui/src/components/RoomCameraFeed.jsx

@@ -1,7 +1,74 @@
-import { useEffect, useMemo, useState } from 'react';
+import { useEffect, useMemo, useRef, useState } from 'react';
+import { WhepPlayer } from '../lib/whepPlayer.js';
 
-export default function RoomCameraFeed({ feed, label }) {
+function RoomCameraVideo({ sessionInfo, label, onStatus }) {
+  const videoRef = useRef(null);
+  const [status, setStatus] = useState('idle');
+  const [detail, setDetail] = useState(null);
+
+  useEffect(() => {
+    if (!sessionInfo?.url || !videoRef.current) return undefined;
+    let active = true;
+    let player;
+
+    const handleStatus = (nextStatus, info) => {
+      if (!active) return;
+      setStatus(nextStatus);
+      setDetail(info || null);
+      if (typeof onStatus === 'function') {
+        onStatus(nextStatus);
+      }
+    };
+
+    player = new WhepPlayer({
+      url: sessionInfo.url,
+      token: sessionInfo.token,
+      video: videoRef.current,
+      onStatus: handleStatus,
+    });
+
+    player.start().catch((err) => {
+      if (!active) return;
+      setStatus('error');
+      setDetail(err.message);
+      if (typeof onStatus === 'function') {
+        onStatus('error');
+      }
+    });
+
+    return () => {
+      active = false;
+      player?.stop();
+    };
+  }, [sessionInfo?.url, sessionInfo?.token, onStatus]);
+
+  return (
+    <div className="relative h-full w-full">
+      <video
+        ref={videoRef}
+        className="h-full w-full object-cover"
+        muted
+        playsInline
+        autoPlay
+        controls={false}
+        aria-label={label}
+      />
+      {status !== 'playing' && (
+        <div className="absolute inset-0 flex items-center justify-center text-sm text-slate-300">
+          {detail ? `Video error: ${detail}` : 'Connecting video…'}
+        </div>
+      )}
+    </div>
+  );
+}
+
+export default function RoomCameraFeed({ feed, label, videoSession = null, preferVideo = false }) {
   const [blink, setBlink] = useState(false);
+  const [videoFailed, setVideoFailed] = useState(false);
+
+  useEffect(() => {
+    setVideoFailed(false);
+  }, [videoSession?.url, videoSession?.token]);
 
   useEffect(() => {
     if (!feed) return;
@@ -15,12 +82,27 @@ export default function RoomCameraFeed({ feed, label }) {
     return feed.status || 'Connecting…';
   }, [feed]);
 
+  const showVideo = Boolean(preferVideo && videoSession?.url && !videoFailed);
+  const showSnapshot = Boolean(!showVideo && feed?.objectUrl);
+
   return (
     <div className="relative w-full overflow-hidden rounded bg-black" style={{ aspectRatio: '4 / 3' }}>
-      {feed?.objectUrl ? (
+      {showVideo ? (
+        <RoomCameraVideo
+          sessionInfo={videoSession}
+          label={label}
+          onStatus={(nextStatus) => {
+            if (['error', 'failed', 'disconnected', 'closed'].includes(nextStatus)) {
+              setVideoFailed(true);
+            }
+          }}
+        />
+      ) : showSnapshot ? (
         <img src={feed.objectUrl} alt={label} className="h-full w-full object-cover" />
       ) : (
-        <div className="flex h-full w-full items-center justify-center text-sm text-slate-300">Waiting for frame…</div>
+        <div className="flex h-full w-full items-center justify-center text-sm text-slate-300">
+          {preferVideo ? 'Waiting for video…' : 'Waiting for frame…'}
+        </div>
       )}
       <div className="pointer-events-none absolute left-0 top-0 bg-black/70 px-0.5 py-0.5 text-xs font-semibold text-white">
         {label}

webui/src/components/RoomCameraPanel.jsx

@@ -2,6 +2,8 @@ import { useEffect, useState } from 'react';
 import { useSession } from '../context/SessionContext.jsx';
 import { useSettingsNamespace } from '../settings/index.js';
 import { useRoomCameraSnapshots } from '../hooks/useRoomCameraSnapshots.js';
+import { useVideoRequests } from '../hooks/useVideoRequests.js';
+import { supportsAv1WebRtc } from '../lib/mediaSupport.js';
 import RoomCameraFeed from './RoomCameraFeed.jsx';
 
 function EmptyState() {
@@ -32,6 +34,15 @@ export default function RoomCameraPanel({
   const { session } = useSession();
   const cameras = session?.roomCameras || [];
   const feedMap = useRoomCameraSnapshots(cameras.map((camera) => ({ id: camera.id })));
+  const av1Supported = supportsAv1WebRtc();
+  const previewEntries = cameras.map((camera) => ({
+    type: 'room',
+    id: camera.id,
+    key: `room:${camera.id}:preview:av1`,
+    preview: true,
+    codec: 'av1',
+  }));
+  const previewSources = useVideoRequests(previewEntries, { enabled: av1Supported });
   const { value: orientationSettings, save: saveOrientationSettings } = useSettingsNamespace('roomCameraPanels', {});
   const [orientation, setOrientation] = useState(() =>
     normalizeOrientation(
@@ -96,13 +107,19 @@ export default function RoomCameraPanel({
       <div className={containerClass}>
         {cameras.map((camera) => {
           const feed = feedMap[camera.id] || null;
+          const previewSession = previewSources[`room:${camera.id}:preview:av1`] || null;
           return (
             <article key={camera.id} className="w-full space-y-0.5 rounded bg-zinc-950 p-0.5 shadow-inner shadow-black/40">
               {/* <header className="space-y-0.5">
                 <p className="text-lg font-semibold text-white">{camera.name || camera.id}</p>
                 {camera.description && <p className="text-xs text-slate-500">{camera.description}</p>}
               </header> */}
-              <RoomCameraFeed feed={feed} label={camera.name || camera.id} />
+              <RoomCameraFeed
+                feed={feed}
+                label={camera.name || camera.id}
+                videoSession={previewSession}
+                preferVideo={av1Supported}
+              />
             </article>
           );
         })}

webui/src/hooks/useVideoRequests.js

@@ -11,23 +11,42 @@ function normalizeEntry(entry) {
   if (typeof entry === 'object') {
     if (entry.type && entry.id) {
       const id = String(entry.id);
+      const preview = Boolean(entry.preview);
+      const codec = entry.codec ? String(entry.codec) : null;
       let key = entry.key;
       if (!key) {
         key = entry.type === 'room' ? `room:${id}` : id;
+        if (preview) {
+          key = `${key}:preview${codec ? `:${codec}` : ''}`;
+        }
       }
       return {
         type: entry.type,
         id,
         key,
+        preview,
+        codec,
       };
     }
     if (entry.roverId) {
       const id = String(entry.roverId);
-      return { type: 'rover', id, key: entry.key || id };
+      return {
+        type: 'rover',
+        id,
+        key: entry.key || id,
+        preview: Boolean(entry.preview),
+        codec: entry.codec ? String(entry.codec) : null,
+      };
     }
     if (entry.roomCameraId) {
       const id = String(entry.roomCameraId);
-      return { type: 'room', id, key: entry.key || `room:${id}` };
+      return {
+        type: 'room',
+        id,
+        key: entry.key || `room:${id}`,
+        preview: Boolean(entry.preview),
+        codec: entry.codec ? String(entry.codec) : null,
+      };
     }
   }
   return null;
@@ -86,6 +105,12 @@ export function useVideoRequests(sourceList = [], options = {}) {
 
     function requestEntry(entry) {
       const payload = entry.type === 'room' ? { roomCameraId: entry.id } : { roverId: entry.id };
+      if (entry.preview) {
+        payload.preview = true;
+        if (entry.codec) {
+          payload.codec = entry.codec;
+        }
+      }
       socket.emit('video:request', payload, (resp = {}) => {
         if (cancelled) return;
         setSources((prev) => ({ ...prev, [entry.key]: resp }));

webui/src/lib/mediaSupport.js

@@ -0,0 +1,28 @@
+let cachedAv1Support = null;
+
+function hasAv1CodecCapability() {
+  if (typeof RTCRtpReceiver !== 'undefined' && RTCRtpReceiver.getCapabilities) {
+    const caps = RTCRtpReceiver.getCapabilities('video');
+    const codecs = caps?.codecs || [];
+    return codecs.some((codec) => {
+      const mime = (codec?.mimeType || '').toLowerCase();
+      return mime === 'video/av1' || mime === 'video/av01' || mime === 'video/av1x';
+    });
+  }
+  if (typeof document !== 'undefined') {
+    const video = document.createElement('video');
+    if (typeof video.canPlayType === 'function') {
+      const result = video.canPlayType('video/mp4; codecs="av01.0.05M.08"');
+      return result === 'probably' || result === 'maybe';
+    }
+  }
+  return false;
+}
+
+export function supportsAv1WebRtc() {
+  if (cachedAv1Support != null) {
+    return cachedAv1Support;
+  }
+  cachedAv1Support = hasAv1CodecCapability();
+  return cachedAv1Support;
+}

webui/src/mini/MiniSummaryApp.jsx

@@ -10,6 +10,8 @@ import VideoTile from '../components/VideoTile.jsx';
 import ChatPanel from '../components/ChatPanel.jsx';
 import AlertFeed from '../components/AlertFeed.jsx';
 import useDefaultNickname from '../hooks/useDefaultNickname.js';
+import RoomCameraFeed from '../components/RoomCameraFeed.jsx';
+import { supportsAv1WebRtc } from '../lib/mediaSupport.js';
 
 const ROTATE_MS = 20000;
 
@@ -34,12 +36,29 @@ function MiniSummaryContent() {
     enabled: !inLockdown,
     version: session?.mode,
   });
+  const av1Supported = supportsAv1WebRtc();
   const [index, setIndex] = useState(0);
 
   const snapshotFeeds = useRoverSnapshots(
     roster.map((rover) => rover.id),
     { enabled: !inLockdown, version: session?.mode },
   );
+  const previewEntries = roster.map((rover) => ({
+    type: 'rover',
+    id: rover.id,
+    key: `rover:${rover.id}:preview:av1`,
+    preview: true,
+    codec: 'av1',
+  }));
+  const roomPreviewEntries = roomCameras.map((camera) => ({
+    type: 'room',
+    id: camera.id,
+    key: `room:${camera.id}:preview:av1`,
+    preview: true,
+    codec: 'av1',
+  }));
+  const previewSources = useVideoRequests(previewEntries, { enabled: !inLockdown && av1Supported });
+  const roomPreviewSources = useVideoRequests(roomPreviewEntries, { enabled: !inLockdown && av1Supported });
   const audioEntries = useMemo(
     () =>
       roster.flatMap((rover) => {
@@ -91,10 +110,18 @@ function MiniSummaryContent() {
   const activeCamera = activeEntry?.type === 'room' ? activeEntry.camera : null;
 
   const activeSnapshot = activeRover ? snapshotFeeds[activeRover.id] || null : null;
+  const activePreview =
+    activeRover && previewSources[`rover:${activeRover.id}:preview:av1`]
+      ? previewSources[`rover:${activeRover.id}:preview:av1`]
+      : null;
   const activeAudio = activeRover ? audioSources[`${activeRover.id}-audio`] || null : null;
   const activeFrame = activeRover ? frames[activeRover.id] || null : null;
   const driverLabel = activeRover ? formatDriverLabel({ roverId: activeRover.id, session }) : null;
   const activeFeed = activeCamera ? feeds[activeCamera.id] || null : null;
+  const activeRoomPreview =
+    activeCamera && roomPreviewSources[`room:${activeCamera.id}:preview:av1`]
+      ? roomPreviewSources[`room:${activeCamera.id}:preview:av1`]
+      : null;
 
   if (inLockdown) {
     return (
@@ -118,8 +145,8 @@ function MiniSummaryContent() {
         ) : activeRover ? (
           <FitViewportFrame>
             <VideoTile
-              sessionInfo={null}
-              videoMode="snapshot"
+              sessionInfo={activePreview?.url ? activePreview : null}
+              videoMode={activePreview?.url ? 'whep' : 'snapshot'}
               snapshotFeed={activeSnapshot}
               audioSessionInfo={activeAudio}
               label={activeRover.name || activeRover.id}
@@ -135,7 +162,7 @@ function MiniSummaryContent() {
           </FitViewportFrame>
         ) : activeCamera ? (
           <FitViewportFrame>
-            <RoomCameraFrame camera={activeCamera} feed={activeFeed} />
+            <RoomCameraFrame camera={activeCamera} feed={activeFeed} videoSession={activeRoomPreview} preferVideo={av1Supported} />
           </FitViewportFrame>
         ) : (
           <div className="flex h-full w-full items-center justify-center text-sm text-slate-500">
@@ -158,26 +185,15 @@ export default function MiniSummaryApp() {
   );
 }
 
-function RoomCameraFrame({ camera, feed }) {
-  const hasImage = feed?.objectUrl;
-  const connecting = feed && feed.status === 'connecting';
+function RoomCameraFrame({ camera, feed, videoSession, preferVideo }) {
   return (
     <div className="relative h-full w-full bg-zinc-950">
-      {hasImage ? (
-        <img
-          src={feed.objectUrl}
-          alt={camera.name || camera.id}
-          className="h-full w-full object-cover"
-          draggable={false}
-        />
-      ) : (
-        <div className="flex h-full w-full items-center justify-center text-sm text-slate-500">
-          {connecting ? `Connecting to ${camera.name || camera.id}…` : 'No frame yet'}
-        </div>
-      )}
-      <div className="absolute left-1 top-1 rounded bg-black/60 px-1 py-0.25 text-xs text-slate-200">
-        {camera.name || camera.id}
-      </div>
+      <RoomCameraFeed
+        feed={feed}
+        label={camera.name || camera.id}
+        videoSession={videoSession}
+        preferVideo={preferVideo}
+      />
     </div>
   );
 }

webui/src/spectate/SpectatorApp.jsx

@@ -13,6 +13,7 @@ import RoverRoster from '../components/RoverRoster.jsx';
 import AlertFeed from '../components/AlertFeed.jsx';
 import useDefaultNickname from '../hooks/useDefaultNickname.js';
 import CommunityGoalBanner from '../components/CommunityGoalBanner.jsx';
+import { supportsAv1WebRtc } from '../lib/mediaSupport.js';
 
 function formatDriverLabel({ roverId, session }) {
   const activeDriverId = session?.activeDrivers?.[roverId] || null;
@@ -25,14 +26,15 @@ function formatDriverLabel({ roverId, session }) {
   return driverText;
 }
 
-function RoverSpectatorCard({ rover, frame, snapshotFeed, audioInfo, session }) {
+function RoverSpectatorCard({ rover, frame, snapshotFeed, previewSession, audioInfo, session }) {
   const driverLabel = formatDriverLabel({ roverId: rover.id, session });
+  const hasPreview = Boolean(previewSession?.url);
   return (
     <article className="min-h-[16rem] rounded bg-zinc-900 p-0.5 sm:min-h-[18rem]">
       <div className="min-h-0 overflow-hidden rounded bg-black/20">
         <VideoTile
-          sessionInfo={null}
-          videoMode="snapshot"
+          sessionInfo={hasPreview ? previewSession : null}
+          videoMode={hasPreview ? 'whep' : 'snapshot'}
           snapshotFeed={snapshotFeed}
           audioSessionInfo={audioInfo}
           label={rover.name}
@@ -48,7 +50,7 @@ function RoverSpectatorCard({ rover, frame, snapshotFeed, audioInfo, session })
   );
 }
 
-function RoverRow({ roster, frames, snapshotFeeds, audioSources, session }) {
+function RoverRow({ roster, frames, snapshotFeeds, previewSources, audioSources, session }) {
   if (roster.length === 0) {
     return <p className="col-span-full text-slate-400">No rovers registered.</p>;
   }
@@ -60,6 +62,7 @@ function RoverRow({ roster, frames, snapshotFeeds, audioSources, session }) {
           rover={rover}
           frame={frames[rover.id]}
           snapshotFeed={snapshotFeeds[rover.id]}
+          previewSession={previewSources[`rover:${rover.id}:preview:av1`] || null}
           audioInfo={audioSources[`${rover.id}-audio`]}
           session={session}
           showHudMap
@@ -100,10 +103,19 @@ function SpectatorContent() {
   useSpectatorMode();
   const frames = useTelemetryFrames();
   const roster = session?.roster ?? [];
+  const av1Supported = supportsAv1WebRtc();
   const snapshotFeeds = useRoverSnapshots(
     roster.map((rover) => rover.id),
     { enabled: !inLockdown, version: session?.mode },
   );
+  const previewEntries = roster.map((rover) => ({
+    type: 'rover',
+    id: rover.id,
+    key: `rover:${rover.id}:preview:av1`,
+    preview: true,
+    codec: 'av1',
+  }));
+  const previewSources = useVideoRequests(previewEntries, { enabled: !inLockdown && av1Supported, version: session?.mode });
   const audioEntries = roster.flatMap((rover) =>
     rover.media?.audioPublishUrl
       ? [{ type: 'rover', id: `${rover.id}-audio`, key: `${rover.id}-audio` }]
@@ -130,6 +142,7 @@ function SpectatorContent() {
             roster={roster}
             frames={frames}
             snapshotFeeds={snapshotFeeds}
+            previewSources={previewSources}
             audioSources={audioSources}
             session={session}
           />