low quality preview for non-drivers

2026-09-16 09:31:20 -04:00 · 2026-01-13 01:45:15 -05:00
parent 56b89b5dff
commit c265121335
14 changed files with 551 additions and 59 deletions
@@ -1,6 +1,8 @@
+import { useEffect, useMemo, useState } from 'react';
 import { useSession } from '../context/SessionContext.jsx';
 import { useTelemetryFrame } from '../context/TelemetryContext.jsx';
 import { useVideoRequests } from '../hooks/useVideoRequests.js';
+import { useRoverSnapshots } from '../hooks/useRoverSnapshots.js';
 import { useControlSystem } from '../controls/index.js';
 import VideoTile from './VideoTile.jsx';

@@ -9,19 +11,51 @@ export default function DriverVideoPanel({layoutFormat = 'desktop'}) {
  const {
    state: { song },
  } = useControlSystem();
+  const [now, setNow] = useState(() => Date.now());
+  useEffect(() => {
+    if (session?.mode !== 'turns') {
+      return undefined;
+    }
+    const timer = setInterval(() => setNow(Date.now()), 250);
+    return () => clearInterval(timer);
+  }, [session?.mode]);
  const roverId = session?.assignment?.roverId;
  const rosterEntry =
    roverId && session?.roster ? session.roster.find((item) => String(item.id) === String(roverId)) : null;
  const hasAudio = Boolean(rosterEntry?.media?.audioPublishUrl);
+  const turnInfo = roverId ? session?.turnQueues?.[roverId] : null;
+  const socketId = session?.socketId || null;
+  const activeDriverId = roverId ? session?.activeDrivers?.[roverId] : null;
+  const isActiveDriver = Boolean(socketId && activeDriverId === socketId);
+  const nextDriverId = useMemo(() => {
+    const queue = turnInfo?.queue || [];
+    if (!queue.length || !turnInfo?.current || queue.length <= 1) return null;
+    const idx = queue.findIndex((id) => id === turnInfo.current);
+    if (idx === -1) {
+      return queue[0] || null;
+    }
+    return queue[(idx + 1) % queue.length] || null;
+  }, [turnInfo?.queue, turnInfo?.current]);
+  const isNextDriver = Boolean(socketId && nextDriverId === socketId);
+  const deadline = turnInfo?.deadline || null;
+  const msUntilTurn = deadline ? deadline - now : null;
+  const isPreSwitchWindow =
+    session?.mode === 'turns' && isNextDriver && msUntilTurn != null && msUntilTurn <= 5000 && msUntilTurn > 0;
+  const shouldShowVideo = session?.mode !== 'turns' || isActiveDriver || isPreSwitchWindow;
  const entries = roverId
    ? [
-        { type: 'rover', id: roverId, key: roverId },
+        ...(shouldShowVideo ? [{ type: 'rover', id: roverId, key: roverId }] : []),
        ...(hasAudio ? [{ type: 'rover', id: `${roverId}-audio`, key: `${roverId}-audio` }] : []),
      ]
    : [];
  const sources = useVideoRequests(entries);
-  const info = roverId ? sources[roverId] : null;
+  const info = roverId && shouldShowVideo ? sources[roverId] : null;
  const audioInfo = roverId && hasAudio ? sources[`${roverId}-audio`] : null;
+  const snapshotFeeds = useRoverSnapshots(roverId ? [roverId] : [], {
+    enabled: Boolean(roverId),
+    version: session?.mode,
+  });
+  const snapshotFeed = roverId ? snapshotFeeds[roverId] || null : null;
  const frame = useTelemetryFrame(roverId);
  const batteryRecord =
    roverId && session?.roster
@@ -36,12 +70,15 @@ export default function DriverVideoPanel({layoutFormat = 'desktop'}) {
      {roverId ? (
        <VideoTile
          sessionInfo={info}
+          videoMode={shouldShowVideo ? 'whep' : 'snapshot'}
+          snapshotFeed={snapshotFeed}
          audioSessionInfo={audioInfo}
          label={roverLabel}
          telemetryFrame={frame}
          batteryConfig={batteryConfig}
          layoutFormat={layoutFormat}
          songNote={song?.note}
+          qualityNotice={!shouldShowVideo ? 'Preview feed (low FPS) until your turn.' : null}
        />
      ) : (
        <div className="panel-muted content-center text-center text-sm text-slate-400 aspect-video">
@@ -45,6 +45,9 @@ function buildBatteryVisual(charge, config) {
 export default function VideoTile({
  sessionInfo,
  audioSessionInfo,
+  videoMode = 'whep',
+  snapshotFeed = null,
+  qualityNotice = null,
  label,
  forceMute = false,
  telemetryFrame,
@@ -70,6 +73,7 @@ export default function VideoTile({
  const [restartToken, setRestartToken] = useState(0);
  const [audioRestartToken, setAudioRestartToken] = useState(0);
  const [muted, setMuted] = useState(true);
+  const usingSnapshot = videoMode === 'snapshot';
  const sensors = telemetryFrame?.sensors;
  const batteryCharge = sensors?.batteryChargeMah ?? null;
  const desktopLayout = layoutFormat === 'desktop';
@@ -162,7 +166,14 @@ export default function VideoTile({
  }, [status, attemptUnmute]);

  useEffect(() => {
-    if (!sessionInfo?.url || !videoRef.current) {
+    if (usingSnapshot) {
+      setStatus('snapshot');
+      setDetail(null);
+    }
+  }, [usingSnapshot]);
+
+  useEffect(() => {
+    if (usingSnapshot || !sessionInfo?.url || !videoRef.current) {
      return undefined;
    }
    let active = true;
@@ -199,7 +210,7 @@ export default function VideoTile({
      clearTimeout(resetMuteId);
      player?.stop();
    };
-  }, [sessionInfo?.url, sessionInfo?.token, restartToken, scheduleRestart, ensurePlayback]);
+  }, [usingSnapshot, sessionInfo?.url, sessionInfo?.token, restartToken, scheduleRestart, ensurePlayback]);

  useEffect(() => {
    if (status === 'stopped' && sessionInfo?.url) {
@@ -329,7 +340,14 @@ export default function VideoTile({
    };
  }, [audioSessionInfo?.url]);

-  const renderedStatus = !sessionInfo?.url
+  const snapshotStatus = snapshotFeed?.error
+    ? `Error: ${snapshotFeed.error}`
+    : snapshotFeed?.objectUrl
+    ? 'snapshot'
+    : snapshotFeed?.status || 'waiting';
+  const renderedStatus = usingSnapshot
+    ? snapshotStatus
+    : !sessionInfo?.url
    ? 'waiting'
    : status === 'error'
    ? `Error: ${detail || 'unknown'}`
@@ -352,14 +370,29 @@ export default function VideoTile({
      <div
        className={`relative w-full overflow-hidden bg-black ${fitParent ? 'h-full flex-1' : 'aspect-video'}`}
      >
-        <video
-          ref={videoRef}
-          muted={forceMute || muted}
-          playsInline
-          autoPlay
-          controls={false}
-          className="h-full w-full object-contain"
-        />
+        {usingSnapshot ? (
+          snapshotFeed?.objectUrl ? (
+            <img
+              src={snapshotFeed.objectUrl}
+              alt={label}
+              className="h-full w-full object-contain"
+              draggable={false}
+            />
+          ) : (
+            <div className="flex h-full w-full items-center justify-center text-sm text-slate-300">
+              Waiting for frame…
+            </div>
+          )
+        ) : (
+          <video
+            ref={videoRef}
+            muted={forceMute || muted}
+            playsInline
+            autoPlay
+            controls={false}
+            className="h-full w-full object-contain"
+          />
+        )}
        <audio ref={audioRef} autoPlay hidden />
        <HudOverlay
          frame={telemetryFrame}
@@ -382,6 +415,11 @@ export default function VideoTile({
        {showVerticalBattery && batteryVisual.available ? (
          <BatteryBarVertical visual={batteryVisual} />
        ) : null}
+        {qualityNotice ? (
+          <div className="pointer-events-none absolute left-1 top-1 rounded bg-black/70 px-1 py-0.5 text-xs font-semibold text-amber-200">
+            {qualityNotice}
+          </div>
+        ) : null}
      </div>
      {!showVerticalBattery && (
        <div className="space-y-0.25">
@@ -0,0 +1,102 @@
+import { useEffect, useMemo, useRef, useState } from 'react';
+import { useSocket } from '../context/SocketContext.jsx';
+
+export function useRoverSnapshots(sourceList = [], options = {}) {
+  const socket = useSocket();
+  const { enabled = true, version = null } = options;
+  const [feeds, setFeeds] = useState({});
+  const objectUrls = useRef(new Map());
+  const ids = useMemo(
+    () => sourceList.map((e) => (typeof e === 'string' ? e : e.id)).filter(Boolean),
+    [sourceList],
+  );
+  const idsKey = useMemo(() => {
+    const base = ids.join('|');
+    return version ? `${base}|v:${version}` : base;
+  }, [ids, version]);
+  const idsRef = useRef([]);
+  const [connectionNonce, setConnectionNonce] = useState(0);
+
+  useEffect(() => {
+    if (!socket) return undefined;
+    const handleConnect = () => setConnectionNonce((prev) => prev + 1);
+    socket.on('connect', handleConnect);
+    return () => socket.off('connect', handleConnect);
+  }, [socket]);
+
+  useEffect(() => {
+    idsRef.current = ids;
+  }, [idsKey, ids]);
+
+  useEffect(() => {
+    objectUrls.current.forEach((url) => URL.revokeObjectURL(url));
+    objectUrls.current.clear();
+    setFeeds({});
+  }, [idsKey]);
+
+  useEffect(() => {
+    if (!enabled) {
+      objectUrls.current.forEach((url) => URL.revokeObjectURL(url));
+      objectUrls.current.clear();
+      setFeeds({});
+      return undefined;
+    }
+    if (!idsRef.current.length || !socket) {
+      return undefined;
+    }
+    let cancelled = false;
+    const currentIds = idsRef.current;
+
+    const handleFrame = (meta = {}, buffer) => {
+      if (cancelled || !meta.id || !buffer) return;
+      const blob = new Blob([buffer], { type: 'image/jpeg' });
+      const url = URL.createObjectURL(blob);
+      const prevUrl = objectUrls.current.get(meta.id);
+      if (prevUrl) {
+        URL.revokeObjectURL(prevUrl);
+      }
+      objectUrls.current.set(meta.id, url);
+      setFeeds((prev) => ({
+        ...prev,
+        [meta.id]: {
+          status: 'playing',
+          ts: meta.ts || Date.now(),
+          error: null,
+          objectUrl: url,
+        },
+      }));
+    };
+
+    const handleStatus = (meta = {}) => {
+      if (cancelled || !meta.id) return;
+      setFeeds((prev) => ({
+        ...prev,
+        [meta.id]: {
+          ...(prev[meta.id] || {}),
+          status: meta.error ? 'error' : prev[meta.id]?.status || 'connecting',
+          error: meta.error || null,
+          ts: meta.ts || prev[meta.id]?.ts || null,
+          objectUrl: prev[meta.id]?.objectUrl || null,
+        },
+      }));
+    };
+
+    socket.on('roverSnapshot:frame', handleFrame);
+    socket.on('roverSnapshot:status', handleStatus);
+
+    socket.emit('roverSnapshot:subscribe', { ids: currentIds }, (resp = {}) => {
+      if (resp.error) return;
+    });
+
+    return () => {
+      cancelled = true;
+      socket.emit('roverSnapshot:unsubscribe', { ids: currentIds });
+      socket.off('roverSnapshot:frame', handleFrame);
+      socket.off('roverSnapshot:status', handleStatus);
+      objectUrls.current.forEach((url) => URL.revokeObjectURL(url));
+      objectUrls.current.clear();
+    };
+  }, [socket, idsKey, enabled, connectionNonce]);
+
+  return feeds;
+}
@@ -3,6 +3,7 @@ import { SettingsProvider } from '../settings/index.js';
 import { useSession } from '../context/SessionContext.jsx';
 import { useTelemetryFrames } from '../context/TelemetryContext.jsx';
 import { useVideoRequests } from '../hooks/useVideoRequests.js';
+import { useRoverSnapshots } from '../hooks/useRoverSnapshots.js';
 import { useSpectatorMode } from '../hooks/useSpectatorMode.js';
 import { useRoomCameraSnapshots } from '../hooks/useRoomCameraSnapshots.js';
 import VideoTile from '../components/VideoTile.jsx';
@@ -33,30 +34,26 @@ function MiniSummaryContent() {
  });
  const [index, setIndex] = useState(0);

-  const entries = useMemo(
+  const snapshotFeeds = useRoverSnapshots(
+    roster.map((rover) => rover.id),
+    { enabled: !inLockdown, version: session?.mode },
+  );
+  const audioEntries = useMemo(
    () =>
      roster.flatMap((rover) => {
-        if (!rover?.id) return [];
+        if (!rover?.id || !rover.media?.audioPublishUrl) return [];
        const id = String(rover.id);
-        const base = [{ type: 'rover', id, key: id }];
-        if (rover.media?.audioPublishUrl) {
-          base.push({ type: 'rover', id: `${id}-audio`, key: `${id}-audio` });
-        }
-        return base;
+        return [{ type: 'rover', id: `${id}-audio`, key: `${id}-audio` }];
      }),
    [roster],
  );
-
-  const videoSourcesEnabled = useVideoRequests(entries, {
-    enabled: !inLockdown,
-    version: session?.mode,
-  });
+  const audioSources = useVideoRequests(audioEntries, { enabled: !inLockdown, version: session?.mode });

  const roverPool = useMemo(() => {
    if (!roster.length) return [];
-    const withVideo = roster.filter((rover) => videoSourcesEnabled[rover.id]?.url);
-    return withVideo.length ? withVideo : roster;
-  }, [roster, videoSourcesEnabled]);
+    const withSnapshot = roster.filter((rover) => snapshotFeeds[rover.id]?.objectUrl);
+    return withSnapshot.length ? withSnapshot : roster;
+  }, [roster, snapshotFeeds]);

  const rotationPool = useMemo(() => {
    const items = [];
@@ -91,8 +88,8 @@ function MiniSummaryContent() {
  const activeRover = activeEntry?.type === 'rover' ? activeEntry.rover : null;
  const activeCamera = activeEntry?.type === 'room' ? activeEntry.camera : null;

-  const activeVideo = activeRover ? videoSourcesEnabled[activeRover.id] || null : null;
-  const activeAudio = activeRover ? videoSourcesEnabled[`${activeRover.id}-audio`] || null : null;
+  const activeSnapshot = activeRover ? snapshotFeeds[activeRover.id] || null : null;
+  const activeAudio = activeRover ? audioSources[`${activeRover.id}-audio`] || null : null;
  const activeFrame = activeRover ? frames[activeRover.id] || null : null;
  const driverLabel = activeRover ? formatDriverLabel({ roverId: activeRover.id, session }) : null;
  const activeFeed = activeCamera ? feeds[activeCamera.id] || null : null;
@@ -119,7 +116,9 @@ function MiniSummaryContent() {
        ) : activeRover ? (
          <FitViewportFrame>
            <VideoTile
-              sessionInfo={activeVideo}
+              sessionInfo={null}
+              videoMode="snapshot"
+              snapshotFeed={activeSnapshot}
              audioSessionInfo={activeAudio}
              label={activeRover.name || activeRover.id}
              telemetryFrame={activeFrame}
@@ -3,6 +3,7 @@ import { useSession } from '../context/SessionContext.jsx';
 import { useSpectatorMode } from '../hooks/useSpectatorMode.js';
 import { useTelemetryFrames } from '../context/TelemetryContext.jsx';
 import { useVideoRequests } from '../hooks/useVideoRequests.js';
+import { useRoverSnapshots } from '../hooks/useRoverSnapshots.js';
 import VideoTile from '../components/VideoTile.jsx';
 import RoomCameraPanel from '../components/RoomCameraPanel.jsx';
 import UserListPanel from '../components/UserListPanel.jsx';
@@ -22,13 +23,15 @@ function formatDriverLabel({ roverId, session }) {
  return driverText;
 }

-function RoverSpectatorCard({ rover, frame, videoInfo, audioInfo, session }) {
+function RoverSpectatorCard({ rover, frame, snapshotFeed, audioInfo, session }) {
  const driverLabel = formatDriverLabel({ roverId: rover.id, session });
  return (
    <article className="min-h-[16rem] rounded bg-zinc-900 p-0.5 sm:min-h-[18rem]">
      <div className="min-h-0 overflow-hidden rounded bg-black/20">
        <VideoTile
-          sessionInfo={videoInfo}
+          sessionInfo={null}
+          videoMode="snapshot"
+          snapshotFeed={snapshotFeed}
          audioSessionInfo={audioInfo}
          label={rover.name}
          telemetryFrame={frame}
@@ -43,7 +46,7 @@ function RoverSpectatorCard({ rover, frame, videoInfo, audioInfo, session }) {
  );
 }

-function RoverRow({ roster, frames, videoSources, session }) {
+function RoverRow({ roster, frames, snapshotFeeds, audioSources, session }) {
  if (roster.length === 0) {
    return <p className="col-span-full text-slate-400">No rovers registered.</p>;
  }
@@ -54,8 +57,8 @@ function RoverRow({ roster, frames, videoSources, session }) {
          key={rover.id}
          rover={rover}
          frame={frames[rover.id]}
-          videoInfo={videoSources[rover.id]}
-          audioInfo={videoSources[`${rover.id}-audio`]}
+          snapshotFeed={snapshotFeeds[rover.id]}
+          audioInfo={audioSources[`${rover.id}-audio`]}
          session={session}
          showHudMap
          hudMapPosition="bottom-left"
@@ -94,14 +97,16 @@ export default function SpectatorApp() {
  useSpectatorMode();
  const frames = useTelemetryFrames();
  const roster = session?.roster ?? [];
-  const entries = roster.flatMap((rover) => {
-    const base = { type: 'rover', id: rover.id, key: rover.id };
-    if (rover.media?.audioPublishUrl) {
-      return [base, { type: 'rover', id: `${rover.id}-audio`, key: `${rover.id}-audio` }];
-    }
-    return [base];
-  });
-  const videoSources = useVideoRequests(entries, { enabled: !inLockdown, version: session?.mode });
+  const snapshotFeeds = useRoverSnapshots(
+    roster.map((rover) => rover.id),
+    { enabled: !inLockdown, version: session?.mode },
+  );
+  const audioEntries = roster.flatMap((rover) =>
+    rover.media?.audioPublishUrl
+      ? [{ type: 'rover', id: `${rover.id}-audio`, key: `${rover.id}-audio` }]
+      : [],
+  );
+  const audioSources = useVideoRequests(audioEntries, { enabled: !inLockdown, version: session?.mode });

  if (inLockdown) {
    return (
@@ -121,7 +126,13 @@ export default function SpectatorApp() {
      <div className="min-h-screen bg-black text-slate-100 md:h-screen md:overflow-hidden">
        <main className="grid min-h-screen grid-cols-1 gap-0.5 p-0.5 md:h-full md:min-h-0 md:grid-cols-[minmax(0,1fr)_18rem] lg:grid-cols-[minmax(0,1fr)_20rem]">
          <section className="flex min-h-0 min-w-0 flex-col gap-0.5 md:overflow-y-auto">
-            <RoverRow roster={roster} frames={frames} videoSources={videoSources} session={session} />
+            <RoverRow
+              roster={roster}
+              frames={frames}
+              snapshotFeeds={snapshotFeeds}
+              audioSources={audioSources}
+              session={session}
+            />
            <SecondaryRow />
          </section>
          <section className="flex min-h-0 min-w-0 flex-col gap-0.5 md:h-full">