jpeg roomcams first pass

README.md

@@ -81,6 +81,8 @@ Publishing rovers lives on a trusted network, so the shipped config (tracked at
 
 Once finished, update `server/config.yaml` with your admin passwords and `media.whepBaseUrl` (set it to the URL you expose publicly, e.g. `https://rover.otter.land/video`). If your proxy can’t rewrite paths, create the `/video` location there and add a custom nginx snippet to rewrite `/video/<rover>/whep` to `/<>/whep` before forwarding to mediaMTX. Restart `multirover.service` whenever you edit the config. To pull updates later, just `git pull`, re-run `npm install --production` inside `server/`, and restart the service—no need to rerun the installer.
 
+Room cameras now use JPEG snapshots (4 fps) instead of WHEP. Each entry in `roomCameras` must include a `url` pointing at the snapshot endpoint; the server polls and relays frames over socket.io with the same access rules as before.
+
 ### Video handshake + diagnostics
 
 - Every `video:request` returns `{ url, token }`. The browser posts the SDP offer to `url` and includes `Authorization: Basic base64(token:token)`. mediaMTX forwards the username (`token`) to `/mediamtx/auth`, which checks the socket’s permissions (driver assignment, admin/spectator role, lockdown state) and either returns 200 or 401—no query parameters are involved anymore.

server/config.example.yaml

@@ -26,9 +26,11 @@ roomCameras:
   - id: "lobby"
     name: "Lobby Camera"
     description: "Wide shot of the staging area."
+    url: "http://192.168.0.50/snapshot.jpg"
   - id: "workshop"
     name: "Workshop Bench"
     description: "Shows the workbench and charging docks."
+    url: "http://192.168.0.51/snapshot.jpg"
 
 discord:
   token: "DISCORD_BOT_TOKEN"

server/index.js

@@ -20,6 +20,7 @@ require('./src/services/chatService');
 require('./src/services/videoSessions');
 require('./src/services/videoAuthService');
 require('./src/services/videoSocketService');
+require('./src/services/roomCameraSocketService');
 require('./src/services/logStreamService');
 require('./src/services/homeAssistantService');
 require('./src/services/sessionService');

server/src/services/roomCameraService.js

@@ -14,10 +14,15 @@ function normalizeCamera(camera) {
     logger.warn('Room camera missing id', camera);
     return null;
   }
+  if (!camera.url) {
+    logger.warn('Room camera missing url', { id, camera });
+    return null;
+  }
   return {
     id: String(id),
     name: camera.name || camera.id || String(id),
     description: camera.description || null,
+    url: camera.url,
   };
 }
 

server/src/services/roomCameraSnapshotService.js

@@ -0,0 +1,99 @@
+const EventEmitter = require('events');
+const logger = require('../globals/logger').child('roomCameraSnapshot');
+const { getRoomCameras, roomCameraEvents } = require('./roomCameraService');
+
+const POLL_INTERVAL_MS = 250; // 4 fps target
+const FETCH_TIMEOUT_MS = 2000;
+const STALE_AFTER_MS = 4000;
+
+const cameraState = new Map(); // id -> {frame, ts, stale, error, failures, fetching}
+const pollers = new Map();
+const events = new EventEmitter(); // frame, status
+
+function markState(id, updates = {}) {
+  const prev = cameraState.get(id) || {};
+  const next = { ...prev, ...updates };
+  if (next.ts != null) {
+    next.stale = Date.now() - next.ts > STALE_AFTER_MS || !!next.stale;
+  } else {
+    next.stale = true;
+  }
+  cameraState.set(id, next);
+  return next;
+}
+
+async function fetchSnapshot(camera) {
+  const { id, url } = camera;
+  const state = cameraState.get(id);
+  if (!url || state?.fetching) return;
+  markState(id, { fetching: true });
+  const abortController = new AbortController();
+  const timeout = setTimeout(() => abortController.abort(), FETCH_TIMEOUT_MS);
+  try {
+    const res = await fetch(url, { signal: abortController.signal });
+    if (!res.ok) {
+      throw new Error(`HTTP ${res.status}`);
+    }
+    const arrayBuffer = await res.arrayBuffer();
+    const buffer = Buffer.from(arrayBuffer);
+    const ts = Date.now();
+    const next = markState(id, { frame: buffer, ts, error: null, failures: 0 });
+    events.emit('frame', { id, buffer, ts, stale: !!next.stale });
+  } catch (err) {
+    const failures = (state?.failures || 0) + 1;
+    markState(id, { error: err.message, failures });
+    events.emit('status', { id, error: err.message });
+    logger.debug('Snapshot fetch failed', { id, err: err.message });
+  } finally {
+    clearTimeout(timeout);
+    markState(id, { fetching: false });
+  }
+}
+
+function startCamera(camera) {
+  const { id, url } = camera;
+  if (!url) {
+    logger.warn('Skipping room camera without URL', { id });
+    return;
+  }
+  if (pollers.has(id)) return;
+  markState(id, { frame: null, ts: null, error: null, failures: 0, fetching: false });
+  const poller = setInterval(() => fetchSnapshot(camera), POLL_INTERVAL_MS);
+  pollers.set(id, poller);
+  logger.info('Started snapshot polling', { id, url });
+}
+
+function stopAll() {
+  pollers.forEach((poller) => clearInterval(poller));
+  pollers.clear();
+  cameraState.clear();
+}
+
+function startAll() {
+  stopAll();
+  getRoomCameras().forEach((camera) => startCamera(camera));
+}
+
+function getState(id) {
+  const state = cameraState.get(id);
+  if (!state) return null;
+  const stale = state.ts == null || Date.now() - state.ts > STALE_AFTER_MS;
+  return {
+    frame: state.frame || null,
+    ts: state.ts || null,
+    stale,
+    error: state.error || null,
+  };
+}
+
+roomCameraEvents.on('update', () => {
+  logger.info('Room cameras changed; restarting snapshot pollers');
+  startAll();
+});
+
+startAll();
+
+module.exports = {
+  roomCameraStreamEvents: events,
+  getRoomCameraState: getState,
+};

server/src/services/roomCameraSocketService.js

@@ -0,0 +1,126 @@
+const io = require('../globals/io');
+const logger = require('../globals/logger').child('roomCameraSocket');
+const { getMode, MODES } = require('./modeManager');
+const { isAdmin, isLockdownAdmin, getRole } = require('./roleService');
+const { getRoomCamera } = require('./roomCameraService');
+const { roomCameraStreamEvents, getRoomCameraState } = require('./roomCameraSnapshotService');
+
+function passesMode(socket) {
+  const mode = getMode();
+  if (mode === MODES.LOCKDOWN) {
+    return isLockdownAdmin(socket);
+  }
+  if (mode === MODES.ADMIN) {
+    const role = getRole(socket);
+    return role === 'spectator' || isAdmin(socket);
+  }
+  return true;
+}
+
+function canViewRoomCamera(socket) {
+  return passesMode(socket);
+}
+
+const cameraSubscribers = new Map(); // id -> Set(socketId)
+const socketSubscriptions = new Map(); // socketId -> Set(id)
+
+function addSubscription(socket, cameraId) {
+  if (!cameraSubscribers.has(cameraId)) {
+    cameraSubscribers.set(cameraId, new Set());
+  }
+  cameraSubscribers.get(cameraId).add(socket.id);
+
+  if (!socketSubscriptions.has(socket.id)) {
+    socketSubscriptions.set(socket.id, new Set());
+  }
+  socketSubscriptions.get(socket.id).add(cameraId);
+}
+
+function removeSubscription(socketId, cameraId) {
+  const bucket = cameraSubscribers.get(cameraId);
+  if (bucket) {
+    bucket.delete(socketId);
+    if (bucket.size === 0) {
+      cameraSubscribers.delete(cameraId);
+    }
+  }
+  const socketBucket = socketSubscriptions.get(socketId);
+  if (socketBucket) {
+    socketBucket.delete(cameraId);
+    if (socketBucket.size === 0) {
+      socketSubscriptions.delete(socketId);
+    }
+  }
+}
+
+function removeAllSubscriptions(socketId) {
+  const bucket = socketSubscriptions.get(socketId);
+  if (!bucket) return;
+  bucket.forEach((cameraId) => removeSubscription(socketId, cameraId));
+}
+
+function sendFrame(socket, cameraId, payload, buffer) {
+  socket.emit('roomCamera:frame', { id: cameraId, ...payload }, buffer);
+}
+
+function sendStatus(socket, cameraId, status) {
+  socket.emit('roomCamera:status', { id: cameraId, ...status });
+}
+
+roomCameraStreamEvents.on('frame', ({ id, buffer, ts, stale }) => {
+  const bucket = cameraSubscribers.get(id);
+  if (!bucket || !buffer) return;
+  bucket.forEach((socketId) => {
+    const socket = io.sockets.sockets.get(socketId);
+    if (!socket) return;
+    sendFrame(socket, id, { ts, stale: !!stale }, buffer);
+  });
+});
+
+roomCameraStreamEvents.on('status', ({ id, error }) => {
+  const bucket = cameraSubscribers.get(id);
+  if (!bucket) return;
+  bucket.forEach((socketId) => {
+    const socket = io.sockets.sockets.get(socketId);
+    if (!socket) return;
+    sendStatus(socket, id, { error: error || null });
+  });
+});
+
+io.on('connection', (socket) => {
+  socket.on('roomCamera:subscribe', (payload = {}, cb = () => {}) => {
+    const cameraId = payload.roomCameraId || payload.id;
+    try {
+      if (!cameraId) {
+        throw new Error('roomCameraId required');
+      }
+      const camera = getRoomCamera(cameraId);
+      if (!camera) {
+        throw new Error('Unknown room camera');
+      }
+      if (!canViewRoomCamera(socket)) {
+        throw new Error('Not authorized for room camera');
+      }
+      addSubscription(socket, camera.id);
+      const state = getRoomCameraState(camera.id);
+      if (state?.frame) {
+        sendFrame(socket, camera.id, { ts: state.ts, stale: !!state.stale }, state.frame);
+      }
+      sendStatus(socket, camera.id, { ts: state?.ts || null, stale: state?.stale ?? true, error: state?.error || null });
+      cb({ ok: true });
+    } catch (err) {
+      logger.warn('Room camera subscribe failed', { socketId: socket.id, cameraId, err: err.message });
+      cb({ error: err.message });
+    }
+  });
+
+  socket.on('roomCamera:unsubscribe', (payload = {}) => {
+    const cameraId = payload.roomCameraId || payload.id;
+    if (!cameraId) return;
+    removeSubscription(socket.id, String(cameraId));
+  });
+
+  socket.on('disconnect', () => {
+    removeAllSubscriptions(socket.id);
+  });
+});

server/src/services/videoSocketService.js

@@ -4,7 +4,6 @@ const { getMode, MODES } = require('./modeManager');
 const { isAdmin, isLockdownAdmin, getRole } = require('./roleService');
 const videoSessions = require('./videoSessions');
 const roverManager = require('./roverManager');
-const { getRoomCamera } = require('./roomCameraService');
 const { loadConfig } = require('../helpers/configLoader');
 
 const config = loadConfig();
@@ -94,12 +93,7 @@ io.on('connection', (socket) => {
           throw new Error('Not authorized for video');
         }
       } else if (target.type === 'room') {
-        if (!getRoomCamera(target.id)) {
-          throw new Error('Unknown room camera');
-        }
-        if (!canViewRoomCamera(socket)) {
-          throw new Error('Not authorized for room camera');
-        }
+        throw new Error('Room cameras now use the snapshot feed');
       } else {
         throw new Error('Unsupported video source');
       }

webui/src/components/RoomCameraFeed.jsx

@@ -1,145 +1,25 @@
-import { useCallback, useEffect, useRef, useState } from 'react';
-import { WhepPlayer } from '../lib/whepPlayer.js';
+import { useMemo } from 'react';
 
-const RESTART_DELAY_MS = 2000;
-const UNMUTE_RETRY_MS = 3000;
-
-export default function RoomCameraFeed({ sessionInfo, label }) {
-  const videoRef = useRef(null);
-  const restartTimer = useRef(null);
-  const unmuteTimer = useRef(null);
-  const [status, setStatus] = useState('idle');
-  const [detail, setDetail] = useState(null);
-  const [restartToken, setRestartToken] = useState(0);
-  const [muted, setMuted] = useState(true);
-
-  const scheduleRestart = useCallback(() => {
-    clearTimeout(restartTimer.current);
-    restartTimer.current = setTimeout(() => setRestartToken(Date.now()), RESTART_DELAY_MS);
-  }, []);
-
-  const ensurePlayback = useCallback(async () => {
-    const video = videoRef.current;
-    if (!video) return;
-    try {
-      video.muted = true;
-      await video.play();
-    } catch {
-      // Autoplay might be blocked; retry later.
-    }
-  }, []);
-
-  const attemptUnmute = useCallback(
-    (delay = 0) => {
-      clearTimeout(unmuteTimer.current);
-
-      const scheduleRetry = () => {
-        clearTimeout(unmuteTimer.current);
-        unmuteTimer.current = setTimeout(() => {
-          tryPlay();
-        }, UNMUTE_RETRY_MS);
-      };
-
-      const tryPlay = async () => {
-        const video = videoRef.current;
-        if (!video) return;
-        try {
-          await ensurePlayback();
-          video.muted = false;
-          await video.play();
-          setMuted(false);
-        } catch {
-          video.muted = true;
-          setMuted(true);
-          scheduleRetry();
-        }
-      };
-
-      unmuteTimer.current = setTimeout(tryPlay, delay);
-    },
-    [ensurePlayback],
-  );
-
-  useEffect(
-    () => () => {
-      clearTimeout(restartTimer.current);
-      clearTimeout(unmuteTimer.current);
-    },
-    [],
-  );
-
-  useEffect(() => {
-    if (!sessionInfo?.url || !videoRef.current) {
-      return undefined;
-    }
-    let active = true;
-    let player;
-    const resetMuteId = setTimeout(() => setMuted(true), 0);
-    const handleStatus = (nextStatus, info) => {
-      if (!active) return;
-      setStatus(nextStatus);
-      setDetail(info || null);
-      if (nextStatus === 'playing') {
-        ensurePlayback();
-        attemptUnmute(0);
-      }
-      if (['error', 'failed', 'disconnected', 'closed'].includes(nextStatus)) {
-        scheduleRestart();
-      }
-    };
-
-    player = new WhepPlayer({
-      url: sessionInfo.url,
-      token: sessionInfo.token,
-      video: videoRef.current,
-      onStatus: handleStatus,
-    });
-
-    player.start().catch((err) => {
-      if (!active) return;
-      setStatus('error');
-      setDetail(err.message);
-      scheduleRestart();
-    });
-
-    return () => {
-      active = false;
-      clearTimeout(resetMuteId);
-      player?.stop();
-    };
-  }, [sessionInfo?.url, sessionInfo?.token, restartToken, scheduleRestart, ensurePlayback, attemptUnmute]);
-
-  useEffect(() => {
-    if (status === 'stopped' && sessionInfo?.url) {
-      scheduleRestart();
-    }
-  }, [status, sessionInfo?.url, scheduleRestart]);
-
-  const renderedStatus = sessionInfo?.error
-    ? `Error: ${sessionInfo.error}`
-    : !sessionInfo?.url
-    ? 'Waiting for stream session'
-    : status === 'error'
-    ? `Error: ${detail || 'unknown'}`
-    : detail
-    ? `${status} (${detail})`
-    : status;
+export default function RoomCameraFeed({ feed, label }) {
+  const statusText = useMemo(() => {
+    if (!feed) return 'Connecting…';
+    if (feed.error) return `Error: ${feed.error}`;
+    if (feed.status === 'playing' && feed.stale) return 'Stale frame';
+    return feed.status || 'Connecting…';
+  }, [feed]);
 
   return (
-    <div className="relative aspect-video w-full overflow-hidden rounded bg-black">
-      <video
-        ref={videoRef}
-        muted={muted}
-        playsInline
-        autoPlay
-        controls={false}
-        className="h-full w-full object-cover"
-      />
+    <div className="relative w-full overflow-hidden rounded bg-black" style={{ aspectRatio: '4 / 3' }}>
+      {feed?.objectUrl ? (
+        <img src={feed.objectUrl} alt={label} className="h-full w-full object-cover" />
+      ) : (
+        <div className="flex h-full w-full items-center justify-center text-sm text-slate-300">Waiting for frame…</div>
+      )}
       <div className="pointer-events-none absolute left-0 top-0 bg-black/70 px-0.5 py-0.5 text-xs font-semibold text-white">
         {label}
       </div>
       <div className="pointer-events-none absolute bottom-0 left-0 m-0.5 rounded bg-black/70 px-0.5 py-0.25 text-[0.7rem] text-slate-100">
-        {renderedStatus}
+        {statusText}
       </div>
     </div>
   );

webui/src/components/RoomCameraPanel.jsx

@@ -1,7 +1,7 @@
 import { useEffect, useState } from 'react';
 import { useSession } from '../context/SessionContext.jsx';
-import { useVideoRequests } from '../hooks/useVideoRequests.js';
 import { useSettingsNamespace } from '../settings/index.js';
+import { useRoomCameraSnapshots } from '../hooks/useRoomCameraSnapshots.js';
 import RoomCameraFeed from './RoomCameraFeed.jsx';
 
 function EmptyState() {
@@ -31,8 +31,7 @@ export default function RoomCameraPanel({
 }) {
   const { session } = useSession();
   const cameras = session?.roomCameras || [];
-  const sourceDescriptors = cameras.map((camera) => ({ type: 'room', id: camera.id, key: `room:${camera.id}` }));
-  const videoSources = useVideoRequests(sourceDescriptors);
+  const feedMap = useRoomCameraSnapshots(cameras.map((camera) => ({ id: camera.id })));
   const { value: orientationSettings, save: saveOrientationSettings } = useSettingsNamespace('roomCameraPanels', {});
   const [orientation, setOrientation] = useState(() =>
     normalizeOrientation(
@@ -94,15 +93,14 @@ export default function RoomCameraPanel({
       )}
       <div className={containerClass}>
         {cameras.map((camera) => {
-          const key = `room:${camera.id}`;
-          const sessionInfo = videoSources[key];
+          const feed = feedMap[camera.id] || null;
           return (
             <article key={camera.id} className="w-full space-y-0.5 rounded bg-zinc-950 p-0.5 shadow-inner shadow-black/40">
               {/* <header className="space-y-0.5">
                 <p className="text-lg font-semibold text-white">{camera.name || camera.id}</p>
                 {camera.description && <p className="text-xs text-slate-500">{camera.description}</p>}
               </header> */}
-              <RoomCameraFeed sessionInfo={sessionInfo} label={camera.name || camera.id} />
+              <RoomCameraFeed feed={feed} label={camera.name || camera.id} />
             </article>
           );
         })}

webui/src/hooks/useRoomCameraSnapshots.js

@@ -0,0 +1,118 @@
+import { useEffect, useMemo, useRef, useState } from 'react';
+import { useSocket } from '../context/SocketContext.jsx';
+
+function normalizeCamera(entry) {
+  if (!entry) return null;
+  if (typeof entry === 'string') {
+    return { id: entry, key: entry };
+  }
+  if (entry.id) {
+    return { id: String(entry.id), key: entry.id };
+  }
+  return null;
+}
+
+function dedupe(entries = []) {
+  const seen = new Set();
+  const unique = [];
+  entries.forEach((entry) => {
+    if (!entry?.key || seen.has(entry.key)) return;
+    seen.add(entry.key);
+    unique.push(entry);
+  });
+  return unique.sort((a, b) => a.key.localeCompare(b.key));
+}
+
+export function useRoomCameraSnapshots(sourceList = []) {
+  const socket = useSocket();
+  const [feeds, setFeeds] = useState({});
+  const objectUrls = useRef(new Map());
+  const normalizedEntries = useMemo(() => dedupe(sourceList.map(normalizeCamera).filter(Boolean)), [sourceList]);
+  const entriesKey = useMemo(() => normalizedEntries.map((e) => e.key).join('|'), [normalizedEntries]);
+
+  useEffect(() => {
+    objectUrls.current.forEach((url) => URL.revokeObjectURL(url));
+    objectUrls.current.clear();
+    setFeeds({});
+  }, [entriesKey]);
+
+  useEffect(() => {
+    if (!normalizedEntries.length || !socket) {
+      return undefined;
+    }
+    let cancelled = false;
+
+    const handleFrame = (meta = {}, buffer) => {
+      if (cancelled || !meta.id || !buffer) return;
+      const blob = new Blob([buffer], { type: 'image/jpeg' });
+      const url = URL.createObjectURL(blob);
+      const prevUrl = objectUrls.current.get(meta.id);
+      if (prevUrl) {
+        URL.revokeObjectURL(prevUrl);
+      }
+      objectUrls.current.set(meta.id, url);
+      setFeeds((prev) => ({
+        ...prev,
+        [meta.id]: {
+          status: 'playing',
+          stale: !!meta.stale,
+          ts: meta.ts || Date.now(),
+          error: null,
+          objectUrl: url,
+        },
+      }));
+    };
+
+    const handleStatus = (meta = {}) => {
+      if (cancelled || !meta.id) return;
+      setFeeds((prev) => ({
+        ...prev,
+        [meta.id]: {
+          ...(prev[meta.id] || {}),
+          status: meta.error ? 'error' : prev[meta.id]?.status || 'connecting',
+          error: meta.error || null,
+          stale: meta.stale ?? prev[meta.id]?.stale ?? true,
+          ts: meta.ts || prev[meta.id]?.ts || null,
+          objectUrl: prev[meta.id]?.objectUrl || null,
+        },
+      }));
+    };
+
+    socket.on('roomCamera:frame', handleFrame);
+    socket.on('roomCamera:status', handleStatus);
+
+    normalizedEntries.forEach((entry) => {
+      socket.emit('roomCamera:subscribe', { roomCameraId: entry.id }, (resp = {}) => {
+        if (resp.error) {
+          handleStatus({ id: entry.id, error: resp.error });
+        } else {
+          handleStatus({ id: entry.id, stale: true });
+        }
+      });
+    });
+
+    return () => {
+      cancelled = true;
+      normalizedEntries.forEach((entry) => {
+        socket.emit('roomCamera:unsubscribe', { roomCameraId: entry.id });
+      });
+      socket.off('roomCamera:frame', handleFrame);
+      socket.off('roomCamera:status', handleStatus);
+      objectUrls.current.forEach((url) => URL.revokeObjectURL(url));
+      objectUrls.current.clear();
+    };
+  }, [socket, normalizedEntries, entriesKey]);
+
+  const filtered = useMemo(() => {
+    if (!entriesKey) return {};
+    const next = {};
+    normalizedEntries.forEach((entry) => {
+      if (feeds[entry.id]) {
+        next[entry.id] = feeds[entry.id];
+      }
+    });
+    return next;
+  }, [feeds, entriesKey, normalizedEntries]);
+
+  return filtered;
+}