health service and panel

server/src/services/discordBotService.js

@@ -295,14 +295,18 @@ function resolveReplaySources(query) {
   return { sources };
 }
 
-function buildReplayCaption(requester, sources = []) {
+function buildReplayCaption(requester, sources = [], missingSources = []) {
   const requesterLabel = requester || 'unknown';
   const sourceLabel = sources.length
     ? `Sources: ${sources.map((source) => source.label || `${source.type}:${source.id}`).join(', ')}.`
     : 'No sources.';
+  const missingLabel = missingSources.length
+    ? `Missing: ${missingSources.map((source) => source.label || `${source.type}:${source.id}`).join(', ')}.`
+    : null;
   return [
     `Replay requested by ${requesterLabel}.`,
     sourceLabel,
+    missingLabel,
     buildDriverCaption(),
   ]
     .filter(Boolean)
@@ -313,9 +317,9 @@ async function sendReplayToChannel(channelId, requester, sources = []) {
   if (!channelId) {
     throw new Error('Replay channel not configured');
   }
-  const buffer = await buildReplayVideo({ sources });
+  const { buffer, usedSources, missingSources } = await buildReplayVideo({ sources });
   const attachment = new AttachmentBuilder(buffer, { name: 'replay.mp4' });
-  const caption = buildReplayCaption(requester, sources);
+  const caption = buildReplayCaption(requester, usedSources, missingSources);
   await sendToChannel(channelId, caption, { files: [attachment] }, { parse: [] });
 }
 
@@ -351,9 +355,9 @@ async function handleReplayCommand(message, query) {
   const requester =
     message.member?.nickname || message.author?.globalName || message.author?.username || 'Discord';
   try {
-    const buffer = await buildReplayVideo({ sources });
+    const { buffer, usedSources, missingSources } = await buildReplayVideo({ sources });
     const attachment = new AttachmentBuilder(buffer, { name: 'replay.mp4' });
-    const caption = buildReplayCaption(requester, sources);
+    const caption = buildReplayCaption(requester, usedSources, missingSources);
     await message.reply({
       content: sanitizeMentions(caption),
       files: [attachment],

server/src/services/healthService.js

@@ -0,0 +1,126 @@
+const fsp = require('fs/promises');
+const path = require('path');
+const roverManager = require('./roverManager');
+const { getRoomCameras } = require('./roomCameraService');
+const { getRoomCameraState } = require('./roomCameraSnapshotService');
+const { getReplaySources } = require('./replaySourceService');
+const { replaySegmentsDir, segmentSeconds, bufferSeconds } = require('./replaySegmentManager');
+
+const ROVER_SNAPSHOT_DIR = process.env.ROVER_SNAPSHOT_DIR || '/var/lib/rover-snapshots';
+const HEALTH_INTERVAL_MS = 5000;
+const ROOM_CAMERA_STALE_MS = 5000;
+const ROVER_SNAPSHOT_STALE_MS = 5000;
+
+let latest = {
+  updatedAt: Date.now(),
+  replay: { sources: [], readyCount: 0, totalCount: 0 },
+  snapshots: { rovers: [], rooms: [] },
+};
+
+async function collectReplayHealth(now) {
+  const neededCount = Math.max(1, Math.ceil(20000 / (segmentSeconds * 1000)));
+  const sources = getReplaySources();
+  const list = [];
+  let readyCount = 0;
+  for (const source of sources) {
+    const key = `${source.type}__${source.id}`;
+    const dir = path.join(replaySegmentsDir, key);
+    let lastSegmentAt = null;
+    let recentCount = 0;
+    try {
+      const entries = await fsp.readdir(dir, { withFileTypes: true });
+      for (const entry of entries) {
+        if (!entry.isFile() || !entry.name.endsWith('.mp4')) continue;
+        const stat = await fsp.stat(path.join(dir, entry.name));
+        if (stat.mtimeMs > (lastSegmentAt || 0)) {
+          lastSegmentAt = stat.mtimeMs;
+        }
+        if (now - stat.mtimeMs <= bufferSeconds * 1000) {
+          recentCount += 1;
+        }
+      }
+    } catch {
+      // directory missing or unreadable
+    }
+    const ready = recentCount >= neededCount;
+    if (ready) readyCount += 1;
+    list.push({
+      type: source.type,
+      id: source.id,
+      label: source.label || `${source.type}:${source.id}`,
+      recentCount,
+      neededCount,
+      lastSegmentAt,
+      ready,
+    });
+  }
+  return { sources: list, readyCount, totalCount: list.length };
+}
+
+async function collectSnapshotHealth(now) {
+  const rovers = roverManager.getRoster().map((rover) => ({
+    id: String(rover.id),
+    name: rover.name || rover.id,
+  }));
+  const roverSnapshots = [];
+  for (const rover of rovers) {
+    const filePath = path.join(ROVER_SNAPSHOT_DIR, `${rover.id}.jpg`);
+    let exists = false;
+    let size = 0;
+    let updatedAt = null;
+    try {
+      const stat = await fsp.stat(filePath);
+      exists = true;
+      size = stat.size;
+      updatedAt = stat.mtimeMs;
+    } catch {
+      // missing snapshot
+    }
+    const ageMs = updatedAt ? now - updatedAt : null;
+    const stale = ageMs != null ? ageMs > ROVER_SNAPSHOT_STALE_MS : true;
+    roverSnapshots.push({
+      id: rover.id,
+      name: rover.name,
+      exists,
+      size,
+      updatedAt,
+      ageMs,
+      stale,
+    });
+  }
+
+  const roomSnapshots = getRoomCameras().map((camera) => {
+    const state = getRoomCameraState(camera.id);
+    const updatedAt = state?.ts || null;
+    const ageMs = updatedAt ? now - updatedAt : null;
+    const stale = ageMs != null ? ageMs > ROOM_CAMERA_STALE_MS : true;
+    return {
+      id: camera.id,
+      name: camera.name || camera.id,
+      updatedAt,
+      ageMs,
+      error: state?.error || null,
+      stale,
+    };
+  });
+
+  return { rovers: roverSnapshots, rooms: roomSnapshots };
+}
+
+async function refreshHealth() {
+  const now = Date.now();
+  const replay = await collectReplayHealth(now);
+  const snapshots = await collectSnapshotHealth(now);
+  latest = { updatedAt: now, replay, snapshots };
+}
+
+refreshHealth();
+setInterval(refreshHealth, HEALTH_INTERVAL_MS);
+
+function getHealthSnapshot() {
+  return latest;
+}
+
+module.exports = {
+  getHealthSnapshot,
+};

server/src/services/replayBuildService.js

@@ -87,22 +87,29 @@ async function buildReplayVideo({ sources = [] } = {}) {
   const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'rover-replay-'));
   try {
     const clipPaths = [];
+    const usedSources = [];
+    const missingSources = [];
     for (let i = 0; i < sources.length; i += 1) {
       const source = sources[i];
       const key = `${source.type}__${source.id}`;
       let segmentPaths;
       try {
         segmentPaths = await listLatestSegments(key, segmentCount);
-      } catch {
-        throw new Error(`Missing replay segments for ${source.type}:${source.id}`);
+      } catch (err) {
+        missingSources.push({ ...source, reason: err.message || 'missing segments' });
+        continue;
       }
       if (segmentPaths.length < segmentCount) {
-        throw new Error(`Not enough replay segments for ${source.type}:${source.id}`);
+        missingSources.push({
+          ...source,
+          reason: `only ${segmentPaths.length}/${segmentCount} segments available`,
+        });
+        continue;
       }
-      const concatPath = path.join(tmpDir, `concat-${i}.txt`);
+      const concatPath = path.join(tmpDir, `concat-${clipPaths.length}.txt`);
       const concatBody = segmentPaths.map((file) => `file '${file}'`).join('\n');
       await fsp.writeFile(concatPath, concatBody);
-      const clipPath = path.join(tmpDir, `clip-${i}.mp4`);
+      const clipPath = path.join(tmpDir, `clip-${clipPaths.length}.mp4`);
       await execFileAsync('ffmpeg', [
         '-hide_banner',
         '-loglevel',
@@ -118,6 +125,15 @@ async function buildReplayVideo({ sources = [] } = {}) {
         clipPath,
       ]);
       clipPaths.push(clipPath);
+      usedSources.push(source);
+    }
+    if (!clipPaths.length) {
+      throw new Error('No replay segments available for selected sources');
+    }
+    if (missingSources.length) {
+      logger.warn('Replay sources missing segments', {
+        missing: missingSources.map((source) => `${source.type}:${source.id}`),
+      });
     }
 
     const { maxWidth, maxHeight } = await probeMaxFrameSize(clipPaths);
@@ -188,7 +204,7 @@ async function buildReplayVideo({ sources = [] } = {}) {
       outPath,
     ]);
     const buffer = await fsp.readFile(outPath);
-    return buffer;
+    return { buffer, usedSources, missingSources };
   } finally {
     try {
       await fsp.rm(tmpDir, { recursive: true, force: true });

server/src/services/sessionService.js

@@ -11,6 +11,7 @@ const { getState: getHomeAssistantState, homeAssistantEvents } = require('./home
 const { getNickname, nicknameEvents } = require('./nicknameService');
 const { getReplayState, replayEvents } = require('./replayService');
 const { getReplaySources } = require('./replaySourceService');
+const { getHealthSnapshot } = require('./healthService');
 const { loadConfig } = require('../helpers/configLoader');
 
 const discordInvite = loadConfig().discord?.invite || null;
@@ -51,6 +52,7 @@ function buildSession(socket) {
     homeAssistant: getHomeAssistantState(),
     replay: getReplayState(),
     replaySources: getReplaySources(),
+    health: getHealthSnapshot(),
     users,
     discord: {
       invite: discordInvite,